A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[CI/Build] mypy: Resolve some errors from checking vllm/engine (#9267)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Oct 16, 2024 at 18:55 UTC
776dbd74f1d6a42a1e71c3b18a0d28e61f2e9ea5
20 files changed
+109
-74
tools/mypy.sh
+1
-11
@@ -13,24 +13,14 @@ run_mypy() {
13
14
run_mypy # Note that this is less strict than CI
15
run_mypy tests
16
-run_mypy vllm/assets
16
run_mypy vllm/attention
18
-#run_mypy vllm/compilation
19
-#run_mypy vllm/core
17
+run_mypy vllm/compilation
18
run_mypy vllm/distributed
19
run_mypy vllm/engine
22
-run_mypy vllm/entrypoints
20
run_mypy vllm/executor
24
-#run_mypy vllm/inputs
25
-run_mypy vllm/logging
21
run_mypy vllm/lora
22
run_mypy vllm/model_executor
28
-run_mypy vllm/multimodal
29
-run_mypy vllm/platforms
23
run_mypy vllm/plugins
24
run_mypy vllm/prompt_adapter
25
run_mypy vllm/spec_decode
33
-run_mypy vllm/transformers_utils
34
-run_mypy vllm/usage
35
-#run_mypy vllm/vllm_flash_attn
26
run_mypy vllm/worker
vllm/attention/layer.py
+1
-1
@@ -92,7 +92,7 @@ class Attention(nn.Module):
92
query: torch.Tensor,
93
key: torch.Tensor,
94
value: torch.Tensor,
95
- kv_cache: Optional[torch.Tensor],
95
+ kv_cache: torch.Tensor,
96
attn_metadata: AttentionMetadata,
97
attn_type: AttentionType = AttentionType.DECODER,
98
) -> torch.Tensor:
vllm/compilation/backends.py
+2
-2
@@ -244,8 +244,8 @@ def vllm_backend(
244
245
def select_default_backend(level: int) -> Union[str, Callable]:
246
if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
247
- backend = "eager"
248
- return backend
247
+ backend_str = "eager"
248
+ return backend_str
249
assert level in [
250
CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
251
], f"Invalid level {level}"
vllm/compilation/decorators.py
+5
-3
@@ -35,6 +35,8 @@ def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
35
def cls_decorator_helper(cls: type):
36
# helper to pass `dynamic_arg_dims`` to `_support_torch_compile``
37
# to avoid too much indentation for `_support_torch_compile``
38
+ if not hasattr(cls, 'forward'):
39
+ raise TypeError("decorated class should have a forward method.")
40
sig = inspect.signature(cls.forward)
41
for k in dynamic_arg_dims:
42
if k not in sig.parameters:
@@ -63,13 +65,13 @@ def _support_torch_compile(cls: type,
65
# other than TorchCompileWrapperWithCustomDispatcher
66
cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
67
66
- old_init = cls.__init__
68
+ old_init = cls.__init__ # type: ignore
69
70
def __init__(self, *args, **kwargs):
71
old_init(self, *args, **kwargs)
72
TorchCompileWrapperWithCustomDispatcher.__init__(self)
73
72
- cls.__init__ = __init__
74
+ cls.__init__ = __init__ # type: ignore
75
76
def __call__(self, *args, **kwargs):
77
# torch.compiler.is_compiling() means we are inside the compilation
@@ -109,5 +111,5 @@ def _support_torch_compile(cls: type,
111
model_output = self.forward(*args, **kwargs)
112
return model_output
113
112
- cls.__call__ = __call__
114
+ cls.__call__ = __call__ # type: ignore
115
return cls
vllm/compilation/wrapper.py
+1
-1
@@ -73,7 +73,7 @@ class TorchCompileWrapperWithCustomDispatcher:
73
return
74
# code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25
75
frame = sys._getframe()
76
- while True:
76
+ while frame and frame.f_back:
77
frame = frame.f_back
78
code_name = frame.f_code.co_name
79
file_name = frame.f_code.co_filename.split(os.path.sep)[-1]
vllm/config.py
+6
-4
@@ -626,13 +626,14 @@ class CacheConfig:
626
self.sliding_window = sliding_window
627
self.enable_prefix_caching = enable_prefix_caching
628
self.cpu_offload_gb = cpu_offload_gb
629
+
630
self._verify_args()
631
self._verify_cache_dtype()
632
self._verify_prefix_caching()
633
634
# Will be set after profiling.
634
- self.num_gpu_blocks = None
635
- self.num_cpu_blocks = None
635
+ self.num_gpu_blocks: Optional[int] = None
636
+ self.num_cpu_blocks: Optional[int] = None
637
638
def metrics_info(self):
639
# convert cache_config to dict(key: str, value: str) for prometheus
@@ -709,7 +710,8 @@ class TokenizerPoolConfig:
710
711
@classmethod
712
def create_config(
712
- cls, tokenizer_pool_size: int, tokenizer_pool_type: str,
713
+ cls, tokenizer_pool_size: int,
714
+ tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]],
715
tokenizer_pool_extra_config: Optional[Union[str, dict]]
716
) -> Optional["TokenizerPoolConfig"]:
717
"""Create a TokenizerPoolConfig from the given parameters.
@@ -1544,7 +1546,7 @@ class LoRAConfig:
1546
max_loras: int
1547
fully_sharded_loras: bool = False
1548
max_cpu_loras: Optional[int] = None
1547
- lora_dtype: Optional[torch.dtype] = None
1549
+ lora_dtype: Optional[Union[torch.dtype, str]] = None
1550
lora_extra_vocab_size: int = 256
1551
# This is a constant.
1552
lora_vocab_padding_size: ClassVar[int] = 256
vllm/core/scheduler.py
+4
-3
@@ -4,8 +4,9 @@ import random
4
import time
5
from collections import deque
6
from dataclasses import dataclass, field
7
-from typing import (Callable, Deque, Dict, Iterable, List, Optional, Set,
8
- Tuple, Union)
7
+from typing import Callable, Deque, Dict, Iterable, List, Optional
8
+from typing import Sequence as GenericSequence
9
+from typing import Set, Tuple, Union
10
11
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
12
from vllm.core.interfaces import AllocStatus, BlockSpaceManager
@@ -115,7 +116,7 @@ class ScheduledSequenceGroup:
116
class SchedulerOutputs:
117
"""The scheduling decision made from a scheduler."""
118
# Scheduled sequence groups.
118
- scheduled_seq_groups: Iterable[ScheduledSequenceGroup]
119
+ scheduled_seq_groups: GenericSequence[ScheduledSequenceGroup]
120
# Number of prefill groups scheduled.
121
num_prefill_groups: int
122
# Total number of batched tokens.
vllm/engine/arg_utils.py
+7
-5
@@ -3,7 +3,7 @@ import dataclasses
3
import json
4
from dataclasses import dataclass
5
from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
6
- Tuple, Type, Union)
6
+ Tuple, Type, Union, cast)
7
8
import torch
9
@@ -89,7 +89,7 @@ class EngineArgs:
89
trust_remote_code: bool = False
90
download_dir: Optional[str] = None
91
load_format: str = 'auto'
92
- config_format: str = 'auto'
92
+ config_format: ConfigFormat = ConfigFormat.AUTO
93
dtype: str = 'auto'
94
kv_cache_dtype: str = 'auto'
95
quantization_param_path: Optional[str] = None
@@ -181,7 +181,7 @@ class EngineArgs:
181
scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
182
183
def __post_init__(self):
184
- if self.tokenizer is None:
184
+ if not self.tokenizer:
185
self.tokenizer = self.model
186
187
# Setup plugins
@@ -837,7 +837,8 @@ class EngineArgs:
837
def create_model_config(self) -> ModelConfig:
838
return ModelConfig(
839
model=self.model,
840
- tokenizer=self.tokenizer,
840
+ # We know this is not None because we set it in __post_init__
841
+ tokenizer=cast(str, self.tokenizer),
842
tokenizer_mode=self.tokenizer_mode,
843
trust_remote_code=self.trust_remote_code,
844
dtype=self.dtype,
@@ -908,8 +909,9 @@ class EngineArgs:
909
self.enable_prefix_caching = False
910
911
cache_config = CacheConfig(
912
+ # neuron needs block_size = max_model_len
913
block_size=self.block_size if self.device != "neuron" else
912
- self.max_model_len, # neuron needs block_size = max_model_len
914
+ (self.max_model_len if self.max_model_len is not None else 0),
915
gpu_memory_utilization=self.gpu_memory_utilization,
916
swap_space=self.swap_space,
917
cache_dtype=self.kv_cache_dtype,
vllm/engine/llm_engine.py
+12
-8
@@ -6,7 +6,7 @@ from functools import partial
6
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7
Iterable, List, Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
9
-from typing import Set, Type, Union, overload
9
+from typing import Set, Type, Union, cast, overload
10
11
import torch
12
from typing_extensions import TypeVar
@@ -44,7 +44,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
44
from vllm.sampling_params import RequestOutputKind, SamplingParams
45
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
46
Sequence, SequenceGroup, SequenceGroupMetadata,
47
- SequenceStatus)
47
+ SequenceGroupOutput, SequenceStatus)
48
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
49
init_tracer)
50
from vllm.transformers_utils.config import try_get_generation_config
@@ -188,7 +188,7 @@ class LLMEngine:
188
raise TypeError(f"Expected output of type {output_type}, "
189
f"but found type {type(output)}")
190
191
- return output
191
+ return cast(_O, output)
192
193
@classmethod
194
def validate_outputs(
@@ -1039,6 +1039,7 @@ class LLMEngine:
1039
scheduler_outputs.scheduled_seq_groups)
1040
1041
has_multiple_outputs: bool = len(outputs) > 1
1042
+ outputs_by_sequence_group: List[List[SequenceGroupOutput]]
1043
if has_multiple_outputs:
1044
assert self.scheduler_config.is_multi_step or \
1045
self.speculative_config
@@ -1084,6 +1085,7 @@ class LLMEngine:
1085
finished_before.append(i)
1086
continue
1087
1088
+ output: List[SequenceGroupOutput]
1089
if has_multiple_outputs:
1090
output = outputs_by_sequence_group[i]
1091
else:
@@ -1096,7 +1098,7 @@ class LLMEngine:
1098
seq_group, seq_group_meta, is_first_step_output)
1099
else:
1100
seq_group.update_num_computed_tokens(
1099
- seq_group_meta.token_chunk_size)
1101
+ seq_group_meta.token_chunk_size or 0)
1102
1103
if outputs:
1104
for o in outputs:
@@ -1104,13 +1106,13 @@ class LLMEngine:
1106
and seq_group.metrics is not None):
1107
if seq_group.metrics.model_forward_time is not None:
1108
seq_group.metrics.model_forward_time += (
1107
- o.model_forward_time)
1109
+ o.model_forward_time or 0)
1110
else:
1111
seq_group.metrics.model_forward_time = (
1112
o.model_forward_time)
1113
if seq_group.metrics.model_execute_time is not None:
1114
seq_group.metrics.model_execute_time += (
1113
- o.model_execute_time)
1115
+ o.model_execute_time or 0)
1116
else:
1117
seq_group.metrics.model_execute_time = (
1118
o.model_execute_time)
@@ -1236,8 +1238,10 @@ class LLMEngine:
1238
seq_group, seq_group_metadata,
1239
seq_group.state.num_steps == 1)
1240
else:
1239
- seq_group.update_num_computed_tokens(
1240
- seq_group_metadata.token_chunk_size)
1241
+ token_chunk_size = (seq_group_metadata.token_chunk_size
1242
+ if seq_group_metadata.token_chunk_size
1243
+ is not None else 0)
1244
+ seq_group.update_num_computed_tokens(token_chunk_size)
1245
1246
if seq_group_metadata.do_sample:
1247
assert len(sequence_group_outputs.samples) == 1, (
vllm/engine/metrics.py
+9
-5
@@ -1,6 +1,6 @@
1
from typing import TYPE_CHECKING
2
from typing import Counter as CollectionsCounter
3
-from typing import Dict, List, Optional, Union
3
+from typing import Dict, List, Optional, Type, Union, cast
4
5
import numpy as np
6
import prometheus_client
@@ -249,10 +249,11 @@ class _RayHistogramWrapper:
249
labelnames: Optional[List[str]] = None,
250
buckets: Optional[List[float]] = None):
251
labelnames_tuple = tuple(labelnames) if labelnames else None
252
+ boundaries = buckets if buckets else []
253
self._histogram = ray_metrics.Histogram(name=name,
254
description=documentation,
255
tag_keys=labelnames_tuple,
255
- boundaries=buckets)
256
+ boundaries=boundaries)
257
258
def labels(self, **labels):
259
self._histogram.set_default_tags(labels)
@@ -267,9 +268,12 @@ class RayMetrics(Metrics):
268
RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
269
Provides the same metrics as Metrics but uses Ray's util.metrics library.
270
"""
270
- _gauge_cls = _RayGaugeWrapper
271
- _counter_cls = _RayCounterWrapper
272
- _histogram_cls = _RayHistogramWrapper
271
+ _gauge_cls: Type[prometheus_client.Gauge] = cast(
272
+ Type[prometheus_client.Gauge], _RayGaugeWrapper)
273
+ _counter_cls: Type[prometheus_client.Counter] = cast(
274
+ Type[prometheus_client.Counter], _RayCounterWrapper)
275
+ _histogram_cls: Type[prometheus_client.Histogram] = cast(
276
+ Type[prometheus_client.Histogram], _RayHistogramWrapper)
277
278
def __init__(self, labelnames: List[str], max_model_len: int):
279
if ray_metrics is None:
vllm/engine/multiprocessing/client.py
+12
-5
@@ -3,7 +3,7 @@ import copy
3
import pickle
4
from contextlib import contextmanager, suppress
5
from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
6
- Optional, Union, overload)
6
+ Optional, Union, cast, overload)
7
8
import cloudpickle
9
import zmq
@@ -513,9 +513,14 @@ class MQLLMEngineClient(EngineClient):
513
assert (prompt is not None and pooling_params is not None
514
and request_id is not None)
515
516
- return self._process_request(prompt, pooling_params, request_id,
517
- lora_request, trace_headers, None,
518
- priority)
516
+ return cast(
517
+ AsyncGenerator[EmbeddingRequestOutput, None],
518
+ self._process_request(prompt,
519
+ pooling_params,
520
+ request_id,
521
+ lora_request,
522
+ trace_headers,
523
+ priority=priority))
524
525
async def _process_request(
526
self,
@@ -543,7 +548,9 @@ class MQLLMEngineClient(EngineClient):
548
build_guided_decoding_logits_processor_async(
549
sampling_params=params,
550
tokenizer=await self.get_tokenizer(lora_request),
546
- default_guided_backend=self.decoding_config.guided_decoding_backend
551
+ default_guided_backend=(self.decoding_config.guided_decoding_backend
552
+ if self.decoding_config
553
+ else DecodingConfig.guided_decoding_backend),
554
)
555
556
# 1) Create output queue for this requests.
vllm/engine/multiprocessing/engine.py
+2
-4
@@ -73,11 +73,9 @@ class MQLLMEngine:
73
# For MQLLMEngine, we can use cached outputs, since each new request
74
# output is immediately pickled and send over the socket, which frees
75
# the python object to be reused again.
76
- use_cached_outputs = True
76
+ kwargs['use_cached_outputs'] = True
77
78
- self.engine = LLMEngine(*args,
79
- **kwargs,
80
- use_cached_outputs=use_cached_outputs)
78
+ self.engine = LLMEngine(*args, **kwargs)
79
self.log_requests = log_requests
80
81
self.use_async_sockets = use_async_sockets
vllm/engine/output_processor/multi_step.py
+19
-6
@@ -1,5 +1,5 @@
1
import functools
2
-from typing import Callable, List
2
+from typing import Callable, List, cast
3
4
from vllm.core.scheduler import Scheduler
5
from vllm.engine.output_processor.interfaces import (
@@ -9,8 +9,10 @@ from vllm.engine.output_processor.single_step import (
9
from vllm.engine.output_processor.stop_checker import StopChecker
10
from vllm.logger import init_logger
11
from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Sequence, SequenceGroup,
13
- SequenceGroupOutput, SequenceOutput, SequenceStatus)
12
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
13
+ CompletionSequenceGroupOutput, Sequence,
14
+ SequenceGroup, SequenceGroupOutput, SequenceOutput,
15
+ SequenceStatus)
16
from vllm.transformers_utils.detokenizer import Detokenizer
17
from vllm.transformers_utils.tokenizer import AnyTokenizer
18
from vllm.utils import Counter
@@ -57,6 +59,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
59
"""
60
for output in outputs:
61
# Concatenate single-step prompt logprob processing results.
62
+ assert isinstance(output, CompletionSequenceGroupOutput)
63
single_step_process_prompt_logprob(self, seq_group, output)
64
65
@staticmethod
@@ -100,8 +103,18 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
103
"Beam search not supported in multi-step decoding.")
104
seq = seqs[0]
105
seq_id = seq.seq_id
103
- assert all(
104
- [seq_id == output.samples[0].parent_seq_id for output in outputs])
106
+ # This method is defined in the more generic
107
+ # SequenceGroupOutputProcessor, but here we assume that the outputs are
108
+ # of a more specific type.
109
+ assert all([
110
+ isinstance(output, CompletionSequenceGroupOutput)
111
+ for output in outputs
112
+ ])
113
+ compl_outputs = cast(List[CompletionSequenceGroupOutput], outputs)
114
+ assert all([
115
+ seq_id == output.samples[0].parent_seq_id
116
+ for output in compl_outputs
117
+ ])
118
119
if is_async:
120
# Async case: We process tokens one by one. Here, we know the token
@@ -113,7 +126,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
126
127
# Since there's only one sequence per sequence group,
128
# we can take the first sample.
116
- samples = [output.samples[0] for output in outputs]
129
+ samples = [output.samples[0] for output in compl_outputs]
130
131
# entries in sample tokens may be invalid (eg. due to spec decode
132
# rejecting tokens).
vllm/engine/output_processor/single_step.py
+5
-3
@@ -6,8 +6,9 @@ from vllm.engine.output_processor.interfaces import (
6
SequenceGroupOutputProcessor)
7
from vllm.engine.output_processor.stop_checker import StopChecker
8
from vllm.logger import init_logger
9
-from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
10
- SequenceOutput, SequenceStatus)
9
+from vllm.sequence import (CompletionSequenceGroupOutput, Sequence,
10
+ SequenceGroup, SequenceGroupOutput, SequenceOutput,
11
+ SequenceStatus)
12
from vllm.transformers_utils.detokenizer import Detokenizer
13
from vllm.utils import Counter
14
@@ -16,7 +17,7 @@ logger = init_logger(__name__)
17
18
def single_step_process_prompt_logprob(
19
sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
19
- output: SequenceGroupOutput) -> None:
20
+ output: CompletionSequenceGroupOutput) -> None:
21
"""Process prompt logprobs associated with the :class:`SequenceGroupOutput`
22
for a given step.
23
@@ -106,6 +107,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
107
"""
108
assert len(outputs) == 1, ("Single step should only has 1 output.")
109
output = outputs[0]
110
+ assert isinstance(output, CompletionSequenceGroupOutput)
111
single_step_process_prompt_logprob(self, seq_group, output)
112
113
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
vllm/engine/output_processor/stop_checker.py
+2
-2
@@ -57,7 +57,7 @@ class StopChecker:
57
# Check if a stop token was encountered.
58
# This assumes a single token produced per step.
59
last_token_id = seq.get_last_token_id()
60
- if last_token_id in sampling_params.stop_token_ids:
60
+ if last_token_id in (sampling_params.stop_token_ids or ()):
61
if new_char_count and (
62
not sampling_params.include_stop_str_in_output):
63
# Remove last token
@@ -92,7 +92,7 @@ class StopChecker:
92
93
Returns the stop string if matched or else None.
94
"""
95
- if not new_char_count:
95
+ if not new_char_count or not sampling_params.stop:
96
return None
97
98
for stop_str in sampling_params.stop:
vllm/engine/output_processor/util.py
+8
-5
@@ -1,22 +1,25 @@
1
from typing import List
2
from typing import Sequence as GenericSequence
3
-from typing import Union
3
+from typing import cast
4
5
from vllm.model_executor.layers.sampler import SamplerOutput
6
-from vllm.sequence import PoolerOutput, SequenceGroupOutput
6
+from vllm.sequence import CompletionSequenceGroupOutput, SequenceGroupOutput
7
8
9
def create_output_by_sequence_group(
10
- outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]],
10
+ outputs: GenericSequence[SamplerOutput],
11
num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
12
"""Helper method which transforms a 2d list organized by
13
[step][sequence group] into [sequence group][step].
14
"""
15
- output_by_sequence_group: List[List[SequenceGroupOutput]] = [
15
+ output_by_sequence_group: List[List[CompletionSequenceGroupOutput]] = [
16
[] for _ in range(num_seq_groups)
17
]
18
for step in outputs:
19
+ sequence_group_output: CompletionSequenceGroupOutput
20
for i, sequence_group_output in enumerate(step):
21
output_by_sequence_group[i].append(sequence_group_output)
22
22
- return output_by_sequence_group
23
+ # Cast to the more generic type that CompletionSequenceGroupOutput
24
+ # inherits from.
25
+ return cast(List[List[SequenceGroupOutput]], output_by_sequence_group)
vllm/inputs/parse.py
+4
-1
@@ -1,4 +1,4 @@
1
-from typing import List, Literal, Sequence, TypedDict, Union, overload
1
+from typing import List, Literal, Sequence, TypedDict, Union, cast, overload
2
3
from typing_extensions import TypeIs
4
@@ -44,13 +44,16 @@ def parse_and_batch_prompt(
44
45
if is_list_of(prompt, str):
46
# case 2: array of strings
47
+ prompt = cast(List[str], prompt)
48
return [
49
ParsedText(content=elem, is_tokens=False) for elem in prompt
50
]
51
if is_list_of(prompt, int):
52
# case 3: array of tokens
53
+ prompt = cast(List[int], prompt)
54
return [ParsedTokens(content=prompt, is_tokens=True)]
55
if is_list_of(prompt, list):
56
+ prompt = cast(List[List[int]], prompt)
57
if len(prompt[0]) == 0:
58
raise ValueError("please provide at least one prompt")
59
vllm/model_executor/layers/sampler.py
+5
-2
@@ -4,7 +4,7 @@ import warnings
4
from dataclasses import dataclass
5
from importlib.util import find_spec
6
from math import inf
7
-from typing import Dict, List, Optional, Tuple, Union
7
+from typing import Dict, Iterator, List, Optional, Tuple, Union
8
9
import msgspec
10
import torch
@@ -117,12 +117,15 @@ class SamplerOutput(
117
# block/sync across workers, cpu-gpu sync time and sampling time.
118
model_execute_time: Optional[float] = None
119
120
- def __getitem__(self, idx: int):
120
+ def __getitem__(self, idx: int) -> CompletionSequenceGroupOutput:
121
return self.outputs[idx]
122
123
def __setitem__(self, idx: int, value):
124
self.outputs[idx] = value
125
126
+ def __iter__(self) -> Iterator[CompletionSequenceGroupOutput]:
127
+ return iter(self.outputs)
128
+
129
def __len__(self):
130
return len(self.outputs)
131
vllm/outputs.py
+2
-1
@@ -4,6 +4,7 @@ from typing import List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Union
6
7
+from vllm.inputs import PromptType
8
from vllm.lora.request import LoRARequest
9
from vllm.sampling_params import RequestOutputKind
10
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -92,7 +93,7 @@ class RequestOutput:
93
def __init__(
94
self,
95
request_id: str,
95
- prompt: Optional[str],
96
+ prompt: Optional[PromptType],
97
prompt_token_ids: Optional[List[int]],
98
prompt_logprobs: Optional[PromptLogprobs],
99
outputs: List[CompletionOutput],
vllm/sequence.py
+2
-2
@@ -788,7 +788,7 @@ class SequenceGroup:
788
assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill
789
self.init_multi_step(num_steps=num_lookahead_slots + 1)
790
791
- def get_last_latency(self, now: float) -> Optional[float]:
791
+ def get_last_latency(self, now: float) -> float:
792
"""Sets the last token time for Request level timings."""
793
# If still in prefill phase, raise Error.
794
if self.is_prefill():
@@ -1198,7 +1198,7 @@ class PoolerOutput(
1198
1199
spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1200
1201
- def __getitem__(self, idx: int):
1201
+ def __getitem__(self, idx: int) -> EmbeddingSequenceGroupOutput:
1202
return self.outputs[idx]
1203
1204
def __setitem__(self, idx: int, value):