@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 776dbd74f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[CI/Build] mypy: Resolve some errors from checking vllm/engine (#9267)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Oct 16, 2024 at 18:55 UTC 776dbd74f1d6a42a1e71c3b18a0d28e61f2e9ea5
20 files changed +109 -74
tools/mypy.sh
+1 -11
@@ -13,24 +13,14 @@ run_mypy() {
13
14 run_mypy # Note that this is less strict than CI
15 run_mypy tests
16 -run_mypy vllm/assets
16 run_mypy vllm/attention
18 -#run_mypy vllm/compilation
19 -#run_mypy vllm/core
17 +run_mypy vllm/compilation
18 run_mypy vllm/distributed
19 run_mypy vllm/engine
22 -run_mypy vllm/entrypoints
20 run_mypy vllm/executor
24 -#run_mypy vllm/inputs
25 -run_mypy vllm/logging
21 run_mypy vllm/lora
22 run_mypy vllm/model_executor
28 -run_mypy vllm/multimodal
29 -run_mypy vllm/platforms
23 run_mypy vllm/plugins
24 run_mypy vllm/prompt_adapter
25 run_mypy vllm/spec_decode
33 -run_mypy vllm/transformers_utils
34 -run_mypy vllm/usage
35 -#run_mypy vllm/vllm_flash_attn
26 run_mypy vllm/worker
vllm/attention/layer.py
+1 -1
@@ -92,7 +92,7 @@ class Attention(nn.Module):
92 query: torch.Tensor,
93 key: torch.Tensor,
94 value: torch.Tensor,
95 - kv_cache: Optional[torch.Tensor],
95 + kv_cache: torch.Tensor,
96 attn_metadata: AttentionMetadata,
97 attn_type: AttentionType = AttentionType.DECODER,
98 ) -> torch.Tensor:
vllm/compilation/backends.py
+2 -2
@@ -244,8 +244,8 @@ def vllm_backend(
244
245 def select_default_backend(level: int) -> Union[str, Callable]:
246 if level in [CompilationLevel.DYNAMO_AS_IS, CompilationLevel.DYNAMO_ONCE]:
247 - backend = "eager"
248 - return backend
247 + backend_str = "eager"
248 + return backend_str
249 assert level in [
250 CompilationLevel.INDUCTOR, CompilationLevel.INDUCTOR_MAX_AUTOTUNE
251 ], f"Invalid level {level}"
vllm/compilation/decorators.py
+5 -3
@@ -35,6 +35,8 @@ def support_torch_compile(dynamic_arg_dims: Dict[str, Union[int, List[int]]]):
35 def cls_decorator_helper(cls: type):
36 # helper to pass `dynamic_arg_dims`` to `_support_torch_compile``
37 # to avoid too much indentation for `_support_torch_compile``
38 + if not hasattr(cls, 'forward'):
39 + raise TypeError("decorated class should have a forward method.")
40 sig = inspect.signature(cls.forward)
41 for k in dynamic_arg_dims:
42 if k not in sig.parameters:
@@ -63,13 +65,13 @@ def _support_torch_compile(cls: type,
65 # other than TorchCompileWrapperWithCustomDispatcher
66 cls.__bases__ = cls.__bases__ + (TorchCompileWrapperWithCustomDispatcher, )
67
66 - old_init = cls.__init__
68 + old_init = cls.__init__ # type: ignore
69
70 def __init__(self, *args, **kwargs):
71 old_init(self, *args, **kwargs)
72 TorchCompileWrapperWithCustomDispatcher.__init__(self)
73
72 - cls.__init__ = __init__
74 + cls.__init__ = __init__ # type: ignore
75
76 def __call__(self, *args, **kwargs):
77 # torch.compiler.is_compiling() means we are inside the compilation
@@ -109,5 +111,5 @@ def _support_torch_compile(cls: type,
111 model_output = self.forward(*args, **kwargs)
112 return model_output
113
112 - cls.__call__ = __call__
114 + cls.__call__ = __call__ # type: ignore
115 return cls
vllm/compilation/wrapper.py
+1 -1
@@ -73,7 +73,7 @@ class TorchCompileWrapperWithCustomDispatcher:
73 return
74 # code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25
75 frame = sys._getframe()
76 - while True:
76 + while frame and frame.f_back:
77 frame = frame.f_back
78 code_name = frame.f_code.co_name
79 file_name = frame.f_code.co_filename.split(os.path.sep)[-1]
vllm/config.py
+6 -4
@@ -626,13 +626,14 @@ class CacheConfig:
626 self.sliding_window = sliding_window
627 self.enable_prefix_caching = enable_prefix_caching
628 self.cpu_offload_gb = cpu_offload_gb
629 +
630 self._verify_args()
631 self._verify_cache_dtype()
632 self._verify_prefix_caching()
633
634 # Will be set after profiling.
634 - self.num_gpu_blocks = None
635 - self.num_cpu_blocks = None
635 + self.num_gpu_blocks: Optional[int] = None
636 + self.num_cpu_blocks: Optional[int] = None
637
638 def metrics_info(self):
639 # convert cache_config to dict(key: str, value: str) for prometheus
@@ -709,7 +710,8 @@ class TokenizerPoolConfig:
710
711 @classmethod
712 def create_config(
712 - cls, tokenizer_pool_size: int, tokenizer_pool_type: str,
713 + cls, tokenizer_pool_size: int,
714 + tokenizer_pool_type: Union[str, Type["BaseTokenizerGroup"]],
715 tokenizer_pool_extra_config: Optional[Union[str, dict]]
716 ) -> Optional["TokenizerPoolConfig"]:
717 """Create a TokenizerPoolConfig from the given parameters.
@@ -1544,7 +1546,7 @@ class LoRAConfig:
1546 max_loras: int
1547 fully_sharded_loras: bool = False
1548 max_cpu_loras: Optional[int] = None
1547 - lora_dtype: Optional[torch.dtype] = None
1549 + lora_dtype: Optional[Union[torch.dtype, str]] = None
1550 lora_extra_vocab_size: int = 256
1551 # This is a constant.
1552 lora_vocab_padding_size: ClassVar[int] = 256
vllm/core/scheduler.py
+4 -3
@@ -4,8 +4,9 @@ import random
4 import time
5 from collections import deque
6 from dataclasses import dataclass, field
7 -from typing import (Callable, Deque, Dict, Iterable, List, Optional, Set,
8 - Tuple, Union)
7 +from typing import Callable, Deque, Dict, Iterable, List, Optional
8 +from typing import Sequence as GenericSequence
9 +from typing import Set, Tuple, Union
10
11 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
12 from vllm.core.interfaces import AllocStatus, BlockSpaceManager
@@ -115,7 +116,7 @@ class ScheduledSequenceGroup:
116 class SchedulerOutputs:
117 """The scheduling decision made from a scheduler."""
118 # Scheduled sequence groups.
118 - scheduled_seq_groups: Iterable[ScheduledSequenceGroup]
119 + scheduled_seq_groups: GenericSequence[ScheduledSequenceGroup]
120 # Number of prefill groups scheduled.
121 num_prefill_groups: int
122 # Total number of batched tokens.
vllm/engine/arg_utils.py
+7 -5
@@ -3,7 +3,7 @@ import dataclasses
3 import json
4 from dataclasses import dataclass
5 from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
6 - Tuple, Type, Union)
6 + Tuple, Type, Union, cast)
7
8 import torch
9
@@ -89,7 +89,7 @@ class EngineArgs:
89 trust_remote_code: bool = False
90 download_dir: Optional[str] = None
91 load_format: str = 'auto'
92 - config_format: str = 'auto'
92 + config_format: ConfigFormat = ConfigFormat.AUTO
93 dtype: str = 'auto'
94 kv_cache_dtype: str = 'auto'
95 quantization_param_path: Optional[str] = None
@@ -181,7 +181,7 @@ class EngineArgs:
181 scheduling_policy: Literal["fcfs", "priority"] = "fcfs"
182
183 def __post_init__(self):
184 - if self.tokenizer is None:
184 + if not self.tokenizer:
185 self.tokenizer = self.model
186
187 # Setup plugins
@@ -837,7 +837,8 @@ class EngineArgs:
837 def create_model_config(self) -> ModelConfig:
838 return ModelConfig(
839 model=self.model,
840 - tokenizer=self.tokenizer,
840 + # We know this is not None because we set it in __post_init__
841 + tokenizer=cast(str, self.tokenizer),
842 tokenizer_mode=self.tokenizer_mode,
843 trust_remote_code=self.trust_remote_code,
844 dtype=self.dtype,
@@ -908,8 +909,9 @@ class EngineArgs:
909 self.enable_prefix_caching = False
910
911 cache_config = CacheConfig(
912 + # neuron needs block_size = max_model_len
913 block_size=self.block_size if self.device != "neuron" else
912 - self.max_model_len, # neuron needs block_size = max_model_len
914 + (self.max_model_len if self.max_model_len is not None else 0),
915 gpu_memory_utilization=self.gpu_memory_utilization,
916 swap_space=self.swap_space,
917 cache_dtype=self.kv_cache_dtype,
vllm/engine/llm_engine.py
+12 -8
@@ -6,7 +6,7 @@ from functools import partial
6 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7 Iterable, List, Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
9 -from typing import Set, Type, Union, overload
9 +from typing import Set, Type, Union, cast, overload
10
11 import torch
12 from typing_extensions import TypeVar
@@ -44,7 +44,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
44 from vllm.sampling_params import RequestOutputKind, SamplingParams
45 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
46 Sequence, SequenceGroup, SequenceGroupMetadata,
47 - SequenceStatus)
47 + SequenceGroupOutput, SequenceStatus)
48 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
49 init_tracer)
50 from vllm.transformers_utils.config import try_get_generation_config
@@ -188,7 +188,7 @@ class LLMEngine:
188 raise TypeError(f"Expected output of type {output_type}, "
189 f"but found type {type(output)}")
190
191 - return output
191 + return cast(_O, output)
192
193 @classmethod
194 def validate_outputs(
@@ -1039,6 +1039,7 @@ class LLMEngine:
1039 scheduler_outputs.scheduled_seq_groups)
1040
1041 has_multiple_outputs: bool = len(outputs) > 1
1042 + outputs_by_sequence_group: List[List[SequenceGroupOutput]]
1043 if has_multiple_outputs:
1044 assert self.scheduler_config.is_multi_step or \
1045 self.speculative_config
@@ -1084,6 +1085,7 @@ class LLMEngine:
1085 finished_before.append(i)
1086 continue
1087
1088 + output: List[SequenceGroupOutput]
1089 if has_multiple_outputs:
1090 output = outputs_by_sequence_group[i]
1091 else:
@@ -1096,7 +1098,7 @@ class LLMEngine:
1098 seq_group, seq_group_meta, is_first_step_output)
1099 else:
1100 seq_group.update_num_computed_tokens(
1099 - seq_group_meta.token_chunk_size)
1101 + seq_group_meta.token_chunk_size or 0)
1102
1103 if outputs:
1104 for o in outputs:
@@ -1104,13 +1106,13 @@ class LLMEngine:
1106 and seq_group.metrics is not None):
1107 if seq_group.metrics.model_forward_time is not None:
1108 seq_group.metrics.model_forward_time += (
1107 - o.model_forward_time)
1109 + o.model_forward_time or 0)
1110 else:
1111 seq_group.metrics.model_forward_time = (
1112 o.model_forward_time)
1113 if seq_group.metrics.model_execute_time is not None:
1114 seq_group.metrics.model_execute_time += (
1113 - o.model_execute_time)
1115 + o.model_execute_time or 0)
1116 else:
1117 seq_group.metrics.model_execute_time = (
1118 o.model_execute_time)
@@ -1236,8 +1238,10 @@ class LLMEngine:
1238 seq_group, seq_group_metadata,
1239 seq_group.state.num_steps == 1)
1240 else:
1239 - seq_group.update_num_computed_tokens(
1240 - seq_group_metadata.token_chunk_size)
1241 + token_chunk_size = (seq_group_metadata.token_chunk_size
1242 + if seq_group_metadata.token_chunk_size
1243 + is not None else 0)
1244 + seq_group.update_num_computed_tokens(token_chunk_size)
1245
1246 if seq_group_metadata.do_sample:
1247 assert len(sequence_group_outputs.samples) == 1, (
vllm/engine/metrics.py
+9 -5
@@ -1,6 +1,6 @@
1 from typing import TYPE_CHECKING
2 from typing import Counter as CollectionsCounter
3 -from typing import Dict, List, Optional, Union
3 +from typing import Dict, List, Optional, Type, Union, cast
4
5 import numpy as np
6 import prometheus_client
@@ -249,10 +249,11 @@ class _RayHistogramWrapper:
249 labelnames: Optional[List[str]] = None,
250 buckets: Optional[List[float]] = None):
251 labelnames_tuple = tuple(labelnames) if labelnames else None
252 + boundaries = buckets if buckets else []
253 self._histogram = ray_metrics.Histogram(name=name,
254 description=documentation,
255 tag_keys=labelnames_tuple,
255 - boundaries=buckets)
256 + boundaries=boundaries)
257
258 def labels(self, **labels):
259 self._histogram.set_default_tags(labels)
@@ -267,9 +268,12 @@ class RayMetrics(Metrics):
268 RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
269 Provides the same metrics as Metrics but uses Ray's util.metrics library.
270 """
270 - _gauge_cls = _RayGaugeWrapper
271 - _counter_cls = _RayCounterWrapper
272 - _histogram_cls = _RayHistogramWrapper
271 + _gauge_cls: Type[prometheus_client.Gauge] = cast(
272 + Type[prometheus_client.Gauge], _RayGaugeWrapper)
273 + _counter_cls: Type[prometheus_client.Counter] = cast(
274 + Type[prometheus_client.Counter], _RayCounterWrapper)
275 + _histogram_cls: Type[prometheus_client.Histogram] = cast(
276 + Type[prometheus_client.Histogram], _RayHistogramWrapper)
277
278 def __init__(self, labelnames: List[str], max_model_len: int):
279 if ray_metrics is None:
vllm/engine/multiprocessing/client.py
+12 -5
@@ -3,7 +3,7 @@ import copy
3 import pickle
4 from contextlib import contextmanager, suppress
5 from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
6 - Optional, Union, overload)
6 + Optional, Union, cast, overload)
7
8 import cloudpickle
9 import zmq
@@ -513,9 +513,14 @@ class MQLLMEngineClient(EngineClient):
513 assert (prompt is not None and pooling_params is not None
514 and request_id is not None)
515
516 - return self._process_request(prompt, pooling_params, request_id,
517 - lora_request, trace_headers, None,
518 - priority)
516 + return cast(
517 + AsyncGenerator[EmbeddingRequestOutput, None],
518 + self._process_request(prompt,
519 + pooling_params,
520 + request_id,
521 + lora_request,
522 + trace_headers,
523 + priority=priority))
524
525 async def _process_request(
526 self,
@@ -543,7 +548,9 @@ class MQLLMEngineClient(EngineClient):
548 build_guided_decoding_logits_processor_async(
549 sampling_params=params,
550 tokenizer=await self.get_tokenizer(lora_request),
546 - default_guided_backend=self.decoding_config.guided_decoding_backend
551 + default_guided_backend=(self.decoding_config.guided_decoding_backend
552 + if self.decoding_config
553 + else DecodingConfig.guided_decoding_backend),
554 )
555
556 # 1) Create output queue for this requests.
vllm/engine/multiprocessing/engine.py
+2 -4
@@ -73,11 +73,9 @@ class MQLLMEngine:
73 # For MQLLMEngine, we can use cached outputs, since each new request
74 # output is immediately pickled and send over the socket, which frees
75 # the python object to be reused again.
76 - use_cached_outputs = True
76 + kwargs['use_cached_outputs'] = True
77
78 - self.engine = LLMEngine(*args,
79 - **kwargs,
80 - use_cached_outputs=use_cached_outputs)
78 + self.engine = LLMEngine(*args, **kwargs)
79 self.log_requests = log_requests
80
81 self.use_async_sockets = use_async_sockets
vllm/engine/output_processor/multi_step.py
+19 -6
@@ -1,5 +1,5 @@
1 import functools
2 -from typing import Callable, List
2 +from typing import Callable, List, cast
3
4 from vllm.core.scheduler import Scheduler
5 from vllm.engine.output_processor.interfaces import (
@@ -9,8 +9,10 @@ from vllm.engine.output_processor.single_step import (
9 from vllm.engine.output_processor.stop_checker import StopChecker
10 from vllm.logger import init_logger
11 from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Sequence, SequenceGroup,
13 - SequenceGroupOutput, SequenceOutput, SequenceStatus)
12 +from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
13 + CompletionSequenceGroupOutput, Sequence,
14 + SequenceGroup, SequenceGroupOutput, SequenceOutput,
15 + SequenceStatus)
16 from vllm.transformers_utils.detokenizer import Detokenizer
17 from vllm.transformers_utils.tokenizer import AnyTokenizer
18 from vllm.utils import Counter
@@ -57,6 +59,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
59 """
60 for output in outputs:
61 # Concatenate single-step prompt logprob processing results.
62 + assert isinstance(output, CompletionSequenceGroupOutput)
63 single_step_process_prompt_logprob(self, seq_group, output)
64
65 @staticmethod
@@ -100,8 +103,18 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
103 "Beam search not supported in multi-step decoding.")
104 seq = seqs[0]
105 seq_id = seq.seq_id
103 - assert all(
104 - [seq_id == output.samples[0].parent_seq_id for output in outputs])
106 + # This method is defined in the more generic
107 + # SequenceGroupOutputProcessor, but here we assume that the outputs are
108 + # of a more specific type.
109 + assert all([
110 + isinstance(output, CompletionSequenceGroupOutput)
111 + for output in outputs
112 + ])
113 + compl_outputs = cast(List[CompletionSequenceGroupOutput], outputs)
114 + assert all([
115 + seq_id == output.samples[0].parent_seq_id
116 + for output in compl_outputs
117 + ])
118
119 if is_async:
120 # Async case: We process tokens one by one. Here, we know the token
@@ -113,7 +126,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
126
127 # Since there's only one sequence per sequence group,
128 # we can take the first sample.
116 - samples = [output.samples[0] for output in outputs]
129 + samples = [output.samples[0] for output in compl_outputs]
130
131 # entries in sample tokens may be invalid (eg. due to spec decode
132 # rejecting tokens).
vllm/engine/output_processor/single_step.py
+5 -3
@@ -6,8 +6,9 @@ from vllm.engine.output_processor.interfaces import (
6 SequenceGroupOutputProcessor)
7 from vllm.engine.output_processor.stop_checker import StopChecker
8 from vllm.logger import init_logger
9 -from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
10 - SequenceOutput, SequenceStatus)
9 +from vllm.sequence import (CompletionSequenceGroupOutput, Sequence,
10 + SequenceGroup, SequenceGroupOutput, SequenceOutput,
11 + SequenceStatus)
12 from vllm.transformers_utils.detokenizer import Detokenizer
13 from vllm.utils import Counter
14
@@ -16,7 +17,7 @@ logger = init_logger(__name__)
17
18 def single_step_process_prompt_logprob(
19 sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
19 - output: SequenceGroupOutput) -> None:
20 + output: CompletionSequenceGroupOutput) -> None:
21 """Process prompt logprobs associated with the :class:`SequenceGroupOutput`
22 for a given step.
23
@@ -106,6 +107,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
107 """
108 assert len(outputs) == 1, ("Single step should only has 1 output.")
109 output = outputs[0]
110 + assert isinstance(output, CompletionSequenceGroupOutput)
111 single_step_process_prompt_logprob(self, seq_group, output)
112
113 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
vllm/engine/output_processor/stop_checker.py
+2 -2
@@ -57,7 +57,7 @@ class StopChecker:
57 # Check if a stop token was encountered.
58 # This assumes a single token produced per step.
59 last_token_id = seq.get_last_token_id()
60 - if last_token_id in sampling_params.stop_token_ids:
60 + if last_token_id in (sampling_params.stop_token_ids or ()):
61 if new_char_count and (
62 not sampling_params.include_stop_str_in_output):
63 # Remove last token
@@ -92,7 +92,7 @@ class StopChecker:
92
93 Returns the stop string if matched or else None.
94 """
95 - if not new_char_count:
95 + if not new_char_count or not sampling_params.stop:
96 return None
97
98 for stop_str in sampling_params.stop:
vllm/engine/output_processor/util.py
+8 -5
@@ -1,22 +1,25 @@
1 from typing import List
2 from typing import Sequence as GenericSequence
3 -from typing import Union
3 +from typing import cast
4
5 from vllm.model_executor.layers.sampler import SamplerOutput
6 -from vllm.sequence import PoolerOutput, SequenceGroupOutput
6 +from vllm.sequence import CompletionSequenceGroupOutput, SequenceGroupOutput
7
8
9 def create_output_by_sequence_group(
10 - outputs: GenericSequence[Union[SamplerOutput, PoolerOutput]],
10 + outputs: GenericSequence[SamplerOutput],
11 num_seq_groups: int) -> List[List[SequenceGroupOutput]]:
12 """Helper method which transforms a 2d list organized by
13 [step][sequence group] into [sequence group][step].
14 """
15 - output_by_sequence_group: List[List[SequenceGroupOutput]] = [
15 + output_by_sequence_group: List[List[CompletionSequenceGroupOutput]] = [
16 [] for _ in range(num_seq_groups)
17 ]
18 for step in outputs:
19 + sequence_group_output: CompletionSequenceGroupOutput
20 for i, sequence_group_output in enumerate(step):
21 output_by_sequence_group[i].append(sequence_group_output)
22
22 - return output_by_sequence_group
23 + # Cast to the more generic type that CompletionSequenceGroupOutput
24 + # inherits from.
25 + return cast(List[List[SequenceGroupOutput]], output_by_sequence_group)
vllm/inputs/parse.py
+4 -1
@@ -1,4 +1,4 @@
1 -from typing import List, Literal, Sequence, TypedDict, Union, overload
1 +from typing import List, Literal, Sequence, TypedDict, Union, cast, overload
2
3 from typing_extensions import TypeIs
4
@@ -44,13 +44,16 @@ def parse_and_batch_prompt(
44
45 if is_list_of(prompt, str):
46 # case 2: array of strings
47 + prompt = cast(List[str], prompt)
48 return [
49 ParsedText(content=elem, is_tokens=False) for elem in prompt
50 ]
51 if is_list_of(prompt, int):
52 # case 3: array of tokens
53 + prompt = cast(List[int], prompt)
54 return [ParsedTokens(content=prompt, is_tokens=True)]
55 if is_list_of(prompt, list):
56 + prompt = cast(List[List[int]], prompt)
57 if len(prompt[0]) == 0:
58 raise ValueError("please provide at least one prompt")
59
vllm/model_executor/layers/sampler.py
+5 -2
@@ -4,7 +4,7 @@ import warnings
4 from dataclasses import dataclass
5 from importlib.util import find_spec
6 from math import inf
7 -from typing import Dict, List, Optional, Tuple, Union
7 +from typing import Dict, Iterator, List, Optional, Tuple, Union
8
9 import msgspec
10 import torch
@@ -117,12 +117,15 @@ class SamplerOutput(
117 # block/sync across workers, cpu-gpu sync time and sampling time.
118 model_execute_time: Optional[float] = None
119
120 - def __getitem__(self, idx: int):
120 + def __getitem__(self, idx: int) -> CompletionSequenceGroupOutput:
121 return self.outputs[idx]
122
123 def __setitem__(self, idx: int, value):
124 self.outputs[idx] = value
125
126 + def __iter__(self) -> Iterator[CompletionSequenceGroupOutput]:
127 + return iter(self.outputs)
128 +
129 def __len__(self):
130 return len(self.outputs)
131
vllm/outputs.py
+2 -1
@@ -4,6 +4,7 @@ from typing import List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Union
6
7 +from vllm.inputs import PromptType
8 from vllm.lora.request import LoRARequest
9 from vllm.sampling_params import RequestOutputKind
10 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
@@ -92,7 +93,7 @@ class RequestOutput:
93 def __init__(
94 self,
95 request_id: str,
95 - prompt: Optional[str],
96 + prompt: Optional[PromptType],
97 prompt_token_ids: Optional[List[int]],
98 prompt_logprobs: Optional[PromptLogprobs],
99 outputs: List[CompletionOutput],
vllm/sequence.py
+2 -2
@@ -788,7 +788,7 @@ class SequenceGroup:
788 assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill
789 self.init_multi_step(num_steps=num_lookahead_slots + 1)
790
791 - def get_last_latency(self, now: float) -> Optional[float]:
791 + def get_last_latency(self, now: float) -> float:
792 """Sets the last token time for Request level timings."""
793 # If still in prefill phase, raise Error.
794 if self.is_prefill():
@@ -1198,7 +1198,7 @@ class PoolerOutput(
1198
1199 spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1200
1201 - def __getitem__(self, idx: int):
1201 + def __getitem__(self, idx: int) -> EmbeddingSequenceGroupOutput:
1202 return self.outputs[idx]
1203
1204 def __setitem__(self, idx: int, value):