@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f458112e8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc][Typo] type annotation fix (#4495)

harrywu committed May 1, 2024 at 11:21 UTC f458112e8afdb01bd3cb2e435db314c6bc227973
1 file changed +3 -2
vllm/engine/llm_engine.py
+3 -2
@@ -8,7 +8,8 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
8 LoRAConfig, ModelConfig, ParallelConfig,
9 SchedulerConfig, SpeculativeConfig,
10 VisionLanguageConfig)
11 -from vllm.core.scheduler import Scheduler, SchedulerOutputs
11 +from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
12 + SchedulerOutputs)
13 from vllm.engine.arg_utils import EngineArgs
14 from vllm.engine.metrics import StatLogger, Stats
15 from vllm.engine.output_processor.interfaces import (
@@ -485,7 +486,7 @@ class LLMEngine:
486 def _process_model_outputs(
487 self,
488 output: List[SamplerOutput],
488 - scheduled_seq_groups: List[SequenceGroup],
489 + scheduled_seq_groups: List[ScheduledSequenceGroup],
490 ignored_seq_groups: List[SequenceGroup],
491 seq_group_metadata_list: List[SequenceGroupMetadata],
492 ) -> List[RequestOutput]: