A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc][Typo] type annotation fix (#4495)
harrywu committed
May 1, 2024 at 11:21 UTC
f458112e8afdb01bd3cb2e435db314c6bc227973
1 file changed
+3
-2
vllm/engine/llm_engine.py
+3
-2
@@ -8,7 +8,8 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
8
LoRAConfig, ModelConfig, ParallelConfig,
9
SchedulerConfig, SpeculativeConfig,
10
VisionLanguageConfig)
11
-from vllm.core.scheduler import Scheduler, SchedulerOutputs
11
+from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
12
+ SchedulerOutputs)
13
from vllm.engine.arg_utils import EngineArgs
14
from vllm.engine.metrics import StatLogger, Stats
15
from vllm.engine.output_processor.interfaces import (
@@ -485,7 +486,7 @@ class LLMEngine:
486
def _process_model_outputs(
487
self,
488
output: List[SamplerOutput],
488
- scheduled_seq_groups: List[SequenceGroup],
489
+ scheduled_seq_groups: List[ScheduledSequenceGroup],
490
ignored_seq_groups: List[SequenceGroup],
491
seq_group_metadata_list: List[SequenceGroupMetadata],
492
) -> List[RequestOutput]: