@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 18bd7587b7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[1/N] pass the complete config from engine to executor (#9933)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 1, 2024 at 13:51 UTC 18bd7587b78b3b9868fea29d59ae8c3600c3e5a5
6 files changed +65 -137
vllm/engine/async_llm_engine.py
+1 -1
@@ -680,7 +680,7 @@ class AsyncLLMEngine(EngineClient):
680
681 # Create the async LLM engine.
682 engine = cls(
683 - **engine_config.to_dict(),
683 + vllm_config=engine_config,
684 executor_class=executor_class,
685 log_requests=not engine_args.disable_log_requests,
686 log_stats=not engine_args.disable_log_stats,
vllm/engine/llm_engine.py
+21 -29
@@ -13,11 +13,8 @@ import torch
13 from typing_extensions import TypeIs, TypeVar
14
15 import vllm.envs as envs
16 -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
17 - EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
18 - ObservabilityConfig, ParallelConfig,
19 - PromptAdapterConfig, SchedulerConfig,
20 - SpeculativeConfig)
16 +from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
17 + ObservabilityConfig, ParallelConfig, SchedulerConfig)
18 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
19 SchedulerOutputs)
20 from vllm.engine.arg_utils import EngineArgs
@@ -222,17 +219,7 @@ class LLMEngine:
219
220 def __init__(
221 self,
225 - model_config: ModelConfig,
226 - cache_config: CacheConfig,
227 - parallel_config: ParallelConfig,
228 - scheduler_config: SchedulerConfig,
229 - device_config: DeviceConfig,
230 - load_config: LoadConfig,
231 - lora_config: Optional[LoRAConfig],
232 - speculative_config: Optional[SpeculativeConfig],
233 - decoding_config: Optional[DecodingConfig],
234 - observability_config: Optional[ObservabilityConfig],
235 - prompt_adapter_config: Optional[PromptAdapterConfig],
222 + vllm_config: EngineConfig,
223 executor_class: Type[ExecutorBase],
224 log_stats: bool,
225 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -240,6 +227,22 @@ class LLMEngine:
227 input_registry: InputRegistry = INPUT_REGISTRY,
228 use_cached_outputs: bool = False,
229 ) -> None:
230 +
231 + # TODO: remove the local variables and use self.* throughout the class.
232 + model_config = self.model_config = vllm_config.model_config
233 + cache_config = self.cache_config = vllm_config.cache_config
234 + lora_config = self.lora_config = vllm_config.lora_config
235 + parallel_config = self.parallel_config = vllm_config.parallel_config
236 + scheduler_config = self.scheduler_config = vllm_config.scheduler_config
237 + device_config = self.device_config = vllm_config.device_config
238 + speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
239 + load_config = self.load_config = vllm_config.load_config
240 + decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
241 + )
242 + prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
243 + observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
244 + )
245 +
246 logger.info(
247 "Initializing an LLM engine (v%s) with config: "
248 "model=%r, speculative_config=%r, tokenizer=%r, "
@@ -340,18 +343,7 @@ class LLMEngine:
343 self.input_processor = input_registry.create_input_processor(
344 model_config)
345
343 - self.model_executor = executor_class(
344 - model_config=model_config,
345 - cache_config=cache_config,
346 - parallel_config=parallel_config,
347 - scheduler_config=scheduler_config,
348 - device_config=device_config,
349 - lora_config=lora_config,
350 - speculative_config=speculative_config,
351 - load_config=load_config,
352 - prompt_adapter_config=prompt_adapter_config,
353 - observability_config=self.observability_config,
354 - )
346 + self.model_executor = executor_class(vllm_config=vllm_config, )
347
348 if self.model_config.task != "embedding":
349 self._initialize_kv_caches()
@@ -582,7 +574,7 @@ class LLMEngine:
574 executor_class = cls._get_executor_cls(engine_config)
575 # Create the LLM engine.
576 engine = cls(
585 - **engine_config.to_dict(),
577 + vllm_config=engine_config,
578 executor_class=executor_class,
579 log_stats=not engine_args.disable_log_stats,
580 usage_context=usage_context,
vllm/engine/multiprocessing/engine.py
+1 -6
@@ -7,8 +7,6 @@ import cloudpickle
7 import zmq
8
9 from vllm import AsyncEngineArgs, SamplingParams
10 -from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11 - ParallelConfig, SchedulerConfig)
10 # yapf conflicts with isort for this block
11 # yapf: disable
12 from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
@@ -30,9 +28,6 @@ if VLLM_USE_V1:
28 else:
29 from vllm.engine.llm_engine import LLMEngine
30
33 -CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
34 - SchedulerConfig, LoRAConfig]
35 -
31 logger = init_logger(__name__)
32
33 POLLING_TIMEOUT_MS = 10000
@@ -130,7 +125,7 @@ class MQLLMEngine:
125
126 return cls(ipc_path=ipc_path,
127 use_async_sockets=use_async_sockets,
133 - **engine_config.to_dict(),
128 + vllm_config=engine_config,
129 executor_class=executor_class,
130 log_requests=not engine_args.disable_log_requests,
131 log_stats=not engine_args.disable_log_stats,
vllm/executor/executor_base.py
+13 -24
@@ -1,10 +1,7 @@
1 from abc import ABC, abstractmethod
2 from typing import List, Optional, Set, Tuple
3
4 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5 - ModelConfig, ObservabilityConfig, ParallelConfig,
6 - PromptAdapterConfig, SchedulerConfig,
7 - SpeculativeConfig)
4 +from vllm.config import EngineConfig
5 from vllm.lora.request import LoRARequest
6 from vllm.model_executor.layers.sampler import SamplerOutput
7 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -23,27 +20,19 @@ class ExecutorBase(ABC):
20
21 def __init__(
22 self,
26 - model_config: ModelConfig,
27 - cache_config: CacheConfig,
28 - parallel_config: ParallelConfig,
29 - scheduler_config: SchedulerConfig,
30 - device_config: DeviceConfig,
31 - load_config: LoadConfig,
32 - lora_config: Optional[LoRAConfig],
33 - speculative_config: Optional[SpeculativeConfig],
34 - prompt_adapter_config: Optional[PromptAdapterConfig],
35 - observability_config: Optional[ObservabilityConfig],
23 + vllm_config: EngineConfig,
24 ) -> None:
37 - self.model_config = model_config
38 - self.cache_config = cache_config
39 - self.lora_config = lora_config
40 - self.load_config = load_config
41 - self.parallel_config = parallel_config
42 - self.scheduler_config = scheduler_config
43 - self.device_config = device_config
44 - self.speculative_config = speculative_config
45 - self.prompt_adapter_config = prompt_adapter_config
46 - self.observability_config = observability_config
25 + self.vllm_config = vllm_config
26 + self.model_config = vllm_config.model_config
27 + self.cache_config = vllm_config.cache_config
28 + self.lora_config = vllm_config.lora_config
29 + self.load_config = vllm_config.load_config
30 + self.parallel_config = vllm_config.parallel_config
31 + self.scheduler_config = vllm_config.scheduler_config
32 + self.device_config = vllm_config.device_config
33 + self.speculative_config = vllm_config.speculative_config
34 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
35 + self.observability_config = vllm_config.observability_config
36 self._init_executor()
37
38 @abstractmethod
vllm/executor/xpu_executor.py
+8 -36
@@ -2,10 +2,7 @@ from typing import Callable, List, Optional, Tuple, Type, Union
2
3 import torch
4
5 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ObservabilityConfig, ParallelConfig,
7 - PromptAdapterConfig, SchedulerConfig,
8 - SpeculativeConfig)
5 +from vllm.config import ModelConfig, ParallelConfig
6 from vllm.executor.executor_base import ExecutorAsyncBase
7 from vllm.executor.gpu_executor import GPUExecutor
8 from vllm.logger import init_logger
@@ -21,38 +18,13 @@ class XPUExecutor(GPUExecutor):
18
19 uses_ray: bool = False
20
24 - def __init__(
25 - self,
26 - model_config: ModelConfig,
27 - cache_config: CacheConfig,
28 - parallel_config: ParallelConfig,
29 - scheduler_config: SchedulerConfig,
30 - device_config: DeviceConfig,
31 - load_config: LoadConfig,
32 - lora_config: Optional[LoRAConfig],
33 - prompt_adapter_config: Optional[PromptAdapterConfig],
34 - speculative_config: Optional[SpeculativeConfig],
35 - observability_config: Optional[ObservabilityConfig],
36 - ) -> None:
37 - assert device_config.device_type == "xpu"
38 - assert (not speculative_config
39 - ), "Speculative decoding not yet supported for XPU backend"
40 -
41 - model_config = _verify_and_get_model_config(model_config)
42 -
43 - self.model_config = model_config
44 - self.cache_config = cache_config
45 - self.load_config = load_config
46 - self.lora_config = lora_config
47 - self.parallel_config = _verify_and_get_parallel_config(parallel_config)
48 - self.scheduler_config = scheduler_config
49 - self.device_config = device_config
50 - self.prompt_adapter_config = prompt_adapter_config
51 - self.speculative_config = None
52 - self.observability_config = observability_config
53 -
54 - # Instantiate the worker and load the model to GPU.
55 - self._init_executor()
21 + def _init_executor(self) -> None:
22 + assert self.device_config.device_type == "xpu"
23 + assert self.speculative_config is None, (
24 + "Speculative decoding not yet supported for XPU backend")
25 +
26 + self.model_config = _verify_and_get_model_config(self.model_config)
27 + GPUExecutor._init_executor(self)
28
29 def _get_worker_module_and_class(
30 self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
vllm/v1/engine/llm_engine.py
+21 -41
@@ -2,11 +2,8 @@ import time
2 from typing import (Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type,
3 Union)
4
5 -from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
6 - EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
7 - ObservabilityConfig, ParallelConfig,
8 - PromptAdapterConfig, SchedulerConfig,
9 - SpeculativeConfig)
5 +from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
6 + ObservabilityConfig, ParallelConfig, SchedulerConfig)
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.metrics_types import StatLoggerBase
9 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
@@ -35,17 +32,7 @@ class LLMEngine:
32
33 def __init__(
34 self,
38 - model_config: ModelConfig,
39 - cache_config: CacheConfig,
40 - parallel_config: ParallelConfig,
41 - scheduler_config: SchedulerConfig,
42 - device_config: DeviceConfig,
43 - load_config: LoadConfig,
44 - lora_config: Optional[LoRAConfig],
45 - speculative_config: Optional[SpeculativeConfig],
46 - decoding_config: Optional[DecodingConfig],
47 - observability_config: Optional[ObservabilityConfig],
48 - prompt_adapter_config: Optional[PromptAdapterConfig],
35 + vllm_config: EngineConfig,
36 executor_class: Type[GPUExecutor],
37 log_stats: bool,
38 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -53,6 +40,22 @@ class LLMEngine:
40 input_registry: InputRegistry = INPUT_REGISTRY,
41 use_cached_outputs: bool = False,
42 ) -> None:
43 +
44 + # TODO: remove the local variables and use self.* throughout the class.
45 + model_config = self.model_config = vllm_config.model_config
46 + cache_config = self.cache_config = vllm_config.cache_config
47 + lora_config = self.lora_config = vllm_config.lora_config
48 + parallel_config = self.parallel_config = vllm_config.parallel_config
49 + scheduler_config = self.scheduler_config = vllm_config.scheduler_config
50 + device_config = self.device_config = vllm_config.device_config
51 + speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
52 + load_config = self.load_config = vllm_config.load_config
53 + decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
54 + )
55 + prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
56 + observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
57 + )
58 +
59 # Override the configs for V1.
60 # FIXME
61 if usage_context == UsageContext.LLM_CLASS:
@@ -112,18 +115,6 @@ class LLMEngine:
115 model_config.mm_processor_kwargs,
116 )
117
115 - self.model_config = model_config
116 - self.cache_config = cache_config
117 - self.lora_config = lora_config
118 - self.parallel_config = parallel_config
119 - self.scheduler_config = scheduler_config
120 - self.device_config = device_config
121 - self.speculative_config = speculative_config
122 - self.load_config = load_config
123 - self.decoding_config = decoding_config or DecodingConfig()
124 - self.prompt_adapter_config = prompt_adapter_config
125 - self.observability_config = observability_config or ObservabilityConfig(
126 - )
118 self.log_stats = log_stats
119
120 assert not self.model_config.skip_tokenizer_init
@@ -154,18 +145,7 @@ class LLMEngine:
145 # Request id -> RequestOutput
146 self.request_outputs: Dict[str, RequestOutput] = {}
147
157 - self.model_executor = executor_class(
158 - model_config=model_config,
159 - cache_config=cache_config,
160 - parallel_config=parallel_config,
161 - scheduler_config=scheduler_config,
162 - device_config=device_config,
163 - lora_config=lora_config,
164 - speculative_config=speculative_config,
165 - load_config=load_config,
166 - prompt_adapter_config=prompt_adapter_config,
167 - observability_config=self.observability_config,
168 - )
148 + self.model_executor = executor_class(vllm_config=vllm_config)
149 assert self.model_config.task != "embedding"
150 self._initialize_kv_caches()
151
@@ -203,7 +183,7 @@ class LLMEngine:
183 executor_class = cls._get_executor_cls(engine_config)
184 # Create the LLM engine.
185 engine = cls(
206 - **engine_config.to_dict(),
186 + vllm_config=engine_config,
187 executor_class=executor_class,
188 log_stats=not engine_args.disable_log_stats,
189 usage_context=usage_context,