A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[1/N] pass the complete config from engine to executor (#9933)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 1, 2024 at 13:51 UTC
18bd7587b78b3b9868fea29d59ae8c3600c3e5a5
6 files changed
+65
-137
vllm/engine/async_llm_engine.py
+1
-1
@@ -680,7 +680,7 @@ class AsyncLLMEngine(EngineClient):
680
681
# Create the async LLM engine.
682
engine = cls(
683
- **engine_config.to_dict(),
683
+ vllm_config=engine_config,
684
executor_class=executor_class,
685
log_requests=not engine_args.disable_log_requests,
686
log_stats=not engine_args.disable_log_stats,
vllm/engine/llm_engine.py
+21
-29
@@ -13,11 +13,8 @@ import torch
13
from typing_extensions import TypeIs, TypeVar
14
15
import vllm.envs as envs
16
-from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
17
- EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
18
- ObservabilityConfig, ParallelConfig,
19
- PromptAdapterConfig, SchedulerConfig,
20
- SpeculativeConfig)
16
+from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
17
+ ObservabilityConfig, ParallelConfig, SchedulerConfig)
18
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
19
SchedulerOutputs)
20
from vllm.engine.arg_utils import EngineArgs
@@ -222,17 +219,7 @@ class LLMEngine:
219
220
def __init__(
221
self,
225
- model_config: ModelConfig,
226
- cache_config: CacheConfig,
227
- parallel_config: ParallelConfig,
228
- scheduler_config: SchedulerConfig,
229
- device_config: DeviceConfig,
230
- load_config: LoadConfig,
231
- lora_config: Optional[LoRAConfig],
232
- speculative_config: Optional[SpeculativeConfig],
233
- decoding_config: Optional[DecodingConfig],
234
- observability_config: Optional[ObservabilityConfig],
235
- prompt_adapter_config: Optional[PromptAdapterConfig],
222
+ vllm_config: EngineConfig,
223
executor_class: Type[ExecutorBase],
224
log_stats: bool,
225
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -240,6 +227,22 @@ class LLMEngine:
227
input_registry: InputRegistry = INPUT_REGISTRY,
228
use_cached_outputs: bool = False,
229
) -> None:
230
+
231
+ # TODO: remove the local variables and use self.* throughout the class.
232
+ model_config = self.model_config = vllm_config.model_config
233
+ cache_config = self.cache_config = vllm_config.cache_config
234
+ lora_config = self.lora_config = vllm_config.lora_config
235
+ parallel_config = self.parallel_config = vllm_config.parallel_config
236
+ scheduler_config = self.scheduler_config = vllm_config.scheduler_config
237
+ device_config = self.device_config = vllm_config.device_config
238
+ speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
239
+ load_config = self.load_config = vllm_config.load_config
240
+ decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
241
+ )
242
+ prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
243
+ observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
244
+ )
245
+
246
logger.info(
247
"Initializing an LLM engine (v%s) with config: "
248
"model=%r, speculative_config=%r, tokenizer=%r, "
@@ -340,18 +343,7 @@ class LLMEngine:
343
self.input_processor = input_registry.create_input_processor(
344
model_config)
345
343
- self.model_executor = executor_class(
344
- model_config=model_config,
345
- cache_config=cache_config,
346
- parallel_config=parallel_config,
347
- scheduler_config=scheduler_config,
348
- device_config=device_config,
349
- lora_config=lora_config,
350
- speculative_config=speculative_config,
351
- load_config=load_config,
352
- prompt_adapter_config=prompt_adapter_config,
353
- observability_config=self.observability_config,
354
- )
346
+ self.model_executor = executor_class(vllm_config=vllm_config, )
347
348
if self.model_config.task != "embedding":
349
self._initialize_kv_caches()
@@ -582,7 +574,7 @@ class LLMEngine:
574
executor_class = cls._get_executor_cls(engine_config)
575
# Create the LLM engine.
576
engine = cls(
585
- **engine_config.to_dict(),
577
+ vllm_config=engine_config,
578
executor_class=executor_class,
579
log_stats=not engine_args.disable_log_stats,
580
usage_context=usage_context,
vllm/engine/multiprocessing/engine.py
+1
-6
@@ -7,8 +7,6 @@ import cloudpickle
7
import zmq
8
9
from vllm import AsyncEngineArgs, SamplingParams
10
-from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11
- ParallelConfig, SchedulerConfig)
10
# yapf conflicts with isort for this block
11
# yapf: disable
12
from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
@@ -30,9 +28,6 @@ if VLLM_USE_V1:
28
else:
29
from vllm.engine.llm_engine import LLMEngine
30
33
-CONFIG_TYPE = Union[ModelConfig, DecodingConfig, ParallelConfig,
34
- SchedulerConfig, LoRAConfig]
35
-
31
logger = init_logger(__name__)
32
33
POLLING_TIMEOUT_MS = 10000
@@ -130,7 +125,7 @@ class MQLLMEngine:
125
126
return cls(ipc_path=ipc_path,
127
use_async_sockets=use_async_sockets,
133
- **engine_config.to_dict(),
128
+ vllm_config=engine_config,
129
executor_class=executor_class,
130
log_requests=not engine_args.disable_log_requests,
131
log_stats=not engine_args.disable_log_stats,
vllm/executor/executor_base.py
+13
-24
@@ -1,10 +1,7 @@
1
from abc import ABC, abstractmethod
2
from typing import List, Optional, Set, Tuple
3
4
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5
- ModelConfig, ObservabilityConfig, ParallelConfig,
6
- PromptAdapterConfig, SchedulerConfig,
7
- SpeculativeConfig)
4
+from vllm.config import EngineConfig
5
from vllm.lora.request import LoRARequest
6
from vllm.model_executor.layers.sampler import SamplerOutput
7
from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -23,27 +20,19 @@ class ExecutorBase(ABC):
20
21
def __init__(
22
self,
26
- model_config: ModelConfig,
27
- cache_config: CacheConfig,
28
- parallel_config: ParallelConfig,
29
- scheduler_config: SchedulerConfig,
30
- device_config: DeviceConfig,
31
- load_config: LoadConfig,
32
- lora_config: Optional[LoRAConfig],
33
- speculative_config: Optional[SpeculativeConfig],
34
- prompt_adapter_config: Optional[PromptAdapterConfig],
35
- observability_config: Optional[ObservabilityConfig],
23
+ vllm_config: EngineConfig,
24
) -> None:
37
- self.model_config = model_config
38
- self.cache_config = cache_config
39
- self.lora_config = lora_config
40
- self.load_config = load_config
41
- self.parallel_config = parallel_config
42
- self.scheduler_config = scheduler_config
43
- self.device_config = device_config
44
- self.speculative_config = speculative_config
45
- self.prompt_adapter_config = prompt_adapter_config
46
- self.observability_config = observability_config
25
+ self.vllm_config = vllm_config
26
+ self.model_config = vllm_config.model_config
27
+ self.cache_config = vllm_config.cache_config
28
+ self.lora_config = vllm_config.lora_config
29
+ self.load_config = vllm_config.load_config
30
+ self.parallel_config = vllm_config.parallel_config
31
+ self.scheduler_config = vllm_config.scheduler_config
32
+ self.device_config = vllm_config.device_config
33
+ self.speculative_config = vllm_config.speculative_config
34
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
35
+ self.observability_config = vllm_config.observability_config
36
self._init_executor()
37
38
@abstractmethod
vllm/executor/xpu_executor.py
+8
-36
@@ -2,10 +2,7 @@ from typing import Callable, List, Optional, Tuple, Type, Union
2
3
import torch
4
5
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ObservabilityConfig, ParallelConfig,
7
- PromptAdapterConfig, SchedulerConfig,
8
- SpeculativeConfig)
5
+from vllm.config import ModelConfig, ParallelConfig
6
from vllm.executor.executor_base import ExecutorAsyncBase
7
from vllm.executor.gpu_executor import GPUExecutor
8
from vllm.logger import init_logger
@@ -21,38 +18,13 @@ class XPUExecutor(GPUExecutor):
18
19
uses_ray: bool = False
20
24
- def __init__(
25
- self,
26
- model_config: ModelConfig,
27
- cache_config: CacheConfig,
28
- parallel_config: ParallelConfig,
29
- scheduler_config: SchedulerConfig,
30
- device_config: DeviceConfig,
31
- load_config: LoadConfig,
32
- lora_config: Optional[LoRAConfig],
33
- prompt_adapter_config: Optional[PromptAdapterConfig],
34
- speculative_config: Optional[SpeculativeConfig],
35
- observability_config: Optional[ObservabilityConfig],
36
- ) -> None:
37
- assert device_config.device_type == "xpu"
38
- assert (not speculative_config
39
- ), "Speculative decoding not yet supported for XPU backend"
40
-
41
- model_config = _verify_and_get_model_config(model_config)
42
-
43
- self.model_config = model_config
44
- self.cache_config = cache_config
45
- self.load_config = load_config
46
- self.lora_config = lora_config
47
- self.parallel_config = _verify_and_get_parallel_config(parallel_config)
48
- self.scheduler_config = scheduler_config
49
- self.device_config = device_config
50
- self.prompt_adapter_config = prompt_adapter_config
51
- self.speculative_config = None
52
- self.observability_config = observability_config
53
-
54
- # Instantiate the worker and load the model to GPU.
55
- self._init_executor()
21
+ def _init_executor(self) -> None:
22
+ assert self.device_config.device_type == "xpu"
23
+ assert self.speculative_config is None, (
24
+ "Speculative decoding not yet supported for XPU backend")
25
+
26
+ self.model_config = _verify_and_get_model_config(self.model_config)
27
+ GPUExecutor._init_executor(self)
28
29
def _get_worker_module_and_class(
30
self) -> Tuple[str, str, Optional[Callable[[], Type[WorkerBase]]]]:
vllm/v1/engine/llm_engine.py
+21
-41
@@ -2,11 +2,8 @@ import time
2
from typing import (Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type,
3
Union)
4
5
-from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
6
- EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
7
- ObservabilityConfig, ParallelConfig,
8
- PromptAdapterConfig, SchedulerConfig,
9
- SpeculativeConfig)
5
+from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
6
+ ObservabilityConfig, ParallelConfig, SchedulerConfig)
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.metrics_types import StatLoggerBase
9
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs,
@@ -35,17 +32,7 @@ class LLMEngine:
32
33
def __init__(
34
self,
38
- model_config: ModelConfig,
39
- cache_config: CacheConfig,
40
- parallel_config: ParallelConfig,
41
- scheduler_config: SchedulerConfig,
42
- device_config: DeviceConfig,
43
- load_config: LoadConfig,
44
- lora_config: Optional[LoRAConfig],
45
- speculative_config: Optional[SpeculativeConfig],
46
- decoding_config: Optional[DecodingConfig],
47
- observability_config: Optional[ObservabilityConfig],
48
- prompt_adapter_config: Optional[PromptAdapterConfig],
35
+ vllm_config: EngineConfig,
36
executor_class: Type[GPUExecutor],
37
log_stats: bool,
38
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -53,6 +40,22 @@ class LLMEngine:
40
input_registry: InputRegistry = INPUT_REGISTRY,
41
use_cached_outputs: bool = False,
42
) -> None:
43
+
44
+ # TODO: remove the local variables and use self.* throughout the class.
45
+ model_config = self.model_config = vllm_config.model_config
46
+ cache_config = self.cache_config = vllm_config.cache_config
47
+ lora_config = self.lora_config = vllm_config.lora_config
48
+ parallel_config = self.parallel_config = vllm_config.parallel_config
49
+ scheduler_config = self.scheduler_config = vllm_config.scheduler_config
50
+ device_config = self.device_config = vllm_config.device_config
51
+ speculative_config = self.speculative_config = vllm_config.speculative_config # noqa
52
+ load_config = self.load_config = vllm_config.load_config
53
+ decoding_config = self.decoding_config = vllm_config.decoding_config or DecodingConfig( # noqa
54
+ )
55
+ prompt_adapter_config = self.prompt_adapter_config = vllm_config.prompt_adapter_config # noqa
56
+ observability_config = self.observability_config = vllm_config.observability_config or ObservabilityConfig( # noqa
57
+ )
58
+
59
# Override the configs for V1.
60
# FIXME
61
if usage_context == UsageContext.LLM_CLASS:
@@ -112,18 +115,6 @@ class LLMEngine:
115
model_config.mm_processor_kwargs,
116
)
117
115
- self.model_config = model_config
116
- self.cache_config = cache_config
117
- self.lora_config = lora_config
118
- self.parallel_config = parallel_config
119
- self.scheduler_config = scheduler_config
120
- self.device_config = device_config
121
- self.speculative_config = speculative_config
122
- self.load_config = load_config
123
- self.decoding_config = decoding_config or DecodingConfig()
124
- self.prompt_adapter_config = prompt_adapter_config
125
- self.observability_config = observability_config or ObservabilityConfig(
126
- )
118
self.log_stats = log_stats
119
120
assert not self.model_config.skip_tokenizer_init
@@ -154,18 +145,7 @@ class LLMEngine:
145
# Request id -> RequestOutput
146
self.request_outputs: Dict[str, RequestOutput] = {}
147
157
- self.model_executor = executor_class(
158
- model_config=model_config,
159
- cache_config=cache_config,
160
- parallel_config=parallel_config,
161
- scheduler_config=scheduler_config,
162
- device_config=device_config,
163
- lora_config=lora_config,
164
- speculative_config=speculative_config,
165
- load_config=load_config,
166
- prompt_adapter_config=prompt_adapter_config,
167
- observability_config=self.observability_config,
168
- )
148
+ self.model_executor = executor_class(vllm_config=vllm_config)
149
assert self.model_config.task != "embedding"
150
self._initialize_kv_caches()
151
@@ -203,7 +183,7 @@ class LLMEngine:
183
executor_class = cls._get_executor_cls(engine_config)
184
# Create the LLM engine.
185
engine = cls(
206
- **engine_config.to_dict(),
186
+ vllm_config=engine_config,
187
executor_class=executor_class,
188
log_stats=not engine_args.disable_log_stats,
189
usage_context=usage_context,