A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[V1] Fix `EngineArgs` refactor on V1 (#9954)
Robert Shaw committed
Nov 2, 2024 at 10:44 UTC
d6459b4516dbac4f346ce29fe90d43ebfafa1114
1 file changed
+12
-27
vllm/v1/executor/gpu_executor.py
+12
-27
@@ -1,10 +1,7 @@
1
import os
2
from typing import Optional, Tuple
3
4
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
5
- ModelConfig, ObservabilityConfig, ParallelConfig,
6
- PromptAdapterConfig, SchedulerConfig,
7
- SpeculativeConfig)
4
+from vllm.config import EngineConfig
5
from vllm.logger import init_logger
6
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7
from vllm.v1.outputs import ModelRunnerOutput
@@ -15,29 +12,17 @@ logger = init_logger(__name__)
12
13
class GPUExecutor:
14
18
- def __init__(
19
- self,
20
- model_config: ModelConfig,
21
- cache_config: CacheConfig,
22
- parallel_config: ParallelConfig,
23
- scheduler_config: SchedulerConfig,
24
- device_config: DeviceConfig,
25
- load_config: LoadConfig,
26
- lora_config: Optional[LoRAConfig],
27
- speculative_config: Optional[SpeculativeConfig],
28
- prompt_adapter_config: Optional[PromptAdapterConfig],
29
- observability_config: Optional[ObservabilityConfig],
30
- ) -> None:
31
- self.model_config = model_config
32
- self.cache_config = cache_config
33
- self.lora_config = lora_config
34
- self.load_config = load_config
35
- self.parallel_config = parallel_config
36
- self.scheduler_config = scheduler_config
37
- self.device_config = device_config
38
- self.speculative_config = speculative_config
39
- self.prompt_adapter_config = prompt_adapter_config
40
- self.observability_config = observability_config
15
+ def __init__(self, vllm_config: EngineConfig) -> None:
16
+ self.model_config = vllm_config.model_config
17
+ self.cache_config = vllm_config.cache_config
18
+ self.lora_config = vllm_config.lora_config
19
+ self.load_config = vllm_config.load_config
20
+ self.parallel_config = vllm_config.parallel_config
21
+ self.scheduler_config = vllm_config.scheduler_config
22
+ self.device_config = vllm_config.device_config
23
+ self.speculative_config = vllm_config.speculative_config
24
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
25
+ self.observability_config = vllm_config.observability_config
26
27
self.worker = self._create_worker()
28
self.worker.initialize()