@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb46fbfda2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] Simplifications to executor classes (#4071)

Nick Hill committed Apr 15, 2024 at 13:05 UTC eb46fbfda25348422918c4a876e17aef05fc5e34
5 files changed +44 -109
vllm/executor/cpu_executor.py
+10 -21
@@ -1,10 +1,9 @@
1 import os
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Dict, List, Set, Tuple
3
4 import torch
5
6 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7 - ParallelConfig, SchedulerConfig)
6 +from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7 from vllm.executor.executor_base import ExecutorBase
8 from vllm.logger import init_logger
9 from vllm.lora.request import LoRARequest
@@ -16,23 +15,13 @@ logger = init_logger(__name__)
15
16 class CPUExecutor(ExecutorBase):
17
19 - def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
20 - parallel_config: ParallelConfig,
21 - scheduler_config: SchedulerConfig,
22 - device_config: DeviceConfig,
23 - lora_config: Optional[LoRAConfig], *args, **kwargs) -> None:
24 - assert device_config.device_type == "cpu"
25 - assert lora_config is None, "cpu backend doesn't support LoRA"
26 - model_config = _verify_and_get_model_config(model_config)
27 - cache_config = _verify_and_get_cache_config(cache_config)
28 - scheduler_config = _verify_and_get_scheduler_config(scheduler_config)
29 -
30 - self.model_config = model_config
31 - self.cache_config = cache_config
32 - self.lora_config = lora_config
33 - self.parallel_config = parallel_config
34 - self.scheduler_config = scheduler_config
35 - self.device_config = device_config
18 + def _init_executor(self) -> None:
19 + assert self.device_config.device_type == "cpu"
20 + assert self.lora_config is None, "cpu backend doesn't support LoRA"
21 + self.model_config = _verify_and_get_model_config(self.model_config)
22 + self.cache_config = _verify_and_get_cache_config(self.cache_config)
23 + self.scheduler_config = _verify_and_get_scheduler_config(
24 + self.scheduler_config)
25
26 # Instantiate the worker and load the model to CPU.
27 self._init_worker()
@@ -99,7 +88,7 @@ class CPUExecutor(ExecutorBase):
88 def remove_lora(self, lora_id: int) -> bool:
89 return self.driver_worker.remove_lora(lora_id)
90
102 - def list_loras(self) -> List[int]:
91 + def list_loras(self) -> Set[int]:
92 return self.driver_worker.list_loras()
93
94 def check_health(self) -> None:
vllm/executor/executor_base.py
+20 -7
@@ -1,9 +1,9 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, List, Optional, Tuple
2 +from typing import Dict, List, Optional, Set, Tuple
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5 ParallelConfig, SchedulerConfig, SpeculativeConfig,
6 - VisionLanguageConfig)
6 + TensorizerConfig, VisionLanguageConfig)
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
@@ -16,7 +16,6 @@ class ExecutorBase(ABC):
16 that can execute the model on multiple devices.
17 """
18
19 - @abstractmethod
19 def __init__(
20 self,
21 model_config: ModelConfig,
@@ -27,8 +26,23 @@ class ExecutorBase(ABC):
26 lora_config: Optional[LoRAConfig],
27 vision_language_config: Optional[VisionLanguageConfig],
28 speculative_config: Optional[SpeculativeConfig],
29 + tensorizer_config: Optional[TensorizerConfig],
30 ) -> None:
31 - raise NotImplementedError
31 + self.model_config = model_config
32 + self.cache_config = cache_config
33 + self.lora_config = lora_config
34 + self.parallel_config = parallel_config
35 + self.scheduler_config = scheduler_config
36 + self.device_config = device_config
37 + self.vision_language_config = vision_language_config
38 + self.speculative_config = speculative_config
39 + self.tensorizer_config = tensorizer_config
40 +
41 + self._init_executor()
42 +
43 + @abstractmethod
44 + def _init_executor(self) -> None:
45 + pass
46
47 @abstractmethod
48 def determine_num_available_blocks(self) -> Tuple[int, int]:
@@ -71,7 +85,7 @@ class ExecutorBase(ABC):
85 raise NotImplementedError
86
87 @abstractmethod
74 - def list_loras(self) -> List[int]:
88 + def list_loras(self) -> Set[int]:
89 raise NotImplementedError
90
91 @abstractmethod
@@ -94,8 +108,7 @@ class ExecutorAsyncBase(ExecutorBase):
108 """Executes one model step on the given sequences."""
109 raise NotImplementedError
110
97 - @abstractmethod
111 async def check_health_async(self) -> None:
112 """Checks if the executor is healthy. If not, it should raise an
113 exception."""
101 - raise NotImplementedError
114 + self.check_health()
vllm/executor/gpu_executor.py
+4 -28
@@ -1,8 +1,5 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import Dict, List, Set, Tuple
2
3 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
5 - TensorizerConfig, VisionLanguageConfig)
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
@@ -15,24 +12,8 @@ logger = init_logger(__name__)
12
13 class GPUExecutor(ExecutorBase):
14
18 - def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
19 - parallel_config: ParallelConfig,
20 - scheduler_config: SchedulerConfig,
21 - device_config: DeviceConfig,
22 - lora_config: Optional[LoRAConfig],
23 - vision_language_config: Optional[VisionLanguageConfig],
24 - speculative_config: Optional[SpeculativeConfig],
25 - tensorizer_config: Optional[TensorizerConfig]) -> None:
26 - self.model_config = model_config
27 - self.cache_config = cache_config
28 - self.lora_config = lora_config
29 - self.parallel_config = parallel_config
30 - self.scheduler_config = scheduler_config
31 - self.device_config = device_config
32 - self.vision_language_config = vision_language_config
33 - self.tensorizer_config = tensorizer_config
34 -
35 - assert (not speculative_config
15 + def _init_executor(self) -> None:
16 + assert (not self.speculative_config
17 ), "Speculative decoding not yet supported for GPU backend"
18
19 # Instantiate the worker and load the model to GPU.
@@ -103,7 +84,7 @@ class GPUExecutor(ExecutorBase):
84 assert lora_id > 0, "lora_id must be greater than 0."
85 return self.driver_worker.remove_lora(lora_id)
86
106 - def list_loras(self) -> List[int]:
87 + def list_loras(self) -> Set[int]:
88 return self.driver_worker.list_loras()
89
90 def check_health(self) -> None:
@@ -127,8 +108,3 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
108 blocks_to_swap_out=blocks_to_swap_out,
109 blocks_to_copy=blocks_to_copy)
110 return output
130 -
131 - async def check_health_async(self) -> None:
132 - # GPUExecutor will always be healthy as long as
133 - # it's running.
134 - return
vllm/executor/neuron_executor.py
+6 -23
@@ -1,8 +1,5 @@
1 -from typing import Dict, List, Optional, Tuple
1 +from typing import Dict, List, Set, Tuple
2
3 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
5 - VisionLanguageConfig)
3 from vllm.executor.executor_base import ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
@@ -13,24 +10,10 @@ logger = init_logger(__name__)
10
11 class NeuronExecutor(ExecutorBase):
12
16 - def __init__(
17 - self,
18 - model_config: ModelConfig,
19 - cache_config: CacheConfig,
20 - parallel_config: ParallelConfig,
21 - scheduler_config: SchedulerConfig,
22 - device_config: DeviceConfig,
23 - lora_config: Optional[LoRAConfig],
24 - vision_language_config: Optional[VisionLanguageConfig],
25 - speculative_config: Optional[SpeculativeConfig],
26 - ) -> None:
27 - self.model_config = model_config
28 - self.cache_config = cache_config
29 - assert lora_config is None, "LoRA is not supported for Neuron backend."
30 - self.parallel_config = parallel_config
31 - self.scheduler_config = scheduler_config
32 - self.device_config = device_config
33 - assert (not speculative_config
13 + def _init_executor(self) -> None:
14 + assert (self.lora_config is
15 + None), "LoRA is not supported for Neuron backend."
16 + assert (not self.speculative_config
17 ), "Speculative decoding not yet supported for Neuron backend."
18
19 # Instantiate the worker and load the model to the device.
@@ -80,7 +63,7 @@ class NeuronExecutor(ExecutorBase):
63 def remove_lora(self, lora_id: int) -> bool:
64 return self.driver_worker.remove_lora(lora_id)
65
83 - def list_loras(self) -> List[int]:
66 + def list_loras(self) -> Set[int]:
67 return self.driver_worker.list_loras()
68
69 def check_health(self) -> None:
vllm/executor/ray_gpu_executor.py
+4 -30
@@ -3,11 +3,8 @@ import copy
3 import os
4 import pickle
5 from collections import defaultdict
6 -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
7
8 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
10 - TensorizerConfig, VisionLanguageConfig)
8 from vllm.engine.ray_utils import RayWorkerVllm, ray
9 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10 from vllm.logger import init_logger
@@ -32,27 +29,8 @@ USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
29
30 class RayGPUExecutor(ExecutorBase):
31
35 - def __init__(
36 - self,
37 - model_config: ModelConfig,
38 - cache_config: CacheConfig,
39 - parallel_config: ParallelConfig,
40 - scheduler_config: SchedulerConfig,
41 - device_config: DeviceConfig,
42 - lora_config: Optional[LoRAConfig],
43 - vision_language_config: Optional[VisionLanguageConfig],
44 - speculative_config: Optional[SpeculativeConfig],
45 - tensorizer_config: Optional[TensorizerConfig],
46 - ) -> None:
47 - self.model_config = model_config
48 - self.cache_config = cache_config
49 - self.lora_config = lora_config
50 - self.parallel_config = parallel_config
51 - self.scheduler_config = scheduler_config
52 - self.device_config = device_config
53 - self.vision_language_config = vision_language_config
54 - self.tensorizer_config = tensorizer_config
55 - assert (not speculative_config
32 + def _init_executor(self) -> None:
33 + assert (not self.speculative_config
34 ), "Speculative decoding not yet supported for RayGPU backend."
35
36 assert self.parallel_config.worker_use_ray
@@ -273,7 +251,7 @@ class RayGPUExecutor(ExecutorBase):
251 lora_id=lora_id,
252 )
253
276 - def list_loras(self) -> List[int]:
254 + def list_loras(self) -> Set[int]:
255 return self._run_workers("list_loras")
256
257 def _run_workers(
@@ -416,7 +394,3 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
394 # Only the driver worker returns the sampling results.
395 output = all_outputs[0]
396 return output
419 -
420 - async def check_health_async(self) -> None:
421 - """Raises an error if engine is unhealthy."""
422 - self._check_if_any_actor_is_dead()