A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] Simplifications to executor classes (#4071)
Nick Hill committed
Apr 15, 2024 at 13:05 UTC
eb46fbfda25348422918c4a876e17aef05fc5e34
5 files changed
+44
-109
vllm/executor/cpu_executor.py
+10
-21
@@ -1,10 +1,9 @@
1
import os
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import Dict, List, Set, Tuple
3
4
import torch
5
6
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
- ParallelConfig, SchedulerConfig)
6
+from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7
from vllm.executor.executor_base import ExecutorBase
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
@@ -16,23 +15,13 @@ logger = init_logger(__name__)
15
16
class CPUExecutor(ExecutorBase):
17
19
- def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
20
- parallel_config: ParallelConfig,
21
- scheduler_config: SchedulerConfig,
22
- device_config: DeviceConfig,
23
- lora_config: Optional[LoRAConfig], *args, **kwargs) -> None:
24
- assert device_config.device_type == "cpu"
25
- assert lora_config is None, "cpu backend doesn't support LoRA"
26
- model_config = _verify_and_get_model_config(model_config)
27
- cache_config = _verify_and_get_cache_config(cache_config)
28
- scheduler_config = _verify_and_get_scheduler_config(scheduler_config)
29
-
30
- self.model_config = model_config
31
- self.cache_config = cache_config
32
- self.lora_config = lora_config
33
- self.parallel_config = parallel_config
34
- self.scheduler_config = scheduler_config
35
- self.device_config = device_config
18
+ def _init_executor(self) -> None:
19
+ assert self.device_config.device_type == "cpu"
20
+ assert self.lora_config is None, "cpu backend doesn't support LoRA"
21
+ self.model_config = _verify_and_get_model_config(self.model_config)
22
+ self.cache_config = _verify_and_get_cache_config(self.cache_config)
23
+ self.scheduler_config = _verify_and_get_scheduler_config(
24
+ self.scheduler_config)
25
26
# Instantiate the worker and load the model to CPU.
27
self._init_worker()
@@ -99,7 +88,7 @@ class CPUExecutor(ExecutorBase):
88
def remove_lora(self, lora_id: int) -> bool:
89
return self.driver_worker.remove_lora(lora_id)
90
102
- def list_loras(self) -> List[int]:
91
+ def list_loras(self) -> Set[int]:
92
return self.driver_worker.list_loras()
93
94
def check_health(self) -> None:
vllm/executor/executor_base.py
+20
-7
@@ -1,9 +1,9 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import Dict, List, Optional, Set, Tuple
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5
ParallelConfig, SchedulerConfig, SpeculativeConfig,
6
- VisionLanguageConfig)
6
+ TensorizerConfig, VisionLanguageConfig)
7
from vllm.lora.request import LoRARequest
8
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
@@ -16,7 +16,6 @@ class ExecutorBase(ABC):
16
that can execute the model on multiple devices.
17
"""
18
19
- @abstractmethod
19
def __init__(
20
self,
21
model_config: ModelConfig,
@@ -27,8 +26,23 @@ class ExecutorBase(ABC):
26
lora_config: Optional[LoRAConfig],
27
vision_language_config: Optional[VisionLanguageConfig],
28
speculative_config: Optional[SpeculativeConfig],
29
+ tensorizer_config: Optional[TensorizerConfig],
30
) -> None:
31
- raise NotImplementedError
31
+ self.model_config = model_config
32
+ self.cache_config = cache_config
33
+ self.lora_config = lora_config
34
+ self.parallel_config = parallel_config
35
+ self.scheduler_config = scheduler_config
36
+ self.device_config = device_config
37
+ self.vision_language_config = vision_language_config
38
+ self.speculative_config = speculative_config
39
+ self.tensorizer_config = tensorizer_config
40
+
41
+ self._init_executor()
42
+
43
+ @abstractmethod
44
+ def _init_executor(self) -> None:
45
+ pass
46
47
@abstractmethod
48
def determine_num_available_blocks(self) -> Tuple[int, int]:
@@ -71,7 +85,7 @@ class ExecutorBase(ABC):
85
raise NotImplementedError
86
87
@abstractmethod
74
- def list_loras(self) -> List[int]:
88
+ def list_loras(self) -> Set[int]:
89
raise NotImplementedError
90
91
@abstractmethod
@@ -94,8 +108,7 @@ class ExecutorAsyncBase(ExecutorBase):
108
"""Executes one model step on the given sequences."""
109
raise NotImplementedError
110
97
- @abstractmethod
111
async def check_health_async(self) -> None:
112
"""Checks if the executor is healthy. If not, it should raise an
113
exception."""
101
- raise NotImplementedError
114
+ self.check_health()
vllm/executor/gpu_executor.py
+4
-28
@@ -1,8 +1,5 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import Dict, List, Set, Tuple
2
3
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
5
- TensorizerConfig, VisionLanguageConfig)
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
@@ -15,24 +12,8 @@ logger = init_logger(__name__)
12
13
class GPUExecutor(ExecutorBase):
14
18
- def __init__(self, model_config: ModelConfig, cache_config: CacheConfig,
19
- parallel_config: ParallelConfig,
20
- scheduler_config: SchedulerConfig,
21
- device_config: DeviceConfig,
22
- lora_config: Optional[LoRAConfig],
23
- vision_language_config: Optional[VisionLanguageConfig],
24
- speculative_config: Optional[SpeculativeConfig],
25
- tensorizer_config: Optional[TensorizerConfig]) -> None:
26
- self.model_config = model_config
27
- self.cache_config = cache_config
28
- self.lora_config = lora_config
29
- self.parallel_config = parallel_config
30
- self.scheduler_config = scheduler_config
31
- self.device_config = device_config
32
- self.vision_language_config = vision_language_config
33
- self.tensorizer_config = tensorizer_config
34
-
35
- assert (not speculative_config
15
+ def _init_executor(self) -> None:
16
+ assert (not self.speculative_config
17
), "Speculative decoding not yet supported for GPU backend"
18
19
# Instantiate the worker and load the model to GPU.
@@ -103,7 +84,7 @@ class GPUExecutor(ExecutorBase):
84
assert lora_id > 0, "lora_id must be greater than 0."
85
return self.driver_worker.remove_lora(lora_id)
86
106
- def list_loras(self) -> List[int]:
87
+ def list_loras(self) -> Set[int]:
88
return self.driver_worker.list_loras()
89
90
def check_health(self) -> None:
@@ -127,8 +108,3 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
108
blocks_to_swap_out=blocks_to_swap_out,
109
blocks_to_copy=blocks_to_copy)
110
return output
130
-
131
- async def check_health_async(self) -> None:
132
- # GPUExecutor will always be healthy as long as
133
- # it's running.
134
- return
vllm/executor/neuron_executor.py
+6
-23
@@ -1,8 +1,5 @@
1
-from typing import Dict, List, Optional, Tuple
1
+from typing import Dict, List, Set, Tuple
2
3
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
5
- VisionLanguageConfig)
3
from vllm.executor.executor_base import ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
@@ -13,24 +10,10 @@ logger = init_logger(__name__)
10
11
class NeuronExecutor(ExecutorBase):
12
16
- def __init__(
17
- self,
18
- model_config: ModelConfig,
19
- cache_config: CacheConfig,
20
- parallel_config: ParallelConfig,
21
- scheduler_config: SchedulerConfig,
22
- device_config: DeviceConfig,
23
- lora_config: Optional[LoRAConfig],
24
- vision_language_config: Optional[VisionLanguageConfig],
25
- speculative_config: Optional[SpeculativeConfig],
26
- ) -> None:
27
- self.model_config = model_config
28
- self.cache_config = cache_config
29
- assert lora_config is None, "LoRA is not supported for Neuron backend."
30
- self.parallel_config = parallel_config
31
- self.scheduler_config = scheduler_config
32
- self.device_config = device_config
33
- assert (not speculative_config
13
+ def _init_executor(self) -> None:
14
+ assert (self.lora_config is
15
+ None), "LoRA is not supported for Neuron backend."
16
+ assert (not self.speculative_config
17
), "Speculative decoding not yet supported for Neuron backend."
18
19
# Instantiate the worker and load the model to the device.
@@ -80,7 +63,7 @@ class NeuronExecutor(ExecutorBase):
63
def remove_lora(self, lora_id: int) -> bool:
64
return self.driver_worker.remove_lora(lora_id)
65
83
- def list_loras(self) -> List[int]:
66
+ def list_loras(self) -> Set[int]:
67
return self.driver_worker.list_loras()
68
69
def check_health(self) -> None:
vllm/executor/ray_gpu_executor.py
+4
-30
@@ -3,11 +3,8 @@ import copy
3
import os
4
import pickle
5
from collections import defaultdict
6
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
6
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
7
8
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
10
- TensorizerConfig, VisionLanguageConfig)
8
from vllm.engine.ray_utils import RayWorkerVllm, ray
9
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10
from vllm.logger import init_logger
@@ -32,27 +29,8 @@ USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
29
30
class RayGPUExecutor(ExecutorBase):
31
35
- def __init__(
36
- self,
37
- model_config: ModelConfig,
38
- cache_config: CacheConfig,
39
- parallel_config: ParallelConfig,
40
- scheduler_config: SchedulerConfig,
41
- device_config: DeviceConfig,
42
- lora_config: Optional[LoRAConfig],
43
- vision_language_config: Optional[VisionLanguageConfig],
44
- speculative_config: Optional[SpeculativeConfig],
45
- tensorizer_config: Optional[TensorizerConfig],
46
- ) -> None:
47
- self.model_config = model_config
48
- self.cache_config = cache_config
49
- self.lora_config = lora_config
50
- self.parallel_config = parallel_config
51
- self.scheduler_config = scheduler_config
52
- self.device_config = device_config
53
- self.vision_language_config = vision_language_config
54
- self.tensorizer_config = tensorizer_config
55
- assert (not speculative_config
32
+ def _init_executor(self) -> None:
33
+ assert (not self.speculative_config
34
), "Speculative decoding not yet supported for RayGPU backend."
35
36
assert self.parallel_config.worker_use_ray
@@ -273,7 +251,7 @@ class RayGPUExecutor(ExecutorBase):
251
lora_id=lora_id,
252
)
253
276
- def list_loras(self) -> List[int]:
254
+ def list_loras(self) -> Set[int]:
255
return self._run_workers("list_loras")
256
257
def _run_workers(
@@ -416,7 +394,3 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
394
# Only the driver worker returns the sampling results.
395
output = all_outputs[0]
396
return output
419
-
420
- async def check_health_async(self) -> None:
421
- """Raises an error if engine is unhealthy."""
422
- self._check_if_any_actor_is_dead()