A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Frontend] Enable support for CPU backend in AsyncLLMEngine. (#3993)
Signed-off-by: Tao He <sighingnow@gmail.com>
Tao He committed
Apr 22, 2024 at 17:19 UTC
077f0a2e8a873340b1a2cf54d6c9043754eb7514
2 files changed
+30
-2
vllm/engine/async_llm_engine.py
+5
@@ -343,6 +343,11 @@ class AsyncLLMEngine:
343
if engine_config.device_config.device_type == "neuron":
344
from vllm.executor.neuron_executor import NeuronExecutorAsync
345
executor_class = NeuronExecutorAsync
346
+ elif engine_config.device_config.device_type == "cpu":
347
+ assert not engine_config.parallel_config.worker_use_ray, (
348
+ "Ray is not supported with the CPU backend.")
349
+ from vllm.executor.cpu_executor import CPUExecutorAsync
350
+ executor_class = CPUExecutorAsync
351
elif engine_config.parallel_config.worker_use_ray:
352
initialize_ray_cluster(engine_config.parallel_config)
353
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
vllm/executor/cpu_executor.py
+25
-2
@@ -4,11 +4,12 @@ from typing import Dict, List, Set, Tuple
4
import torch
5
6
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7
-from vllm.executor.executor_base import ExecutorBase
7
+from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8
from vllm.logger import init_logger
9
from vllm.lora.request import LoRARequest
10
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
11
-from vllm.utils import get_distributed_init_method, get_ip, get_open_port
11
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12
+ make_async)
13
14
logger = init_logger(__name__)
15
@@ -100,6 +101,28 @@ class CPUExecutor(ExecutorBase):
101
return
102
103
104
+class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
105
+
106
+ async def execute_model_async(
107
+ self,
108
+ seq_group_metadata_list: List[SequenceGroupMetadata],
109
+ blocks_to_swap_in: Dict[int, int],
110
+ blocks_to_swap_out: Dict[int, int],
111
+ blocks_to_copy: Dict[int, List[int]],
112
+ ) -> SamplerOutput:
113
+ output = await make_async(self.driver_worker.execute_model)(
114
+ seq_group_metadata_list=seq_group_metadata_list,
115
+ blocks_to_swap_in=blocks_to_swap_in,
116
+ blocks_to_swap_out=blocks_to_swap_out,
117
+ blocks_to_copy=blocks_to_copy)
118
+ return output
119
+
120
+ async def check_health_async(self) -> None:
121
+ # CPUExecutor will always be healthy as long as
122
+ # it's running.
123
+ return
124
+
125
+
126
def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
127
if config.dtype == torch.float16:
128
logger.warning("float16 is not supported on CPU, casting to bfloat16.")