@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 077f0a2e8a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Frontend] Enable support for CPU backend in AsyncLLMEngine. (#3993)

Signed-off-by: Tao He <sighingnow@gmail.com>

Tao He committed Apr 22, 2024 at 17:19 UTC 077f0a2e8a873340b1a2cf54d6c9043754eb7514
2 files changed +30 -2
vllm/engine/async_llm_engine.py
+5
@@ -343,6 +343,11 @@ class AsyncLLMEngine:
343 if engine_config.device_config.device_type == "neuron":
344 from vllm.executor.neuron_executor import NeuronExecutorAsync
345 executor_class = NeuronExecutorAsync
346 + elif engine_config.device_config.device_type == "cpu":
347 + assert not engine_config.parallel_config.worker_use_ray, (
348 + "Ray is not supported with the CPU backend.")
349 + from vllm.executor.cpu_executor import CPUExecutorAsync
350 + executor_class = CPUExecutorAsync
351 elif engine_config.parallel_config.worker_use_ray:
352 initialize_ray_cluster(engine_config.parallel_config)
353 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
vllm/executor/cpu_executor.py
+25 -2
@@ -4,11 +4,12 @@ from typing import Dict, List, Set, Tuple
4 import torch
5
6 from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
7 -from vllm.executor.executor_base import ExecutorBase
7 +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
8 from vllm.logger import init_logger
9 from vllm.lora.request import LoRARequest
10 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
11 -from vllm.utils import get_distributed_init_method, get_ip, get_open_port
11 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
12 + make_async)
13
14 logger = init_logger(__name__)
15
@@ -100,6 +101,28 @@ class CPUExecutor(ExecutorBase):
101 return
102
103
104 +class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
105 +
106 + async def execute_model_async(
107 + self,
108 + seq_group_metadata_list: List[SequenceGroupMetadata],
109 + blocks_to_swap_in: Dict[int, int],
110 + blocks_to_swap_out: Dict[int, int],
111 + blocks_to_copy: Dict[int, List[int]],
112 + ) -> SamplerOutput:
113 + output = await make_async(self.driver_worker.execute_model)(
114 + seq_group_metadata_list=seq_group_metadata_list,
115 + blocks_to_swap_in=blocks_to_swap_in,
116 + blocks_to_swap_out=blocks_to_swap_out,
117 + blocks_to_copy=blocks_to_copy)
118 + return output
119 +
120 + async def check_health_async(self) -> None:
121 + # CPUExecutor will always be healthy as long as
122 + # it's running.
123 + return
124 +
125 +
126 def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
127 if config.dtype == torch.float16:
128 logger.warning("float16 is not supported on CPU, casting to bfloat16.")