@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d74b5aee9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[platforms] refactor cpu code (#10402)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 16, 2024 at 23:14 UTC 8d74b5aee9e780852de870c936b59707835e84f5
2 files changed +61 -67
vllm/executor/cpu_executor.py
+1 -67
@@ -2,9 +2,6 @@ import os
2 from functools import partial
3 from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
4
5 -import vllm.envs as envs
6 -from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig)
5 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
7 ResultHandler, WorkerMonitor)
@@ -13,7 +10,7 @@ from vllm.lora.request import LoRARequest
10 from vllm.model_executor.layers.sampler import SamplerOutput
11 from vllm.prompt_adapter.request import PromptAdapterRequest
12 from vllm.sequence import ExecuteModelRequest
16 -from vllm.utils import (GiB_bytes, get_distributed_init_method, get_open_port,
13 +from vllm.utils import (get_distributed_init_method, get_open_port,
14 get_vllm_instance_id, make_async)
15 from vllm.worker.worker_base import WorkerWrapperBase
16
@@ -57,13 +54,6 @@ class CPUExecutor(ExecutorBase):
54 os.environ["LOCAL_WORLD_SIZE"] = str(
55 self.parallel_config.tensor_parallel_size)
56
60 - self.model_config = _verify_and_get_model_config(self.model_config)
61 - self.cache_config = _verify_and_get_cache_config(self.cache_config)
62 - self.scheduler_config = _verify_and_get_scheduler_config(
63 - self.scheduler_config)
64 - self.parallel_config = _verify_and_get_parallel_config(
65 - self.parallel_config)
66 -
57 # Multiprocessing-based executor does not support multi-node setting.
58 # Since it only works for single node, we can use the loopback address
59 # 127.0.0.1 for communication.
@@ -313,62 +303,6 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
303 self.check_health()
304
305
316 -def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
317 - # Reminder: Please update docs/source/serving/compatibility_matrix.rst
318 - # If the feature combo become valid
319 - if not config.enforce_eager:
320 - logger.warning(
321 - "CUDA graph is not supported on CPU, fallback to the eager "
322 - "mode.")
323 - config.enforce_eager = True
324 - return config
325 -
326 -
327 -def _verify_and_get_scheduler_config(
328 - config: SchedulerConfig) -> SchedulerConfig:
329 - # Reminder: Please update docs/source/serving/compatibility_matrix.rst
330 - # If the feature combo become valid
331 - if config.chunked_prefill_enabled:
332 - logger.warning("Chunked prefill is not supported on CPU, disable it.")
333 - config.chunked_prefill_enabled = False
334 -
335 - return config
336 -
337 -
338 -def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
339 - # Reminder: Please update docs/source/serving/compatibility_matrix.rst
340 - # If the feature combo become valid
341 - if config.enable_prefix_caching:
342 - logger.warning("Prefix caching is not supported on CPU, disable it.")
343 - config.enable_prefix_caching = False
344 -
345 - kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
346 -
347 - if kv_cache_space >= 0:
348 - if kv_cache_space == 0:
349 - config.cpu_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
350 - logger.warning("Environment variable VLLM_CPU_KVCACHE_SPACE (GB) "
351 - "for CPU backend is not set, using 4 by default.")
352 - else:
353 - config.cpu_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore
354 - else:
355 - raise RuntimeError(
356 - "Invalid environment variable VLLM_CPU_KVCACHE_SPACE"
357 - f" {kv_cache_space}, expect a positive integer value.")
358 -
359 - return config
360 -
361 -
362 -def _verify_and_get_parallel_config(config: ParallelConfig) -> ParallelConfig:
363 - if (config.distributed_executor_backend is not None
364 - and config.distributed_executor_backend != "mp"):
365 - logger.warning(
366 - "%s is not supported on CPU, fallback to mp distributed executor "
367 - "backend.", config.distributed_executor_backend)
368 - config.distributed_executor_backend = "mp"
369 - return config
370 -
371 -
306 def _driver_method_invoker(driver, method: str, *args, **kwargs):
307 return getattr(driver, method)(*args, **kwargs)
308
vllm/platforms/cpu.py
+60
@@ -1,8 +1,19 @@
1 +from typing import TYPE_CHECKING
2 +
3 import psutil
4 import torch
5
6 +from vllm.logger import init_logger
7 +
8 from .interface import Platform, PlatformEnum
9
10 +if TYPE_CHECKING:
11 + from vllm.config import VllmConfig
12 +else:
13 + VllmConfig = None
14 +
15 +logger = init_logger(__name__)
16 +
17
18 class CpuPlatform(Platform):
19 _enum = PlatformEnum.CPU
@@ -18,3 +29,52 @@ class CpuPlatform(Platform):
29 @classmethod
30 def inference_mode(cls):
31 return torch.no_grad()
32 +
33 + @classmethod
34 + def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
35 + import vllm.envs as envs
36 + from vllm.utils import GiB_bytes
37 + model_config = vllm_config.model_config
38 + # Reminder: Please update docs/source/serving/compatibility_matrix.rst
39 + # If the feature combo become valid
40 + if not model_config.enforce_eager:
41 + logger.warning(
42 + "CUDA graph is not supported on CPU, fallback to the eager "
43 + "mode.")
44 + model_config.enforce_eager = True
45 +
46 + cache_config = vllm_config.cache_config
47 +
48 + if cache_config.enable_prefix_caching:
49 + logger.warning(
50 + "Prefix caching is not supported on CPU, disable it.")
51 + cache_config.enable_prefix_caching = False
52 +
53 + kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
54 +
55 + if kv_cache_space >= 0:
56 + if kv_cache_space == 0:
57 + cache_config.cpu_kvcache_space_bytes = 4 * GiB_bytes # type: ignore
58 + logger.warning(
59 + "Environment variable VLLM_CPU_KVCACHE_SPACE (GB) "
60 + "for CPU backend is not set, using 4 by default.")
61 + else:
62 + cache_config.cpu_kvcache_space_bytes = kv_cache_space * GiB_bytes # type: ignore # noqa
63 + else:
64 + raise RuntimeError(
65 + "Invalid environment variable VLLM_CPU_KVCACHE_SPACE"
66 + f" {kv_cache_space}, expect a positive integer value.")
67 +
68 + scheduler_config = vllm_config.scheduler_config
69 + if scheduler_config.chunked_prefill_enabled:
70 + logger.warning(
71 + "Chunked prefill is not supported on CPU, disable it.")
72 + scheduler_config.chunked_prefill_enabled = False
73 +
74 + parallel_config = vllm_config.parallel_config
75 + if (parallel_config.distributed_executor_backend is not None
76 + and parallel_config.distributed_executor_backend != "mp"):
77 + logger.warning(("%s is not supported on CPU, fallback to mp "
78 + "distributed executor backend."),
79 + parallel_config.distributed_executor_backend)
80 + parallel_config.distributed_executor_backend = "mp"