@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3bbb4936dc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Hardware] [Intel] Enable Multiprocessing and tensor parallel in CPU backend and update documentation (#6125)

Li, Jiang committed Jul 27, 2024 at 04:50 UTC 3bbb4936dc5aa7737750410ab4b4647817dcf9a3
14 files changed +404 -90
.buildkite/run-cpu-test.sh
+20 -8
@@ -3,26 +3,38 @@
3 set -ex
4
5 # Try building the docker image
6 -docker build -t cpu-test -f Dockerfile.cpu .
7 -docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
6 +numactl -C 48-95 -N 1 docker build -t cpu-test -f Dockerfile.cpu .
7 +numactl -C 48-95 -N 1 docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
8
9 # Setup cleanup
10 remove_docker_container() { docker rm -f cpu-test cpu-test-avx2 || true; }
11 trap remove_docker_container EXIT
12 remove_docker_container
13
14 -# Run the image
14 +# Run the image, setting --shm-size=4g for tensor parallel.
15 docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
16 - --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
16 + --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
17 docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
18 - --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
18 + --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
19
20 # offline inference
21 -docker exec cpu-test bash -c "python3 examples/offline_inference.py"
21 docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
22
23 # Run basic model test
25 -docker exec cpu-test bash -c "cd tests;
24 +docker exec cpu-test bash -c "
25 pip install pytest Pillow protobuf
27 - cd ../
26 pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py" # Mamba on CPU is not supported
27 +
28 +# online inference
29 +docker exec cpu-test bash -c "
30 + export VLLM_CPU_KVCACHE_SPACE=10
31 + export VLLM_CPU_OMP_THREADS_BIND=48-92
32 + python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
33 + timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
34 + python3 benchmarks/benchmark_serving.py \
35 + --backend vllm \
36 + --dataset-name random \
37 + --model facebook/opt-125m \
38 + --num-prompts 20 \
39 + --endpoint /v1/completions \
40 + --tokenizer facebook/opt-125m"
Dockerfile.cpu
+5 -4
@@ -2,8 +2,8 @@
2
3 FROM ubuntu:22.04 AS cpu-test-1
4
5 -RUN apt-get update -y \
6 - && apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 \
5 +RUN apt-get update -y \
6 + && apt-get install -y curl git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
7 && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
8
9 # https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
@@ -13,8 +13,9 @@ RUN pip install intel-openmp
13
14 ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so:$LD_PRELOAD"
15
16 +RUN echo 'ulimit -c 0' >> ~/.bashrc
17
17 -RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.3.100%2Bgit0eb3473-cp310-cp310-linux_x86_64.whl
18 +RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.4.0%2Bgitfbaa4bc-cp310-cp310-linux_x86_64.whl
19
20 RUN pip install --upgrade pip \
21 && pip install wheel packaging ninja "setuptools>=49.4.0" numpy
@@ -25,7 +26,7 @@ COPY ./ /workspace/vllm
26
27 WORKDIR /workspace/vllm
28
28 -RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
29 +RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/test/cpu
30
31 # Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
32 ARG VLLM_CPU_DISABLE_AVX512
cmake/cpu_extension.cmake
+4
@@ -83,6 +83,8 @@ endif()
83
84 message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
85
86 +list(APPEND LIBS "numa")
87 +
88
89 #
90 # Define extension targets
@@ -95,6 +97,7 @@ set(VLLM_EXT_SRC
97 "csrc/cpu/activation.cpp"
98 "csrc/cpu/attention.cpp"
99 "csrc/cpu/cache.cpp"
100 + "csrc/cpu/utils.cpp"
101 "csrc/cpu/layernorm.cpp"
102 "csrc/cpu/pos_encoding.cpp"
103 "csrc/cpu/torch_bindings.cpp")
@@ -104,6 +107,7 @@ define_gpu_extension_target(
107 DESTINATION vllm
108 LANGUAGE CXX
109 SOURCES ${VLLM_EXT_SRC}
110 + LIBRARIES ${LIBS}
111 COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
112 USE_SABI 3
113 WITH_SOABI
csrc/cpu/torch_bindings.cpp
+7
@@ -4,6 +4,8 @@
4
5 #include <torch/library.h>
6
7 +void init_cpu_threads_env(const std::string& cpu_ids);
8 +
9 TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
10 // vLLM custom ops
11
@@ -107,4 +109,9 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
109 cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
110 }
111
112 +TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _utils), utils) {
113 + // CPU utils
114 + utils.def("init_cpu_threads_env(str cpu_ids) -> ()", &init_cpu_threads_env);
115 +}
116 +
117 REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cpu/utils.cpp new
+65
@@ -0,0 +1,65 @@
1 +#include <numa.h>
2 +#include <unistd.h>
3 +#include <string>
4 +#include <sched.h>
5 +
6 +#include "cpu_types.hpp"
7 +
8 +void init_cpu_threads_env(const std::string& cpu_ids) {
9 + bitmask* omp_cpu_mask = numa_parse_cpustring(cpu_ids.c_str());
10 + TORCH_CHECK(omp_cpu_mask->size > 0);
11 + std::vector<int> omp_cpu_ids;
12 + omp_cpu_ids.reserve(omp_cpu_mask->size);
13 +
14 + constexpr int group_size = 8 * sizeof(*omp_cpu_mask->maskp);
15 +
16 + for (int offset = 0; offset < omp_cpu_mask->size; offset += group_size) {
17 + unsigned long group_mask = omp_cpu_mask->maskp[offset / group_size];
18 + int i = 0;
19 + while (group_mask) {
20 + if (group_mask & 1) {
21 + omp_cpu_ids.emplace_back(offset + i);
22 + }
23 + ++i;
24 + group_mask >>= 1;
25 + }
26 + }
27 +
28 + // Memory node binding
29 + if (numa_available() != -1) {
30 + int mem_node_id = numa_node_of_cpu(omp_cpu_ids.front());
31 + bitmask* mask = numa_parse_nodestring(std::to_string(mem_node_id).c_str());
32 + bitmask* src_mask = numa_get_membind();
33 +
34 + int pid = getpid();
35 +
36 + // move all existing pages to the specified numa node.
37 + *(src_mask->maskp) = *(src_mask->maskp) ^ *(mask->maskp);
38 + int page_num = numa_migrate_pages(pid, src_mask, mask);
39 + if (page_num == -1) {
40 + TORCH_CHECK(false,
41 + "numa_migrate_pages failed. errno: " + std::to_string(errno));
42 + }
43 +
44 + // restrict memory allocation node.
45 + numa_set_membind(mask);
46 + numa_set_strict(1);
47 + }
48 +
49 + // OMP threads binding
50 + omp_set_num_threads((int)omp_cpu_ids.size());
51 + torch::set_num_threads((int)omp_cpu_ids.size());
52 + TORCH_CHECK_EQ(omp_cpu_ids.size(), torch::get_num_threads());
53 + TORCH_CHECK_EQ(omp_cpu_ids.size(), omp_get_max_threads());
54 +#pragma omp parallel for schedule(static, 1)
55 + for (size_t i = 0; i < omp_cpu_ids.size(); ++i) {
56 + cpu_set_t* mask = CPU_ALLOC(omp_cpu_mask->size);
57 + size_t size = CPU_ALLOC_SIZE(omp_cpu_mask->size);
58 + CPU_ZERO_S(size, mask);
59 + CPU_SET_S(omp_cpu_ids[i], size, mask);
60 + sched_setaffinity(0, sizeof(cpu_set_t), mask);
61 + CPU_FREE(mask);
62 + }
63 +
64 + numa_free_nodemask(omp_cpu_mask);
65 +}
docs/source/getting_started/cpu-installation.rst
+47 -8
@@ -10,6 +10,7 @@ Table of contents:
10 #. :ref:`Requirements <cpu_backend_requirements>`
11 #. :ref:`Quick start using Dockerfile <cpu_backend_quick_start_dockerfile>`
12 #. :ref:`Build from source <build_cpu_backend_from_source>`
13 +#. :ref:`Related runtime environment variables <env_intro>`
14 #. :ref:`Intel Extension for PyTorch <ipex_guidance>`
15 #. :ref:`Performance tips <cpu_backend_performance_tips>`
16
@@ -47,7 +48,7 @@ Build from source
48 .. code-block:: console
49
50 $ sudo apt-get update -y
50 - $ sudo apt-get install -y gcc-12 g++-12
51 + $ sudo apt-get install -y gcc-12 g++-12 libnuma-dev
52 $ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
53
54 - Second, install Python packages for vLLM CPU backend building:
@@ -71,6 +72,15 @@ Build from source
72
73 - If you want to force enable AVX512_BF16 for the cross-compilation, please set environment variable VLLM_CPU_AVX512BF16=1 before the building.
74
75 +.. _env_intro:
76 +
77 +Related runtime environment variables
78 +-------------------------------------
79 +
80 +- ``VLLM_CPU_KVCACHE_SPACE``: specify the KV Cache size (e.g, ``VLLM_CPU_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
81 +
82 +- ``VLLM_CPU_OMP_THREADS_BIND``: specify the CPU cores dedicated to the OpenMP threads. For example, ``VLLM_CPU_OMP_THREADS_BIND=0-31`` means there will be 32 OpenMP threads bound on 0-31 CPU cores. ``VLLM_CPU_OMP_THREADS_BIND=0-31|32-63`` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores.
83 +
84 .. _ipex_guidance:
85
86 Intel Extension for PyTorch
@@ -78,15 +88,11 @@ Intel Extension for PyTorch
88
89 - `Intel Extension for PyTorch (IPEX) <https://github.com/intel/intel-extension-for-pytorch>`_ extends PyTorch with up-to-date features optimizations for an extra performance boost on Intel hardware.
90
81 -- IPEX after the ``2.3.0`` can be enabled in the CPU backend by default if it is installed.
82 -
91 .. _cpu_backend_performance_tips:
92
93 Performance tips
94 -----------------
95
88 -- vLLM CPU backend uses environment variable ``VLLM_CPU_KVCACHE_SPACE`` to specify the KV Cache size (e.g, ``VLLM_CPU_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
89 -
96 - We highly recommend to use TCMalloc for high performance memory allocation and better cache locality. For example, on Ubuntu 22.4, you can run:
97
98 .. code-block:: console
@@ -96,11 +102,44 @@ Performance tips
102 $ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
103 $ python examples/offline_inference.py # run vLLM
104
99 -- vLLM CPU backend uses OpenMP for thread-parallel computation. If you want the best performance on CPU, it will be very critical to isolate CPU cores for OpenMP threads with other thread pools (like web-service event-loop), to avoid CPU oversubscription.
105 +- When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
106
101 -- If using vLLM CPU backend on a bare-metal machine, it is recommended to disable the hyper-threading.
107 +.. code-block:: console
108 +
109 + $ export VLLM_CPU_KVCACHE_SPACE=40
110 + $ export VLLM_CPU_OMP_THREADS_BIND=0-29
111 + $ vllm serve facebook/opt-125m
112 +
113 +- If using vLLM CPU backend on a machine with hyper-threading, it is recommended to bind only one OpenMP thread on each physical CPU core using ``VLLM_CPU_OMP_THREADS_BIND``. On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores:
114 +
115 +.. code-block:: console
116
103 -- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores and memory nodes, to avoid the remote memory node access. ``numactl`` is an useful tool for CPU core and memory binding on NUMA platform. Besides, ``--cpuset-cpus`` and ``--cpuset-mems`` arguments of ``docker run`` are also useful.
117 + $ lscpu -e # check the mapping between logical CPU cores and physical CPU cores
118 +
119 + # The "CPU" column means the logical CPU core IDs, and the "CORE" column means the physical core IDs. On this platform, two logical cores are sharing one physical core.
120 + CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
121 + 0 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
122 + 1 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
123 + 2 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
124 + 3 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
125 + 4 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
126 + 5 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
127 + 6 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
128 + 7 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
129 + 8 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
130 + 9 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
131 + 10 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
132 + 11 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
133 + 12 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
134 + 13 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
135 + 14 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
136 + 15 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
137 +
138 + # On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
139 + $ export VLLM_CPU_OMP_THREADS_BIND=0-7
140 + $ python examples/offline_inference.py
141 +
142 +- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using ``VLLM_CPU_OMP_THREADS_BIND`` to avoid cross NUMA node memory access.
143
144
145
requirements-cpu.txt
+2 -2
@@ -2,6 +2,6 @@
2 -r requirements-common.txt
3
4 # Dependencies for x86_64 CPUs
5 -torch == 2.3.1+cpu; platform_machine != "ppc64le"
6 -torchvision == 0.18.1+cpu; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
5 +torch == 2.4.0; platform_machine != "ppc64le"
6 +torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
7 triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
vllm/distributed/parallel_state.py
+3
@@ -296,6 +296,9 @@ class GroupCoordinator:
296 pynccl_comm = self.pynccl_comm
297 if (pynccl_comm is not None and not pynccl_comm.disabled):
298 pynccl_comm.all_reduce(input_)
299 + elif input_.is_cpu:
300 + import intel_extension_for_pytorch as ipex
301 + ipex.distributed.all_reduce(input_, group=self.device_group)
302 else:
303 torch.distributed.all_reduce(input_, group=self.device_group)
304 return input_
vllm/engine/async_llm_engine.py
-2
@@ -410,8 +410,6 @@ class AsyncLLMEngine:
410 from vllm.executor.tpu_executor import TPUExecutorAsync
411 executor_class = TPUExecutorAsync
412 elif engine_config.device_config.device_type == "cpu":
413 - assert distributed_executor_backend is None, (
414 - "Distributed execution is not supported with the CPU backend.")
413 from vllm.executor.cpu_executor import CPUExecutorAsync
414 executor_class = CPUExecutorAsync
415 elif engine_config.device_config.device_type == "openvino":
vllm/envs.py
+7 -1
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
29 VLLM_TRACE_FUNCTION: int = 0
30 VLLM_ATTENTION_BACKEND: Optional[str] = None
31 VLLM_CPU_KVCACHE_SPACE: int = 0
32 + VLLM_CPU_OMP_THREADS_BIND: str = ""
33 VLLM_OPENVINO_KVCACHE_SPACE: int = 0
34 VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None
35 VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
@@ -241,11 +242,16 @@ environment_variables: Dict[str, Callable[[], Any]] = {
242 "VLLM_ATTENTION_BACKEND":
243 lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
244
244 - # CPU key-value cache space
245 + # (CPU backend only) CPU key-value cache space.
246 # default is 4GB
247 "VLLM_CPU_KVCACHE_SPACE":
248 lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")),
249
250 + # (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
251 + # "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
252 + "VLLM_CPU_OMP_THREADS_BIND":
253 + lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "all"),
254 +
255 # OpenVINO key-value cache space
256 # default is 4GB
257 "VLLM_OPENVINO_KVCACHE_SPACE":
vllm/executor/cpu_executor.py
+227 -37
@@ -1,16 +1,21 @@
1 -from typing import List, Set, Tuple
1 +import os
2 +from functools import partial
3 +from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
4
5 import torch
6
7 import vllm.envs as envs
8 from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
9 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10 +from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
11 + ResultHandler, WorkerMonitor)
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 -from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13 - make_async)
16 +from vllm.utils import (get_distributed_init_method, get_open_port,
17 + get_vllm_instance_id, make_async)
18 +from vllm.worker.worker_base import WorkerWrapperBase
19
20 logger = init_logger(__name__)
21
@@ -22,46 +27,173 @@ class CPUExecutor(ExecutorBase):
27 def _init_executor(self) -> None:
28 assert self.device_config.device_type == "cpu"
29 assert self.lora_config is None, "cpu backend doesn't support LoRA"
30 +
31 + #
32 + # Environment variables for CPU executor
33 + #
34 +
35 + # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
36 + os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
37 +
38 + # Disable torch async compiling which won't work with daemonic processes
39 + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
40 +
41 + # Intel OpenMP setting
42 + ld_prealod_str = os.getenv("LD_PRELOAD", "")
43 + if "libiomp5.so" in ld_prealod_str:
44 + # The time(milliseconds) that a thread should wait after
45 + # completing the execution of a parallel region, before sleeping.
46 + os.environ['KMP_BLOCKTIME'] = "1"
47 + # Prevents the CPU to run into low performance state
48 + os.environ['KMP_TPAUSE'] = "0"
49 + # Provides fine granularity parallelism
50 + os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
51 + os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
52 + os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
53 +
54 + # To hint IPEX uses shared memory based AllReduce
55 + os.environ["LOCAL_WORLD_SIZE"] = str(
56 + self.parallel_config.tensor_parallel_size)
57 +
58 self.model_config = _verify_and_get_model_config(self.model_config)
59 self.cache_config = _verify_and_get_cache_config(self.cache_config)
60 self.scheduler_config = _verify_and_get_scheduler_config(
61 self.scheduler_config)
62
30 - # Instantiate the worker and load the model to CPU.
31 - self._init_worker()
32 -
33 - def _init_worker(self):
34 - from vllm.worker.cpu_worker import CPUWorker
63 + # Multiprocessing-based executor does not support multi-node setting.
64 + # Since it only works for single node, we can use the loopback address
65 + # 127.0.0.1 for communication.
66 + ip = "127.0.0.1"
67 + port = get_open_port()
68 + self.distributed_init_method = get_distributed_init_method(ip, port)
69 +
70 + is_async = isinstance(self, CPUExecutorAsync)
71 +
72 + world_size = self.parallel_config.tensor_parallel_size
73 + result_handler = ResultHandler()
74 + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
75 + self.workers = []
76 +
77 + if is_async:
78 + self.workers = [
79 + ProcessWorkerWrapper(
80 + result_handler,
81 + partial(
82 + self._create_worker,
83 + rank=rank,
84 + local_rank=rank,
85 + )) for rank in range(0, world_size)
86 + ]
87 + self.driver_worker = self.workers[0]
88 + self.workers = self.workers[1:]
89 + self.driver_method_invoker = _async_driver_method_invoker
90 + else:
91 + self.driver_worker = self._create_worker()
92 + self.driver_method_invoker = _driver_method_invoker
93 +
94 + if world_size != 1:
95 + self.workers = [
96 + ProcessWorkerWrapper(
97 + result_handler,
98 + partial(
99 + self._create_worker,
100 + rank=rank,
101 + local_rank=rank,
102 + )) for rank in range(1, world_size)
103 + ]
104 +
105 + if world_size != 1 or is_async:
106 + if is_async:
107 + async_worker_list = self.workers + [self.driver_worker]
108 + else:
109 + async_worker_list = self.workers
110 + self.worker_monitor = WorkerMonitor(async_worker_list,
111 + result_handler)
112 + result_handler.start()
113 + self.worker_monitor.start()
114 +
115 + self._run_workers("init_device")
116 + self._run_workers("load_model")
117 +
118 + def _create_worker(
119 + self,
120 + local_rank: int = 0,
121 + rank: int = 0,
122 + ):
123 + worker_module_name = "vllm.worker.cpu_worker"
124 + worker_class_name = "CPUWorker"
125 +
126 + wrapper = WorkerWrapperBase(
127 + worker_module_name=worker_module_name,
128 + worker_class_name=worker_class_name,
129 + )
130
36 - assert self.parallel_config.world_size == 1, (
37 - "CPUExecutor only supports single CPU socket currently.")
131 + assert self.distributed_init_method is not None
132
39 - distributed_init_method = get_distributed_init_method(
40 - get_ip(), get_open_port())
41 - self.driver_worker = CPUWorker(
133 + kwargs = dict(
134 model_config=self.model_config,
135 parallel_config=self.parallel_config,
136 scheduler_config=self.scheduler_config,
137 device_config=self.device_config,
138 cache_config=self.cache_config,
139 load_config=self.load_config,
48 - local_rank=0,
49 - rank=0,
50 - distributed_init_method=distributed_init_method,
140 + local_rank=local_rank,
141 + rank=rank,
142 + distributed_init_method=self.distributed_init_method,
143 lora_config=self.lora_config,
144 multimodal_config=self.multimodal_config,
145 kv_cache_dtype=self.cache_config.cache_dtype,
146 prompt_adapter_config=self.prompt_adapter_config,
55 - is_driver_worker=True,
147 + is_driver_worker=rank == 0,
148 )
57 - self.driver_worker.init_device()
58 - self.driver_worker.load_model()
149 + wrapper.init_worker(**kwargs)
150 +
151 + return wrapper.worker
152 +
153 + def _run_workers(
154 + self,
155 + method: str,
156 + *args,
157 + async_run_remote_workers_only: bool = False,
158 + max_concurrent_workers: Optional[int] = None,
159 + **kwargs,
160 + ) -> Any:
161 + """Runs the given method on all workers.
162 +
163 + Args:
164 + async_run_remote_workers_only: If True the method will be run only
165 + in the remote workers, not the driver worker. It will also be
166 + run asynchronously and return a list of futures rather than
167 + blocking on the results.
168 + """
169 +
170 + if max_concurrent_workers:
171 + raise NotImplementedError(
172 + "max_concurrent_workers is not supported yet.")
173 +
174 + # Start the workers first.
175 + worker_outputs = [
176 + worker.execute_method(method, *args, **kwargs)
177 + for worker in self.workers
178 + ]
179 +
180 + if async_run_remote_workers_only:
181 + # Just return futures
182 + return worker_outputs
183 +
184 + driver_worker_output = self.driver_method_invoker(
185 + self.driver_worker, method, *args, **kwargs)
186 +
187 + # Get the results of the workers.
188 + return [driver_worker_output
189 + ] + [output.get() for output in worker_outputs]
190
191 def determine_num_available_blocks(self) -> Tuple[int, int]:
192 """Determine the number of available KV blocks by invoking the
193 underlying worker.
194 """
64 - return self.driver_worker.determine_num_available_blocks()
195 + return self.driver_method_invoker(self.driver_worker,
196 + "determine_num_available_blocks")
197
198 def initialize_cache(self, num_gpu_blocks: int,
199 num_cpu_blocks: int) -> None:
@@ -74,43 +206,95 @@ class CPUExecutor(ExecutorBase):
206 # referred as `gpu block`. Because we want to reuse the existing block
207 # management procedure.
208 logger.info("# CPU blocks: %d", num_gpu_blocks)
77 - self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
209 +
210 + self._run_workers("initialize_cache",
211 + num_gpu_blocks=num_gpu_blocks,
212 + num_cpu_blocks=num_cpu_blocks)
213
214 def execute_model(
215 self,
216 execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
82 - output = self.driver_worker.execute_model(execute_model_req)
217 + if (self.parallel_config.tensor_parallel_size > 1
218 + and self.parallel_worker_tasks is None):
219 + self.parallel_worker_tasks = self._run_workers(
220 + "start_worker_execution_loop",
221 + async_run_remote_workers_only=True,
222 + )
223 + output = self.driver_method_invoker(self.driver_worker,
224 + "execute_model", execute_model_req)
225 return output
226
227 + def stop_remote_worker_execution_loop(self) -> None:
228 + if self.parallel_worker_tasks is None:
229 + return
230 + """
231 + Passing None will cause the driver to stop the model execution
232 + loop running in each of the remote workers.
233 + """
234 + self.driver_method_invoker(self.driver_worker, "execute_model", None)
235 + parallel_worker_tasks = self.parallel_worker_tasks
236 + self.parallel_worker_tasks = None
237 + # Ensure that workers exit model loop cleanly
238 + # (this will raise otherwise)
239 + self._wait_for_tasks_completion(parallel_worker_tasks)
240 +
241 def add_lora(self, lora_request: LoRARequest) -> bool:
86 - return self.driver_worker.add_lora(lora_request)
242 + return all(self._run_workers("add_lora", lora_request))
243
244 def remove_lora(self, lora_id: int) -> bool:
89 - return self.driver_worker.remove_lora(lora_id)
245 + return all(self._run_workers("remove_lora", lora_id))
246
247 def pin_lora(self, lora_id: int) -> bool:
92 - return self.driver_worker.pin_lora(lora_id)
248 + assert lora_id > 0, "lora_id must be greater than 0."
249 + return all(self._run_workers(
250 + "pin_lora",
251 + lora_id=lora_id,
252 + ))
253
254 def list_loras(self) -> Set[int]:
95 - return self.driver_worker.list_loras()
255 + return self.driver_method_invoker(self.driver_worker, "list_loras")
256
257 def add_prompt_adapter(
258 self, prompt_adapter_request: PromptAdapterRequest) -> bool:
99 - return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
259 + return all(
260 + self._run_workers(
261 + "add_prompt_adapter",
262 + prompt_adapter_request,
263 + ))
264
265 def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
102 - return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
266 + return all(
267 + self._run_workers(
268 + "remove_prompt_adapter",
269 + prompt_adapter_id,
270 + ))
271
272 def list_prompt_adapters(self) -> Set[int]:
105 - return self.driver_worker.list_prompt_adapters()
273 + return self.driver_method_invoker(self.driver_worker,
274 + "list_prompt_adapters")
275
276 def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
108 - return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
277 + return all(self._run_workers(
278 + "pin_prompt_adapter",
279 + prompt_adapter_id,
280 + ))
281
282 def check_health(self) -> None:
111 - # CPUExecutor will always be healthy as long as
112 - # it's running.
113 - return
283 + """Raises an error if engine is unhealthy."""
284 + if self.worker_monitor is not None and not self.worker_monitor.is_alive(
285 + ):
286 + raise RuntimeError("Worker processes are not running")
287 +
288 + def shutdown(self):
289 + if (worker_monitor := getattr(self, "worker_monitor",
290 + None)) is not None:
291 + worker_monitor.close()
292 +
293 + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
294 + """Wait for futures returned from _run_workers() with
295 + async_run_remote_workers_only to complete."""
296 + for result in parallel_worker_tasks:
297 + result.get()
298
299
300 class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
@@ -118,14 +302,12 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
302 async def execute_model_async(
303 self,
304 execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
121 - output = await make_async(self.driver_worker.execute_model
305 + output = await make_async(self.execute_model
306 )(execute_model_req=execute_model_req, )
307 return output
308
309 async def check_health_async(self) -> None:
126 - # CPUExecutor will always be healthy as long as
127 - # it's running.
128 - return
310 + self.check_health()
311
312
313 def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
@@ -170,3 +352,11 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
352 f" {kv_cache_space}, expect a positive integer value.")
353
354 return config
355 +
356 +
357 +def _driver_method_invoker(driver, method: str, *args, **kwargs):
358 + return getattr(driver, method)(*args, **kwargs)
359 +
360 +
361 +def _async_driver_method_invoker(driver, method: str, *args, **kwargs):
362 + return driver.execute_method(method, *args, **kwargs).get()
vllm/utils.py
-21
@@ -404,27 +404,6 @@ def update_environment_variables(envs: Dict[str, str]):
404 os.environ[k] = v
405
406
407 -def init_kmp_env():
408 - if not is_cpu():
409 - return
410 -
411 - ld_prealod_str = os.getenv("LD_PRELOAD", "")
412 - if "libiomp5.so" not in ld_prealod_str:
413 - return
414 -
415 - # The time(milliseconds) that a thread should wait after completing the
416 - # execution of a parallel region, before sleeping.
417 - os.environ['KMP_BLOCKTIME'] = "1"
418 - # dump settings on start up
419 - os.environ['KMP_SETTINGS'] = "1"
420 - # Prevents the CPU to run into low performance state
421 - os.environ['KMP_TPAUSE'] = "0"
422 - # Provides fine granularity parallelism
423 - os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
424 - os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
425 - os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
426 -
427 -
407 def chunk_list(lst: List[T], chunk_size: int):
408 """Yield successive chunk_size chunks from lst."""
409 for i in range(0, len(lst), chunk_size):
vllm/worker/cpu_model_runner.py
+4 -3
@@ -42,6 +42,7 @@ class CPUModelInput(ModelRunnerInputBase):
42 attn_metadata: Optional["AttentionMetadata"] = None
43 sampling_metadata: Optional["SamplingMetadata"] = None
44 multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
45 + virtual_engine: Optional[int] = None
46
47 def as_broadcastable_tensor_dict(
48 self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -204,8 +205,8 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
205 attn_metadata = self.attn_backend.make_metadata(
206 is_prompt=True,
207 seq_lens=seq_lens,
207 - seq_lens_tensor=None,
208 - max_decode_seq_len=None,
208 + seq_lens_tensor=torch.tensor([]),
209 + max_decode_seq_len=0,
210 num_prefills=len(seq_lens),
211 num_prefill_tokens=num_prompt_tokens,
212 num_decode_tokens=0,
@@ -345,7 +346,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
346 multi_modal_kwargs=multi_modal_kwargs,
347 )
348
348 - @torch.inference_mode()
349 + @torch.no_grad()
350 def execute_model(
351 self,
352 model_input: CPUModelInput,
vllm/worker/cpu_worker.py
+13 -4
@@ -4,6 +4,7 @@ from typing import Dict, List, Optional, Tuple
4 import torch
5 import torch.distributed
6
7 +import vllm.envs as envs
8 from vllm.attention import get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, MultiModalConfig, ParallelConfig,
@@ -13,7 +14,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
14 from vllm.logger import init_logger
15 from vllm.model_executor import set_random_seed
16 from vllm.sequence import ExecuteModelRequest
16 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, init_kmp_env
17 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18 from vllm.worker.cpu_model_runner import CPUModelRunner
19 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
20 LoraNotSupportedWorkerBase, WorkerInput)
@@ -152,13 +153,18 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
153 if self.is_driver_worker:
154 assert self.rank == 0, "The driver worker must have rank 0."
155
155 - # try to initialize intel openmp optimized tunings
156 - init_kmp_env()
157 -
156 if self.model_config.trust_remote_code:
157 # note: lazy import to avoid importing torch before initializing
158 from vllm.utils import init_cached_hf_modules
159 init_cached_hf_modules()
160 +
161 + # Setup OpenMP threads affinity.
162 + omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND
163 + if omp_cpuids == "all":
164 + self.local_omp_cpuid = "all"
165 + else:
166 + self.local_omp_cpuid = omp_cpuids.split("|")[rank]
167 +
168 self.model_runner: CPUModelRunner = CPUModelRunner(
169 model_config,
170 parallel_config,
@@ -177,6 +183,9 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
183 self.cpu_cache: List[List[torch.Tensor]]
184
185 def init_device(self) -> None:
186 + if self.local_omp_cpuid != "all":
187 + torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
188 +
189 self.init_distributed_environment()
190 # Set random seed.
191 set_random_seed(self.model_config.seed)