A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Hardware] [Intel] Enable Multiprocessing and tensor parallel in CPU backend and update documentation (#6125)
Li, Jiang committed
Jul 27, 2024 at 04:50 UTC
3bbb4936dc5aa7737750410ab4b4647817dcf9a3
14 files changed
+404
-90
.buildkite/run-cpu-test.sh
+20
-8
@@ -3,26 +3,38 @@
3
set -ex
4
5
# Try building the docker image
6
-docker build -t cpu-test -f Dockerfile.cpu .
7
-docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
6
+numactl -C 48-95 -N 1 docker build -t cpu-test -f Dockerfile.cpu .
7
+numactl -C 48-95 -N 1 docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" -t cpu-test-avx2 -f Dockerfile.cpu .
8
9
# Setup cleanup
10
remove_docker_container() { docker rm -f cpu-test cpu-test-avx2 || true; }
11
trap remove_docker_container EXIT
12
remove_docker_container
13
14
-# Run the image
14
+# Run the image, setting --shm-size=4g for tensor parallel.
15
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
16
- --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
16
+ --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test
17
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
18
- --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
18
+ --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
19
20
# offline inference
21
-docker exec cpu-test bash -c "python3 examples/offline_inference.py"
21
docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
22
23
# Run basic model test
25
-docker exec cpu-test bash -c "cd tests;
24
+docker exec cpu-test bash -c "
25
pip install pytest Pillow protobuf
27
- cd ../
26
pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py" # Mamba on CPU is not supported
27
+
28
+# online inference
29
+docker exec cpu-test bash -c "
30
+ export VLLM_CPU_KVCACHE_SPACE=10
31
+ export VLLM_CPU_OMP_THREADS_BIND=48-92
32
+ python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
33
+ timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
34
+ python3 benchmarks/benchmark_serving.py \
35
+ --backend vllm \
36
+ --dataset-name random \
37
+ --model facebook/opt-125m \
38
+ --num-prompts 20 \
39
+ --endpoint /v1/completions \
40
+ --tokenizer facebook/opt-125m"
Dockerfile.cpu
+5
-4
@@ -2,8 +2,8 @@
2
3
FROM ubuntu:22.04 AS cpu-test-1
4
5
-RUN apt-get update -y \
6
- && apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 \
5
+RUN apt-get update -y \
6
+ && apt-get install -y curl git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 libnuma-dev \
7
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
8
9
# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
@@ -13,8 +13,9 @@ RUN pip install intel-openmp
13
14
ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so:$LD_PRELOAD"
15
16
+RUN echo 'ulimit -c 0' >> ~/.bashrc
17
17
-RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.3.100%2Bgit0eb3473-cp310-cp310-linux_x86_64.whl
18
+RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.4.0%2Bgitfbaa4bc-cp310-cp310-linux_x86_64.whl
19
20
RUN pip install --upgrade pip \
21
&& pip install wheel packaging ninja "setuptools>=49.4.0" numpy
@@ -25,7 +26,7 @@ COPY ./ /workspace/vllm
26
27
WORKDIR /workspace/vllm
28
28
-RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/cpu
29
+RUN pip install -v -r requirements-cpu.txt --extra-index-url https://download.pytorch.org/whl/test/cpu
30
31
# Support for building with non-AVX512 vLLM: docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" ...
32
ARG VLLM_CPU_DISABLE_AVX512
cmake/cpu_extension.cmake
+4
@@ -83,6 +83,8 @@ endif()
83
84
message(STATUS "CPU extension compile flags: ${CXX_COMPILE_FLAGS}")
85
86
+list(APPEND LIBS "numa")
87
+
88
89
#
90
# Define extension targets
@@ -95,6 +97,7 @@ set(VLLM_EXT_SRC
97
"csrc/cpu/activation.cpp"
98
"csrc/cpu/attention.cpp"
99
"csrc/cpu/cache.cpp"
100
+ "csrc/cpu/utils.cpp"
101
"csrc/cpu/layernorm.cpp"
102
"csrc/cpu/pos_encoding.cpp"
103
"csrc/cpu/torch_bindings.cpp")
@@ -104,6 +107,7 @@ define_gpu_extension_target(
107
DESTINATION vllm
108
LANGUAGE CXX
109
SOURCES ${VLLM_EXT_SRC}
110
+ LIBRARIES ${LIBS}
111
COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
112
USE_SABI 3
113
WITH_SOABI
csrc/cpu/torch_bindings.cpp
+7
@@ -4,6 +4,8 @@
4
5
#include <torch/library.h>
6
7
+void init_cpu_threads_env(const std::string& cpu_ids);
8
+
9
TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
10
// vLLM custom ops
11
@@ -107,4 +109,9 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
109
cache_ops.impl("reshape_and_cache", torch::kCPU, &reshape_and_cache);
110
}
111
112
+TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _utils), utils) {
113
+ // CPU utils
114
+ utils.def("init_cpu_threads_env(str cpu_ids) -> ()", &init_cpu_threads_env);
115
+}
116
+
117
REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/cpu/utils.cpp
new
+65
@@ -0,0 +1,65 @@
1
+#include <numa.h>
2
+#include <unistd.h>
3
+#include <string>
4
+#include <sched.h>
5
+
6
+#include "cpu_types.hpp"
7
+
8
+void init_cpu_threads_env(const std::string& cpu_ids) {
9
+ bitmask* omp_cpu_mask = numa_parse_cpustring(cpu_ids.c_str());
10
+ TORCH_CHECK(omp_cpu_mask->size > 0);
11
+ std::vector<int> omp_cpu_ids;
12
+ omp_cpu_ids.reserve(omp_cpu_mask->size);
13
+
14
+ constexpr int group_size = 8 * sizeof(*omp_cpu_mask->maskp);
15
+
16
+ for (int offset = 0; offset < omp_cpu_mask->size; offset += group_size) {
17
+ unsigned long group_mask = omp_cpu_mask->maskp[offset / group_size];
18
+ int i = 0;
19
+ while (group_mask) {
20
+ if (group_mask & 1) {
21
+ omp_cpu_ids.emplace_back(offset + i);
22
+ }
23
+ ++i;
24
+ group_mask >>= 1;
25
+ }
26
+ }
27
+
28
+ // Memory node binding
29
+ if (numa_available() != -1) {
30
+ int mem_node_id = numa_node_of_cpu(omp_cpu_ids.front());
31
+ bitmask* mask = numa_parse_nodestring(std::to_string(mem_node_id).c_str());
32
+ bitmask* src_mask = numa_get_membind();
33
+
34
+ int pid = getpid();
35
+
36
+ // move all existing pages to the specified numa node.
37
+ *(src_mask->maskp) = *(src_mask->maskp) ^ *(mask->maskp);
38
+ int page_num = numa_migrate_pages(pid, src_mask, mask);
39
+ if (page_num == -1) {
40
+ TORCH_CHECK(false,
41
+ "numa_migrate_pages failed. errno: " + std::to_string(errno));
42
+ }
43
+
44
+ // restrict memory allocation node.
45
+ numa_set_membind(mask);
46
+ numa_set_strict(1);
47
+ }
48
+
49
+ // OMP threads binding
50
+ omp_set_num_threads((int)omp_cpu_ids.size());
51
+ torch::set_num_threads((int)omp_cpu_ids.size());
52
+ TORCH_CHECK_EQ(omp_cpu_ids.size(), torch::get_num_threads());
53
+ TORCH_CHECK_EQ(omp_cpu_ids.size(), omp_get_max_threads());
54
+#pragma omp parallel for schedule(static, 1)
55
+ for (size_t i = 0; i < omp_cpu_ids.size(); ++i) {
56
+ cpu_set_t* mask = CPU_ALLOC(omp_cpu_mask->size);
57
+ size_t size = CPU_ALLOC_SIZE(omp_cpu_mask->size);
58
+ CPU_ZERO_S(size, mask);
59
+ CPU_SET_S(omp_cpu_ids[i], size, mask);
60
+ sched_setaffinity(0, sizeof(cpu_set_t), mask);
61
+ CPU_FREE(mask);
62
+ }
63
+
64
+ numa_free_nodemask(omp_cpu_mask);
65
+}
docs/source/getting_started/cpu-installation.rst
+47
-8
@@ -10,6 +10,7 @@ Table of contents:
10
#. :ref:`Requirements <cpu_backend_requirements>`
11
#. :ref:`Quick start using Dockerfile <cpu_backend_quick_start_dockerfile>`
12
#. :ref:`Build from source <build_cpu_backend_from_source>`
13
+#. :ref:`Related runtime environment variables <env_intro>`
14
#. :ref:`Intel Extension for PyTorch <ipex_guidance>`
15
#. :ref:`Performance tips <cpu_backend_performance_tips>`
16
@@ -47,7 +48,7 @@ Build from source
48
.. code-block:: console
49
50
$ sudo apt-get update -y
50
- $ sudo apt-get install -y gcc-12 g++-12
51
+ $ sudo apt-get install -y gcc-12 g++-12 libnuma-dev
52
$ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
53
54
- Second, install Python packages for vLLM CPU backend building:
@@ -71,6 +72,15 @@ Build from source
72
73
- If you want to force enable AVX512_BF16 for the cross-compilation, please set environment variable VLLM_CPU_AVX512BF16=1 before the building.
74
75
+.. _env_intro:
76
+
77
+Related runtime environment variables
78
+-------------------------------------
79
+
80
+- ``VLLM_CPU_KVCACHE_SPACE``: specify the KV Cache size (e.g, ``VLLM_CPU_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
81
+
82
+- ``VLLM_CPU_OMP_THREADS_BIND``: specify the CPU cores dedicated to the OpenMP threads. For example, ``VLLM_CPU_OMP_THREADS_BIND=0-31`` means there will be 32 OpenMP threads bound on 0-31 CPU cores. ``VLLM_CPU_OMP_THREADS_BIND=0-31|32-63`` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores.
83
+
84
.. _ipex_guidance:
85
86
Intel Extension for PyTorch
@@ -78,15 +88,11 @@ Intel Extension for PyTorch
88
89
- `Intel Extension for PyTorch (IPEX) <https://github.com/intel/intel-extension-for-pytorch>`_ extends PyTorch with up-to-date features optimizations for an extra performance boost on Intel hardware.
90
81
-- IPEX after the ``2.3.0`` can be enabled in the CPU backend by default if it is installed.
82
-
91
.. _cpu_backend_performance_tips:
92
93
Performance tips
94
-----------------
95
88
-- vLLM CPU backend uses environment variable ``VLLM_CPU_KVCACHE_SPACE`` to specify the KV Cache size (e.g, ``VLLM_CPU_KVCACHE_SPACE=40`` means 40 GB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users.
89
-
96
- We highly recommend to use TCMalloc for high performance memory allocation and better cache locality. For example, on Ubuntu 22.4, you can run:
97
98
.. code-block:: console
@@ -96,11 +102,44 @@ Performance tips
102
$ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD # prepend the library to LD_PRELOAD
103
$ python examples/offline_inference.py # run vLLM
104
99
-- vLLM CPU backend uses OpenMP for thread-parallel computation. If you want the best performance on CPU, it will be very critical to isolate CPU cores for OpenMP threads with other thread pools (like web-service event-loop), to avoid CPU oversubscription.
105
+- When using the online serving, it is recommended to reserve 1-2 CPU cores for the serving framework to avoid CPU oversubscription. For example, on a platform with 32 physical CPU cores, reserving CPU 30 and 31 for the framework and using CPU 0-29 for OpenMP:
106
101
-- If using vLLM CPU backend on a bare-metal machine, it is recommended to disable the hyper-threading.
107
+.. code-block:: console
108
+
109
+ $ export VLLM_CPU_KVCACHE_SPACE=40
110
+ $ export VLLM_CPU_OMP_THREADS_BIND=0-29
111
+ $ vllm serve facebook/opt-125m
112
+
113
+- If using vLLM CPU backend on a machine with hyper-threading, it is recommended to bind only one OpenMP thread on each physical CPU core using ``VLLM_CPU_OMP_THREADS_BIND``. On a hyper-threading enabled platform with 16 logical CPU cores / 8 physical CPU cores:
114
+
115
+.. code-block:: console
116
103
-- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores and memory nodes, to avoid the remote memory node access. ``numactl`` is an useful tool for CPU core and memory binding on NUMA platform. Besides, ``--cpuset-cpus`` and ``--cpuset-mems`` arguments of ``docker run`` are also useful.
117
+ $ lscpu -e # check the mapping between logical CPU cores and physical CPU cores
118
+
119
+ # The "CPU" column means the logical CPU core IDs, and the "CORE" column means the physical core IDs. On this platform, two logical cores are sharing one physical core.
120
+ CPU NODE SOCKET CORE L1d:L1i:L2:L3 ONLINE MAXMHZ MINMHZ MHZ
121
+ 0 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
122
+ 1 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
123
+ 2 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
124
+ 3 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
125
+ 4 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
126
+ 5 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
127
+ 6 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
128
+ 7 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
129
+ 8 0 0 0 0:0:0:0 yes 2401.0000 800.0000 800.000
130
+ 9 0 0 1 1:1:1:0 yes 2401.0000 800.0000 800.000
131
+ 10 0 0 2 2:2:2:0 yes 2401.0000 800.0000 800.000
132
+ 11 0 0 3 3:3:3:0 yes 2401.0000 800.0000 800.000
133
+ 12 0 0 4 4:4:4:0 yes 2401.0000 800.0000 800.000
134
+ 13 0 0 5 5:5:5:0 yes 2401.0000 800.0000 800.000
135
+ 14 0 0 6 6:6:6:0 yes 2401.0000 800.0000 800.000
136
+ 15 0 0 7 7:7:7:0 yes 2401.0000 800.0000 800.000
137
+
138
+ # On this platform, it is recommend to only bind openMP threads on logical CPU cores 0-7 or 8-15
139
+ $ export VLLM_CPU_OMP_THREADS_BIND=0-7
140
+ $ python examples/offline_inference.py
141
+
142
+- If using vLLM CPU backend on a multi-socket machine with NUMA, be aware to set CPU cores using ``VLLM_CPU_OMP_THREADS_BIND`` to avoid cross NUMA node memory access.
143
144
145
requirements-cpu.txt
+2
-2
@@ -2,6 +2,6 @@
2
-r requirements-common.txt
3
4
# Dependencies for x86_64 CPUs
5
-torch == 2.3.1+cpu; platform_machine != "ppc64le"
6
-torchvision == 0.18.1+cpu; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
5
+torch == 2.4.0; platform_machine != "ppc64le"
6
+torchvision; platform_machine != "ppc64le" # required for the image processor of phi3v, this must be updated alongside torch
7
triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
vllm/distributed/parallel_state.py
+3
@@ -296,6 +296,9 @@ class GroupCoordinator:
296
pynccl_comm = self.pynccl_comm
297
if (pynccl_comm is not None and not pynccl_comm.disabled):
298
pynccl_comm.all_reduce(input_)
299
+ elif input_.is_cpu:
300
+ import intel_extension_for_pytorch as ipex
301
+ ipex.distributed.all_reduce(input_, group=self.device_group)
302
else:
303
torch.distributed.all_reduce(input_, group=self.device_group)
304
return input_
vllm/engine/async_llm_engine.py
-2
@@ -410,8 +410,6 @@ class AsyncLLMEngine:
410
from vllm.executor.tpu_executor import TPUExecutorAsync
411
executor_class = TPUExecutorAsync
412
elif engine_config.device_config.device_type == "cpu":
413
- assert distributed_executor_backend is None, (
414
- "Distributed execution is not supported with the CPU backend.")
413
from vllm.executor.cpu_executor import CPUExecutorAsync
414
executor_class = CPUExecutorAsync
415
elif engine_config.device_config.device_type == "openvino":
vllm/envs.py
+7
-1
@@ -29,6 +29,7 @@ if TYPE_CHECKING:
29
VLLM_TRACE_FUNCTION: int = 0
30
VLLM_ATTENTION_BACKEND: Optional[str] = None
31
VLLM_CPU_KVCACHE_SPACE: int = 0
32
+ VLLM_CPU_OMP_THREADS_BIND: str = ""
33
VLLM_OPENVINO_KVCACHE_SPACE: int = 0
34
VLLM_OPENVINO_CPU_KV_CACHE_PRECISION: Optional[str] = None
35
VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
@@ -241,11 +242,16 @@ environment_variables: Dict[str, Callable[[], Any]] = {
242
"VLLM_ATTENTION_BACKEND":
243
lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
244
244
- # CPU key-value cache space
245
+ # (CPU backend only) CPU key-value cache space.
246
# default is 4GB
247
"VLLM_CPU_KVCACHE_SPACE":
248
lambda: int(os.getenv("VLLM_CPU_KVCACHE_SPACE", "0")),
249
250
+ # (CPU backend only) CPU core ids bound by OpenMP threads, e.g., "0-31",
251
+ # "0,1,2", "0-31,33". CPU cores of different ranks are separated by '|'.
252
+ "VLLM_CPU_OMP_THREADS_BIND":
253
+ lambda: os.getenv("VLLM_CPU_OMP_THREADS_BIND", "all"),
254
+
255
# OpenVINO key-value cache space
256
# default is 4GB
257
"VLLM_OPENVINO_KVCACHE_SPACE":
vllm/executor/cpu_executor.py
+227
-37
@@ -1,16 +1,21 @@
1
-from typing import List, Set, Tuple
1
+import os
2
+from functools import partial
3
+from typing import Any, Awaitable, List, Optional, Set, Tuple, Union
4
5
import torch
6
7
import vllm.envs as envs
8
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig
9
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10
+from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
11
+ ResultHandler, WorkerMonitor)
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
-from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13
- make_async)
16
+from vllm.utils import (get_distributed_init_method, get_open_port,
17
+ get_vllm_instance_id, make_async)
18
+from vllm.worker.worker_base import WorkerWrapperBase
19
20
logger = init_logger(__name__)
21
@@ -22,46 +27,173 @@ class CPUExecutor(ExecutorBase):
27
def _init_executor(self) -> None:
28
assert self.device_config.device_type == "cpu"
29
assert self.lora_config is None, "cpu backend doesn't support LoRA"
30
+
31
+ #
32
+ # Environment variables for CPU executor
33
+ #
34
+
35
+ # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
36
+ os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
37
+
38
+ # Disable torch async compiling which won't work with daemonic processes
39
+ os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
40
+
41
+ # Intel OpenMP setting
42
+ ld_prealod_str = os.getenv("LD_PRELOAD", "")
43
+ if "libiomp5.so" in ld_prealod_str:
44
+ # The time(milliseconds) that a thread should wait after
45
+ # completing the execution of a parallel region, before sleeping.
46
+ os.environ['KMP_BLOCKTIME'] = "1"
47
+ # Prevents the CPU to run into low performance state
48
+ os.environ['KMP_TPAUSE'] = "0"
49
+ # Provides fine granularity parallelism
50
+ os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
51
+ os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
52
+ os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
53
+
54
+ # To hint IPEX uses shared memory based AllReduce
55
+ os.environ["LOCAL_WORLD_SIZE"] = str(
56
+ self.parallel_config.tensor_parallel_size)
57
+
58
self.model_config = _verify_and_get_model_config(self.model_config)
59
self.cache_config = _verify_and_get_cache_config(self.cache_config)
60
self.scheduler_config = _verify_and_get_scheduler_config(
61
self.scheduler_config)
62
30
- # Instantiate the worker and load the model to CPU.
31
- self._init_worker()
32
-
33
- def _init_worker(self):
34
- from vllm.worker.cpu_worker import CPUWorker
63
+ # Multiprocessing-based executor does not support multi-node setting.
64
+ # Since it only works for single node, we can use the loopback address
65
+ # 127.0.0.1 for communication.
66
+ ip = "127.0.0.1"
67
+ port = get_open_port()
68
+ self.distributed_init_method = get_distributed_init_method(ip, port)
69
+
70
+ is_async = isinstance(self, CPUExecutorAsync)
71
+
72
+ world_size = self.parallel_config.tensor_parallel_size
73
+ result_handler = ResultHandler()
74
+ self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
75
+ self.workers = []
76
+
77
+ if is_async:
78
+ self.workers = [
79
+ ProcessWorkerWrapper(
80
+ result_handler,
81
+ partial(
82
+ self._create_worker,
83
+ rank=rank,
84
+ local_rank=rank,
85
+ )) for rank in range(0, world_size)
86
+ ]
87
+ self.driver_worker = self.workers[0]
88
+ self.workers = self.workers[1:]
89
+ self.driver_method_invoker = _async_driver_method_invoker
90
+ else:
91
+ self.driver_worker = self._create_worker()
92
+ self.driver_method_invoker = _driver_method_invoker
93
+
94
+ if world_size != 1:
95
+ self.workers = [
96
+ ProcessWorkerWrapper(
97
+ result_handler,
98
+ partial(
99
+ self._create_worker,
100
+ rank=rank,
101
+ local_rank=rank,
102
+ )) for rank in range(1, world_size)
103
+ ]
104
+
105
+ if world_size != 1 or is_async:
106
+ if is_async:
107
+ async_worker_list = self.workers + [self.driver_worker]
108
+ else:
109
+ async_worker_list = self.workers
110
+ self.worker_monitor = WorkerMonitor(async_worker_list,
111
+ result_handler)
112
+ result_handler.start()
113
+ self.worker_monitor.start()
114
+
115
+ self._run_workers("init_device")
116
+ self._run_workers("load_model")
117
+
118
+ def _create_worker(
119
+ self,
120
+ local_rank: int = 0,
121
+ rank: int = 0,
122
+ ):
123
+ worker_module_name = "vllm.worker.cpu_worker"
124
+ worker_class_name = "CPUWorker"
125
+
126
+ wrapper = WorkerWrapperBase(
127
+ worker_module_name=worker_module_name,
128
+ worker_class_name=worker_class_name,
129
+ )
130
36
- assert self.parallel_config.world_size == 1, (
37
- "CPUExecutor only supports single CPU socket currently.")
131
+ assert self.distributed_init_method is not None
132
39
- distributed_init_method = get_distributed_init_method(
40
- get_ip(), get_open_port())
41
- self.driver_worker = CPUWorker(
133
+ kwargs = dict(
134
model_config=self.model_config,
135
parallel_config=self.parallel_config,
136
scheduler_config=self.scheduler_config,
137
device_config=self.device_config,
138
cache_config=self.cache_config,
139
load_config=self.load_config,
48
- local_rank=0,
49
- rank=0,
50
- distributed_init_method=distributed_init_method,
140
+ local_rank=local_rank,
141
+ rank=rank,
142
+ distributed_init_method=self.distributed_init_method,
143
lora_config=self.lora_config,
144
multimodal_config=self.multimodal_config,
145
kv_cache_dtype=self.cache_config.cache_dtype,
146
prompt_adapter_config=self.prompt_adapter_config,
55
- is_driver_worker=True,
147
+ is_driver_worker=rank == 0,
148
)
57
- self.driver_worker.init_device()
58
- self.driver_worker.load_model()
149
+ wrapper.init_worker(**kwargs)
150
+
151
+ return wrapper.worker
152
+
153
+ def _run_workers(
154
+ self,
155
+ method: str,
156
+ *args,
157
+ async_run_remote_workers_only: bool = False,
158
+ max_concurrent_workers: Optional[int] = None,
159
+ **kwargs,
160
+ ) -> Any:
161
+ """Runs the given method on all workers.
162
+
163
+ Args:
164
+ async_run_remote_workers_only: If True the method will be run only
165
+ in the remote workers, not the driver worker. It will also be
166
+ run asynchronously and return a list of futures rather than
167
+ blocking on the results.
168
+ """
169
+
170
+ if max_concurrent_workers:
171
+ raise NotImplementedError(
172
+ "max_concurrent_workers is not supported yet.")
173
+
174
+ # Start the workers first.
175
+ worker_outputs = [
176
+ worker.execute_method(method, *args, **kwargs)
177
+ for worker in self.workers
178
+ ]
179
+
180
+ if async_run_remote_workers_only:
181
+ # Just return futures
182
+ return worker_outputs
183
+
184
+ driver_worker_output = self.driver_method_invoker(
185
+ self.driver_worker, method, *args, **kwargs)
186
+
187
+ # Get the results of the workers.
188
+ return [driver_worker_output
189
+ ] + [output.get() for output in worker_outputs]
190
191
def determine_num_available_blocks(self) -> Tuple[int, int]:
192
"""Determine the number of available KV blocks by invoking the
193
underlying worker.
194
"""
64
- return self.driver_worker.determine_num_available_blocks()
195
+ return self.driver_method_invoker(self.driver_worker,
196
+ "determine_num_available_blocks")
197
198
def initialize_cache(self, num_gpu_blocks: int,
199
num_cpu_blocks: int) -> None:
@@ -74,43 +206,95 @@ class CPUExecutor(ExecutorBase):
206
# referred as `gpu block`. Because we want to reuse the existing block
207
# management procedure.
208
logger.info("# CPU blocks: %d", num_gpu_blocks)
77
- self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
209
+
210
+ self._run_workers("initialize_cache",
211
+ num_gpu_blocks=num_gpu_blocks,
212
+ num_cpu_blocks=num_cpu_blocks)
213
214
def execute_model(
215
self,
216
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
82
- output = self.driver_worker.execute_model(execute_model_req)
217
+ if (self.parallel_config.tensor_parallel_size > 1
218
+ and self.parallel_worker_tasks is None):
219
+ self.parallel_worker_tasks = self._run_workers(
220
+ "start_worker_execution_loop",
221
+ async_run_remote_workers_only=True,
222
+ )
223
+ output = self.driver_method_invoker(self.driver_worker,
224
+ "execute_model", execute_model_req)
225
return output
226
227
+ def stop_remote_worker_execution_loop(self) -> None:
228
+ if self.parallel_worker_tasks is None:
229
+ return
230
+ """
231
+ Passing None will cause the driver to stop the model execution
232
+ loop running in each of the remote workers.
233
+ """
234
+ self.driver_method_invoker(self.driver_worker, "execute_model", None)
235
+ parallel_worker_tasks = self.parallel_worker_tasks
236
+ self.parallel_worker_tasks = None
237
+ # Ensure that workers exit model loop cleanly
238
+ # (this will raise otherwise)
239
+ self._wait_for_tasks_completion(parallel_worker_tasks)
240
+
241
def add_lora(self, lora_request: LoRARequest) -> bool:
86
- return self.driver_worker.add_lora(lora_request)
242
+ return all(self._run_workers("add_lora", lora_request))
243
244
def remove_lora(self, lora_id: int) -> bool:
89
- return self.driver_worker.remove_lora(lora_id)
245
+ return all(self._run_workers("remove_lora", lora_id))
246
247
def pin_lora(self, lora_id: int) -> bool:
92
- return self.driver_worker.pin_lora(lora_id)
248
+ assert lora_id > 0, "lora_id must be greater than 0."
249
+ return all(self._run_workers(
250
+ "pin_lora",
251
+ lora_id=lora_id,
252
+ ))
253
254
def list_loras(self) -> Set[int]:
95
- return self.driver_worker.list_loras()
255
+ return self.driver_method_invoker(self.driver_worker, "list_loras")
256
257
def add_prompt_adapter(
258
self, prompt_adapter_request: PromptAdapterRequest) -> bool:
99
- return self.driver_worker.add_prompt_adapter(prompt_adapter_request)
259
+ return all(
260
+ self._run_workers(
261
+ "add_prompt_adapter",
262
+ prompt_adapter_request,
263
+ ))
264
265
def remove_prompt_adapter(self, prompt_adapter_id: int) -> bool:
102
- return self.driver_worker.remove_prompt_adapter(prompt_adapter_id)
266
+ return all(
267
+ self._run_workers(
268
+ "remove_prompt_adapter",
269
+ prompt_adapter_id,
270
+ ))
271
272
def list_prompt_adapters(self) -> Set[int]:
105
- return self.driver_worker.list_prompt_adapters()
273
+ return self.driver_method_invoker(self.driver_worker,
274
+ "list_prompt_adapters")
275
276
def pin_prompt_adapter(self, prompt_adapter_id: int) -> bool:
108
- return self.driver_worker.pin_prompt_adapter(prompt_adapter_id)
277
+ return all(self._run_workers(
278
+ "pin_prompt_adapter",
279
+ prompt_adapter_id,
280
+ ))
281
282
def check_health(self) -> None:
111
- # CPUExecutor will always be healthy as long as
112
- # it's running.
113
- return
283
+ """Raises an error if engine is unhealthy."""
284
+ if self.worker_monitor is not None and not self.worker_monitor.is_alive(
285
+ ):
286
+ raise RuntimeError("Worker processes are not running")
287
+
288
+ def shutdown(self):
289
+ if (worker_monitor := getattr(self, "worker_monitor",
290
+ None)) is not None:
291
+ worker_monitor.close()
292
+
293
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
294
+ """Wait for futures returned from _run_workers() with
295
+ async_run_remote_workers_only to complete."""
296
+ for result in parallel_worker_tasks:
297
+ result.get()
298
299
300
class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
@@ -118,14 +302,12 @@ class CPUExecutorAsync(CPUExecutor, ExecutorAsyncBase):
302
async def execute_model_async(
303
self,
304
execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
121
- output = await make_async(self.driver_worker.execute_model
305
+ output = await make_async(self.execute_model
306
)(execute_model_req=execute_model_req, )
307
return output
308
309
async def check_health_async(self) -> None:
126
- # CPUExecutor will always be healthy as long as
127
- # it's running.
128
- return
310
+ self.check_health()
311
312
313
def _verify_and_get_model_config(config: ModelConfig) -> ModelConfig:
@@ -170,3 +352,11 @@ def _verify_and_get_cache_config(config: CacheConfig) -> CacheConfig:
352
f" {kv_cache_space}, expect a positive integer value.")
353
354
return config
355
+
356
+
357
+def _driver_method_invoker(driver, method: str, *args, **kwargs):
358
+ return getattr(driver, method)(*args, **kwargs)
359
+
360
+
361
+def _async_driver_method_invoker(driver, method: str, *args, **kwargs):
362
+ return driver.execute_method(method, *args, **kwargs).get()
vllm/utils.py
-21
@@ -404,27 +404,6 @@ def update_environment_variables(envs: Dict[str, str]):
404
os.environ[k] = v
405
406
407
-def init_kmp_env():
408
- if not is_cpu():
409
- return
410
-
411
- ld_prealod_str = os.getenv("LD_PRELOAD", "")
412
- if "libiomp5.so" not in ld_prealod_str:
413
- return
414
-
415
- # The time(milliseconds) that a thread should wait after completing the
416
- # execution of a parallel region, before sleeping.
417
- os.environ['KMP_BLOCKTIME'] = "1"
418
- # dump settings on start up
419
- os.environ['KMP_SETTINGS'] = "1"
420
- # Prevents the CPU to run into low performance state
421
- os.environ['KMP_TPAUSE'] = "0"
422
- # Provides fine granularity parallelism
423
- os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
424
- os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
425
- os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
426
-
427
-
407
def chunk_list(lst: List[T], chunk_size: int):
408
"""Yield successive chunk_size chunks from lst."""
409
for i in range(0, len(lst), chunk_size):
vllm/worker/cpu_model_runner.py
+4
-3
@@ -42,6 +42,7 @@ class CPUModelInput(ModelRunnerInputBase):
42
attn_metadata: Optional["AttentionMetadata"] = None
43
sampling_metadata: Optional["SamplingMetadata"] = None
44
multi_modal_kwargs: Optional[Mapping[str, BatchedTensors]] = None
45
+ virtual_engine: Optional[int] = None
46
47
def as_broadcastable_tensor_dict(
48
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -204,8 +205,8 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
205
attn_metadata = self.attn_backend.make_metadata(
206
is_prompt=True,
207
seq_lens=seq_lens,
207
- seq_lens_tensor=None,
208
- max_decode_seq_len=None,
208
+ seq_lens_tensor=torch.tensor([]),
209
+ max_decode_seq_len=0,
210
num_prefills=len(seq_lens),
211
num_prefill_tokens=num_prompt_tokens,
212
num_decode_tokens=0,
@@ -345,7 +346,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
346
multi_modal_kwargs=multi_modal_kwargs,
347
)
348
348
- @torch.inference_mode()
349
+ @torch.no_grad()
350
def execute_model(
351
self,
352
model_input: CPUModelInput,
vllm/worker/cpu_worker.py
+13
-4
@@ -4,6 +4,7 @@ from typing import Dict, List, Optional, Tuple
4
import torch
5
import torch.distributed
6
7
+import vllm.envs as envs
8
from vllm.attention import get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, MultiModalConfig, ParallelConfig,
@@ -13,7 +14,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
14
from vllm.logger import init_logger
15
from vllm.model_executor import set_random_seed
16
from vllm.sequence import ExecuteModelRequest
16
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, init_kmp_env
17
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
18
from vllm.worker.cpu_model_runner import CPUModelRunner
19
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
20
LoraNotSupportedWorkerBase, WorkerInput)
@@ -152,13 +153,18 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
153
if self.is_driver_worker:
154
assert self.rank == 0, "The driver worker must have rank 0."
155
155
- # try to initialize intel openmp optimized tunings
156
- init_kmp_env()
157
-
156
if self.model_config.trust_remote_code:
157
# note: lazy import to avoid importing torch before initializing
158
from vllm.utils import init_cached_hf_modules
159
init_cached_hf_modules()
160
+
161
+ # Setup OpenMP threads affinity.
162
+ omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND
163
+ if omp_cpuids == "all":
164
+ self.local_omp_cpuid = "all"
165
+ else:
166
+ self.local_omp_cpuid = omp_cpuids.split("|")[rank]
167
+
168
self.model_runner: CPUModelRunner = CPUModelRunner(
169
model_config,
170
parallel_config,
@@ -177,6 +183,9 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
183
self.cpu_cache: List[List[torch.Tensor]]
184
185
def init_device(self) -> None:
186
+ if self.local_omp_cpuid != "all":
187
+ torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
188
+
189
self.init_distributed_environment()
190
# Set random seed.
191
set_random_seed(self.model_config.seed)