A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Hardware][Intel CPU] Adding intel openmp tunings in Docker file (#6008)
Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
Yuan committed
Jul 5, 2024 at 06:22 UTC
81d7a50f2402ef5b622ac8d3a081994a1a4641b0
4 files changed
+37
-5
.buildkite/run-cpu-test.sh
+4
-2
@@ -12,8 +12,10 @@ trap remove_docker_container EXIT
12
remove_docker_container
13
14
# Run the image
15
-docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
16
-docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
15
+docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
16
+ --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
17
+docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
18
+ --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
19
20
# offline inference
21
docker exec cpu-test bash -c "python3 examples/offline_inference.py"
Dockerfile.cpu
+8
-2
@@ -6,7 +6,13 @@ RUN apt-get update -y \
6
&& apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 \
7
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
8
9
-RUN echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD' >> ~/.bashrc
9
+# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
10
+# intel-openmp provides additional performance improvement vs. openmp
11
+# tcmalloc provides better memory allocation efficiency, e.g, holding memory in caches to speed up access of commonly-used objects.
12
+RUN pip install intel-openmp
13
+
14
+ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so:$LD_PRELOAD"
15
+
16
17
RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.3.100%2Bgit0eb3473-cp310-cp310-linux_x86_64.whl
18
@@ -31,4 +37,4 @@ WORKDIR /workspace/
37
38
RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
39
34
-CMD ["/bin/bash"]
40
+ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
vllm/utils.py
+21
@@ -398,6 +398,27 @@ def update_environment_variables(envs: Dict[str, str]):
398
os.environ[k] = v
399
400
401
+def init_kmp_env():
402
+ if not is_cpu():
403
+ return
404
+
405
+ ld_prealod_str = os.getenv("LD_PRELOAD", "")
406
+ if "libiomp5.so" not in ld_prealod_str:
407
+ return
408
+
409
+ # The time(milliseconds) that a thread should wait after completing the
410
+ # execution of a parallel region, before sleeping.
411
+ os.environ['KMP_BLOCKTIME'] = "1"
412
+ # dump settings on start up
413
+ os.environ['KMP_SETTINGS'] = "1"
414
+ # Prevents the CPU to run into low performance state
415
+ os.environ['KMP_TPAUSE'] = "0"
416
+ # Provides fine granularity parallelism
417
+ os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
418
+ os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
419
+ os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
420
+
421
+
422
def chunk_list(lst: List[T], chunk_size: int) -> List[List[T]]:
423
"""Yield successive chunk_size chunks from lst."""
424
return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
vllm/worker/cpu_worker.py
+4
-1
@@ -13,7 +13,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
13
from vllm.logger import init_logger
14
from vllm.model_executor import set_random_seed
15
from vllm.sequence import ExecuteModelRequest
16
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
16
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, init_kmp_env
17
from vllm.worker.cpu_model_runner import CPUModelRunner
18
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
19
LoraNotSupportedWorkerBase, WorkerInput)
@@ -150,6 +150,9 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
150
if self.is_driver_worker:
151
assert self.rank == 0, "The driver worker must have rank 0."
152
153
+ # try to initialize intel openmp optimized tunings
154
+ init_kmp_env()
155
+
156
if self.model_config.trust_remote_code:
157
# note: lazy import to avoid importing torch before initializing
158
from vllm.utils import init_cached_hf_modules