@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 81d7a50f24
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Hardware][Intel CPU] Adding intel openmp tunings in Docker file (#6008)

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>

Yuan committed Jul 5, 2024 at 06:22 UTC 81d7a50f2402ef5b622ac8d3a081994a1a4641b0
4 files changed +37 -5
.buildkite/run-cpu-test.sh
+4 -2
@@ -12,8 +12,10 @@ trap remove_docker_container EXIT
12 remove_docker_container
13
14 # Run the image
15 -docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
16 -docker run -itd -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
15 +docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
16 + --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test cpu-test
17 +docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus=48-95 \
18 + --cpuset-mems=1 --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --name cpu-test-avx2 cpu-test-avx2
19
20 # offline inference
21 docker exec cpu-test bash -c "python3 examples/offline_inference.py"
Dockerfile.cpu
+8 -2
@@ -6,7 +6,13 @@ RUN apt-get update -y \
6 && apt-get install -y git wget vim numactl gcc-12 g++-12 python3 python3-pip libtcmalloc-minimal4 \
7 && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12
8
9 -RUN echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:$LD_PRELOAD' >> ~/.bashrc
9 +# https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/performance_tuning/tuning_guide.html
10 +# intel-openmp provides additional performance improvement vs. openmp
11 +# tcmalloc provides better memory allocation efficiency, e.g, holding memory in caches to speed up access of commonly-used objects.
12 +RUN pip install intel-openmp
13 +
14 +ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/lib/libiomp5.so:$LD_PRELOAD"
15 +
16
17 RUN pip install https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/cpu/intel_extension_for_pytorch-2.3.100%2Bgit0eb3473-cp310-cp310-linux_x86_64.whl
18
@@ -31,4 +37,4 @@ WORKDIR /workspace/
37
38 RUN ln -s /workspace/vllm/tests && ln -s /workspace/vllm/examples && ln -s /workspace/vllm/benchmarks
39
34 -CMD ["/bin/bash"]
40 +ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
vllm/utils.py
+21
@@ -398,6 +398,27 @@ def update_environment_variables(envs: Dict[str, str]):
398 os.environ[k] = v
399
400
401 +def init_kmp_env():
402 + if not is_cpu():
403 + return
404 +
405 + ld_prealod_str = os.getenv("LD_PRELOAD", "")
406 + if "libiomp5.so" not in ld_prealod_str:
407 + return
408 +
409 + # The time(milliseconds) that a thread should wait after completing the
410 + # execution of a parallel region, before sleeping.
411 + os.environ['KMP_BLOCKTIME'] = "1"
412 + # dump settings on start up
413 + os.environ['KMP_SETTINGS'] = "1"
414 + # Prevents the CPU to run into low performance state
415 + os.environ['KMP_TPAUSE'] = "0"
416 + # Provides fine granularity parallelism
417 + os.environ['KMP_FORKJOIN_BARRIER_PATTERN'] = "dist,dist"
418 + os.environ['KMP_PLAIN_BARRIER_PATTERN'] = "dist,dist"
419 + os.environ['KMP_REDUCTION_BARRIER_PATTERN'] = "dist,dist"
420 +
421 +
422 def chunk_list(lst: List[T], chunk_size: int) -> List[List[T]]:
423 """Yield successive chunk_size chunks from lst."""
424 return [lst[i:i + chunk_size] for i in range(0, len(lst), chunk_size)]
vllm/worker/cpu_worker.py
+4 -1
@@ -13,7 +13,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
13 from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
15 from vllm.sequence import ExecuteModelRequest
16 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
16 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, init_kmp_env
17 from vllm.worker.cpu_model_runner import CPUModelRunner
18 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
19 LoraNotSupportedWorkerBase, WorkerInput)
@@ -150,6 +150,9 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
150 if self.is_driver_worker:
151 assert self.rank == 0, "The driver worker must have rank 0."
152
153 + # try to initialize intel openmp optimized tunings
154 + init_kmp_env()
155 +
156 if self.model_config.trust_remote_code:
157 # note: lazy import to avoid importing torch before initializing
158 from vllm.utils import init_cached_hf_modules