@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 519cc6ca12
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc][XPU] Avoid torch compile for XPU platform (#10747)

Signed-off-by: yan ma <yan.ma@intel.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Yan Ma committed Dec 3, 2024 at 01:53 UTC 519cc6ca12dc89eec35bc2579494e399da33c31a
2 files changed +8 -2
.buildkite/run-xpu-test.sh
+4 -2
@@ -12,5 +12,7 @@ remove_docker_container() { docker rm -f xpu-test || true; }
12 trap remove_docker_container EXIT
13 remove_docker_container
14
15 -# Run the image and launch offline inference
16 -docker run --network host --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path --entrypoint="" xpu-test python3 examples/offline_inference.py
15 +# Run the image and test offline inference/tensor parallel
16 +docker run -it -d --name xpu-test --device /dev/dri -v /dev/dri/by-path:/dev/dri/by-path xpu-test /bin/bash
17 +docker exec xpu-test bash -c "python3 examples/offline_inference.py"
18 +docker exec xpu-test bash -c "python3 examples/offline_inference_cli.py -tp 2"
vllm/plugins/__init__.py
+4
@@ -4,6 +4,7 @@ import os
4 import torch
5
6 import vllm.envs as envs
7 +from vllm.platforms import current_platform
8
9 logger = logging.getLogger(__name__)
10
@@ -25,6 +26,9 @@ def load_general_plugins():
26 os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
27 # see https://github.com/vllm-project/vllm/issues/10619
28 torch._inductor.config.compile_threads = 1
29 + if current_platform.is_xpu():
30 + # see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa
31 + os.environ['TORCH_COMPILE_DISABLE'] = 'True'
32 global plugins_loaded
33 if plugins_loaded:
34 return