@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a43065272f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Misc][Gaudi] Avoid torch.compile and enable lazy collectives (#10897)

Signed-off-by: Konrad Zawora <kzawora@habana.ai>

Konrad Zawora committed Dec 5, 2024 at 17:47 UTC a43065272f73a7468b1a35dd44fb5b0ed80f88c7
1 file changed +14
vllm/plugins/__init__.py
+14
@@ -29,6 +29,20 @@ def load_general_plugins():
29 if current_platform.is_xpu():
30 # see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa
31 os.environ['TORCH_COMPILE_DISABLE'] = 'True'
32 + if current_platform.is_hpu():
33 + # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
34 + # does not support torch.compile
35 + # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
36 + # torch.compile support
37 + is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
38 + if is_lazy:
39 + # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
40 + torch._dynamo.config.disable = True
41 + # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
42 + # requires enabling lazy collectives
43 + # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
44 + os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
45 +
46 global plugins_loaded
47 if plugins_loaded:
48 return