@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c222f47992
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core][bugfix] configure env var during import vllm (#12209)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 20, 2025 at 19:35 UTC c222f47992ce0bbcd3ccbce24736e045d8689be8
4 files changed +37 -45
examples/offline_inference/rlhf.py
+1 -6
@@ -19,7 +19,7 @@ from ray.util.placement_group import placement_group
19 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
20 from transformers import AutoModelForCausalLM
21
22 -from vllm import LLM, SamplingParams, configure_as_vllm_process
22 +from vllm import LLM, SamplingParams
23 from vllm.utils import get_ip, get_open_port
24 from vllm.worker.worker import Worker
25
@@ -98,12 +98,7 @@ class MyLLM(LLM):
98 """
99 Start the training process, here we use huggingface transformers
100 as an example to hold a model on GPU 0.
101 -
102 -It is important for all the processes outside of vLLM to call
103 -`configure_as_vllm_process` to set some common environment variables
104 -the same as vLLM workers.
101 """
106 -configure_as_vllm_process()
102
103 train_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
104 train_model.to("cuda:0")
vllm/__init__.py
+13 -36
@@ -1,4 +1,7 @@
1 """vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2 +import os
3 +
4 +import torch
5
6 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
7 from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -17,43 +20,18 @@ from vllm.sampling_params import SamplingParams
20
21 from .version import __version__, __version_tuple__
22
23 +# set some common config/environment variables that should be set
24 +# for all processes created by vllm and all processes
25 +# that interact with vllm workers.
26 +# they are executed whenever `import vllm` is called.
27
21 -def configure_as_vllm_process():
22 - """
23 - set some common config/environment variables that should be set
24 - for all processes created by vllm and all processes
25 - that interact with vllm workers.
26 - """
27 - import os
28 -
29 - import torch
30 -
31 - # see https://github.com/NVIDIA/nccl/issues/1234
32 - os.environ['NCCL_CUMEM_ENABLE'] = '0'
33 -
34 - # see https://github.com/vllm-project/vllm/issues/10480
35 - os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
36 - # see https://github.com/vllm-project/vllm/issues/10619
37 - torch._inductor.config.compile_threads = 1
38 -
39 - from vllm.platforms import current_platform
40 -
41 - if current_platform.is_xpu():
42 - # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
43 - torch._dynamo.config.disable = True
44 - elif current_platform.is_hpu():
45 - # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
46 - # does not support torch.compile
47 - # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
48 - # torch.compile support
49 - is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
50 - if is_lazy:
51 - torch._dynamo.config.disable = True
52 - # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
53 - # requires enabling lazy collectives
54 - # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
55 - os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
28 +# see https://github.com/NVIDIA/nccl/issues/1234
29 +os.environ['NCCL_CUMEM_ENABLE'] = '0'
30
31 +# see https://github.com/vllm-project/vllm/issues/10480
32 +os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
33 +# see https://github.com/vllm-project/vllm/issues/10619
34 +torch._inductor.config.compile_threads = 1
35
36 __all__ = [
37 "__version__",
@@ -80,5 +58,4 @@ __all__ = [
58 "AsyncEngineArgs",
59 "initialize_ray_cluster",
60 "PoolingParams",
83 - "configure_as_vllm_process",
61 ]
vllm/plugins/__init__.py
+23
@@ -1,6 +1,9 @@
1 import logging
2 +import os
3 from typing import Callable, Dict
4
5 +import torch
6 +
7 import vllm.envs as envs
8
9 logger = logging.getLogger(__name__)
@@ -51,6 +54,26 @@ def load_general_plugins():
54 if plugins_loaded:
55 return
56 plugins_loaded = True
57 +
58 + # some platform-specific configurations
59 + from vllm.platforms import current_platform
60 +
61 + if current_platform.is_xpu():
62 + # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
63 + torch._dynamo.config.disable = True
64 + elif current_platform.is_hpu():
65 + # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
66 + # does not support torch.compile
67 + # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
68 + # torch.compile support
69 + is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
70 + if is_lazy:
71 + torch._dynamo.config.disable = True
72 + # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
73 + # requires enabling lazy collectives
74 + # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
75 + os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
76 +
77 plugins = load_plugins_by_group(group='vllm.general_plugins')
78 # general plugins, we only need to execute the loaded functions
79 for func in plugins.values():
vllm/worker/worker_base.py
-3
@@ -535,9 +535,6 @@ class WorkerWrapperBase:
535 kwargs = all_kwargs[self.rpc_rank]
536 enable_trace_function_call_for_thread(self.vllm_config)
537
538 - from vllm import configure_as_vllm_process
539 - configure_as_vllm_process()
540 -
538 from vllm.plugins import load_general_plugins
539 load_general_plugins()
540