A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core][bugfix] configure env var during import vllm (#12209)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 20, 2025 at 19:35 UTC
c222f47992ce0bbcd3ccbce24736e045d8689be8
4 files changed
+37
-45
examples/offline_inference/rlhf.py
+1
-6
@@ -19,7 +19,7 @@ from ray.util.placement_group import placement_group
19
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
20
from transformers import AutoModelForCausalLM
21
22
-from vllm import LLM, SamplingParams, configure_as_vllm_process
22
+from vllm import LLM, SamplingParams
23
from vllm.utils import get_ip, get_open_port
24
from vllm.worker.worker import Worker
25
@@ -98,12 +98,7 @@ class MyLLM(LLM):
98
"""
99
Start the training process, here we use huggingface transformers
100
as an example to hold a model on GPU 0.
101
-
102
-It is important for all the processes outside of vLLM to call
103
-`configure_as_vllm_process` to set some common environment variables
104
-the same as vLLM workers.
101
"""
106
-configure_as_vllm_process()
102
103
train_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
104
train_model.to("cuda:0")
vllm/__init__.py
+13
-36
@@ -1,4 +1,7 @@
1
"""vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2
+import os
3
+
4
+import torch
5
6
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
7
from vllm.engine.async_llm_engine import AsyncLLMEngine
@@ -17,43 +20,18 @@ from vllm.sampling_params import SamplingParams
20
21
from .version import __version__, __version_tuple__
22
23
+# set some common config/environment variables that should be set
24
+# for all processes created by vllm and all processes
25
+# that interact with vllm workers.
26
+# they are executed whenever `import vllm` is called.
27
21
-def configure_as_vllm_process():
22
- """
23
- set some common config/environment variables that should be set
24
- for all processes created by vllm and all processes
25
- that interact with vllm workers.
26
- """
27
- import os
28
-
29
- import torch
30
-
31
- # see https://github.com/NVIDIA/nccl/issues/1234
32
- os.environ['NCCL_CUMEM_ENABLE'] = '0'
33
-
34
- # see https://github.com/vllm-project/vllm/issues/10480
35
- os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
36
- # see https://github.com/vllm-project/vllm/issues/10619
37
- torch._inductor.config.compile_threads = 1
38
-
39
- from vllm.platforms import current_platform
40
-
41
- if current_platform.is_xpu():
42
- # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
43
- torch._dynamo.config.disable = True
44
- elif current_platform.is_hpu():
45
- # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
46
- # does not support torch.compile
47
- # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
48
- # torch.compile support
49
- is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
50
- if is_lazy:
51
- torch._dynamo.config.disable = True
52
- # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
53
- # requires enabling lazy collectives
54
- # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
55
- os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
28
+# see https://github.com/NVIDIA/nccl/issues/1234
29
+os.environ['NCCL_CUMEM_ENABLE'] = '0'
30
31
+# see https://github.com/vllm-project/vllm/issues/10480
32
+os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1'
33
+# see https://github.com/vllm-project/vllm/issues/10619
34
+torch._inductor.config.compile_threads = 1
35
36
__all__ = [
37
"__version__",
@@ -80,5 +58,4 @@ __all__ = [
58
"AsyncEngineArgs",
59
"initialize_ray_cluster",
60
"PoolingParams",
83
- "configure_as_vllm_process",
61
]
vllm/plugins/__init__.py
+23
@@ -1,6 +1,9 @@
1
import logging
2
+import os
3
from typing import Callable, Dict
4
5
+import torch
6
+
7
import vllm.envs as envs
8
9
logger = logging.getLogger(__name__)
@@ -51,6 +54,26 @@ def load_general_plugins():
54
if plugins_loaded:
55
return
56
plugins_loaded = True
57
+
58
+ # some platform-specific configurations
59
+ from vllm.platforms import current_platform
60
+
61
+ if current_platform.is_xpu():
62
+ # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
63
+ torch._dynamo.config.disable = True
64
+ elif current_platform.is_hpu():
65
+ # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
66
+ # does not support torch.compile
67
+ # Eager backend (PT_HPU_LAZY_MODE = 0) must be selected for
68
+ # torch.compile support
69
+ is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
70
+ if is_lazy:
71
+ torch._dynamo.config.disable = True
72
+ # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
73
+ # requires enabling lazy collectives
74
+ # see https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_HPU_Graphs.html # noqa: E501
75
+ os.environ['PT_HPU_ENABLE_LAZY_COLLECTIVES'] = 'true'
76
+
77
plugins = load_plugins_by_group(group='vllm.general_plugins')
78
# general plugins, we only need to execute the loaded functions
79
for func in plugins.values():
vllm/worker/worker_base.py
-3
@@ -535,9 +535,6 @@ class WorkerWrapperBase:
535
kwargs = all_kwargs[self.rpc_rank]
536
enable_trace_function_call_for_thread(self.vllm_config)
537
538
- from vllm import configure_as_vllm_process
539
- configure_as_vllm_process()
540
-
538
from vllm.plugins import load_general_plugins
539
load_general_plugins()
540