A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core] add an option to log every function call to for debugging hang/crash in distributed inference (#4079)
Co-authored-by: Simon Mo <simon.mo@hey.com>
youkaichao committed
Apr 18, 2024 at 16:15 UTC
8a7a3e4436d7284df4c0913f074d77d640a9c6c3
7 files changed
+120
-8
.buildkite/test-pipeline.yaml
+1
-1
@@ -40,7 +40,7 @@ steps:
40
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
41
42
- label: Engine Test
43
- command: pytest -v -s engine tokenization test_sequence.py test_config.py
43
+ command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
44
45
- label: Entrypoints Test
46
commands:
.github/ISSUE_TEMPLATE/400-bug report.yml
+2
@@ -57,6 +57,8 @@ body:
57
If the code is too long (hopefully, it isn't), feel free to put it in a public gist and link it in the issue: https://gist.github.com.
58
59
Please also paste or describe the results you observe instead of the expected results. If you observe an error, please paste the error message including the **full** traceback of the exception. It may be relevant to wrap error messages in ```` ```triple quotes blocks``` ````.
60
+
61
+ If you experienced crashes or hangs, it would be helpful to run vllm with `export VLLM_TRACE_FUNCTION=1` . All the function calls in vllm will be recorded. Inspect these log files, and tell which function crashes or hangs.
62
placeholder: |
63
A clear and concise description of what the bug is.
64
tests/test_logger.py
new
+27
@@ -0,0 +1,27 @@
1
+import os
2
+import sys
3
+import tempfile
4
+
5
+from vllm.logger import enable_trace_function_call
6
+
7
+
8
+def f1(x):
9
+ return f2(x)
10
+
11
+
12
+def f2(x):
13
+ return x
14
+
15
+
16
+def test_trace_function_call():
17
+ fd, path = tempfile.mkstemp()
18
+ cur_dir = os.path.dirname(__file__)
19
+ enable_trace_function_call(path, cur_dir)
20
+ f1(1)
21
+ with open(path, 'r') as f:
22
+ content = f.read()
23
+
24
+ assert "f1" in content
25
+ assert "f2" in content
26
+ sys.settrace(None)
27
+ os.remove(path)
vllm/executor/ray_gpu_executor.py
+9
-3
@@ -10,7 +10,7 @@ from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
11
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13
- make_async)
13
+ get_vllm_instance_id, make_async)
14
15
if ray is not None:
16
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -133,12 +133,18 @@ class RayGPUExecutor(ExecutorBase):
133
for node_id, gpu_ids in node_gpus.items():
134
node_gpus[node_id] = sorted(gpu_ids)
135
136
- # Set CUDA_VISIBLE_DEVICES for the driver and workers.
136
+ VLLM_INSTANCE_ID = get_vllm_instance_id()
137
+
138
+ # Set environment variables for the driver and workers.
139
all_args_to_update_environment_variables = []
140
for (node_id, _) in worker_node_and_gpu_ids:
141
all_args_to_update_environment_variables.append([{
142
"CUDA_VISIBLE_DEVICES":
141
- ",".join(map(str, node_gpus[node_id]))
143
+ ",".join(map(str, node_gpus[node_id])),
144
+ "VLLM_INSTANCE_ID":
145
+ VLLM_INSTANCE_ID,
146
+ "VLLM_TRACE_FUNCTION":
147
+ os.getenv("VLLM_TRACE_FUNCTION", "0"),
148
}])
149
self._run_workers("update_environment_variables",
150
all_args=all_args_to_update_environment_variables)
vllm/logger.py
+52
@@ -1,9 +1,11 @@
1
# Adapted from
2
# https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
3
"""Logging configuration for vLLM."""
4
+import datetime
5
import logging
6
import os
7
import sys
8
+from functools import partial
9
from typing import Optional
10
11
VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
@@ -65,3 +67,53 @@ def init_logger(name: str):
67
logger.addHandler(_default_handler)
68
logger.propagate = False
69
return logger
70
+
71
+
72
+logger = init_logger(__name__)
73
+
74
+
75
+def _trace_calls(log_path, root_dir, frame, event, arg=None):
76
+ if event in ['call', 'return']:
77
+ # Extract the filename, line number, function name, and the code object
78
+ filename = frame.f_code.co_filename
79
+ lineno = frame.f_lineno
80
+ func_name = frame.f_code.co_name
81
+ if not filename.startswith(root_dir):
82
+ # only log the functions in the vllm root_dir
83
+ return
84
+ # Log every function call or return
85
+ try:
86
+ with open(log_path, 'a') as f:
87
+ if event == 'call':
88
+ f.write(f"{datetime.datetime.now()} Call to"
89
+ f" {func_name} in {filename}:{lineno}\n")
90
+ else:
91
+ f.write(f"{datetime.datetime.now()} Return from"
92
+ f" {func_name} in {filename}:{lineno}\n")
93
+ except NameError:
94
+ # modules are deleted during shutdown
95
+ pass
96
+ return partial(_trace_calls, log_path, root_dir)
97
+
98
+
99
+def enable_trace_function_call(log_file_path: str,
100
+ root_dir: Optional[str] = None):
101
+ """
102
+ Enable tracing of every function call in code under `root_dir`.
103
+ This is useful for debugging hangs or crashes.
104
+ `log_file_path` is the path to the log file.
105
+ `root_dir` is the root directory of the code to trace. If None, it is the
106
+ vllm root directory.
107
+
108
+ Note that this call is thread-level, any threads calling this function
109
+ will have the trace enabled. Other threads will not be affected.
110
+ """
111
+ logger.warning(
112
+ "VLLM_TRACE_FUNCTION is enabled. It will record every"
113
+ " function executed by Python. This will slow down the code. It "
114
+ "is suggested to be used for debugging hang or crashes only.")
115
+ logger.info(f"Trace frame log is saved to {log_file_path}")
116
+ if root_dir is None:
117
+ # by default, this is the vllm root directory
118
+ root_dir = os.path.dirname(os.path.dirname(__file__))
119
+ sys.settrace(partial(_trace_calls, log_file_path, root_dir))
vllm/utils.py
+12
-1
@@ -163,6 +163,17 @@ def random_uuid() -> str:
163
return str(uuid.uuid4().hex)
164
165
166
+@lru_cache(maxsize=None)
167
+def get_vllm_instance_id():
168
+ """
169
+ If the environment variable VLLM_INSTANCE_ID is set, return it.
170
+ Otherwise, return a random UUID.
171
+ Instance id represents an instance of the VLLM. All processes in the same
172
+ instance should have the same instance id.
173
+ """
174
+ return os.environ.get("VLLM_INSTANCE_ID", f"vllm-instance-{random_uuid()}")
175
+
176
+
177
@lru_cache(maxsize=None)
178
def in_wsl() -> bool:
179
# Reference: https://github.com/microsoft/WSL/issues/4071
@@ -274,7 +285,7 @@ def get_open_port() -> int:
285
286
def update_environment_variables(envs: Dict[str, str]):
287
for k, v in envs.items():
277
- if k in os.environ:
288
+ if k in os.environ and os.environ[k] != v:
289
logger.warning(f"Overwriting environment variable {k} "
290
f"from '{os.environ[k]}' to '{v}'")
291
os.environ[k] = v
vllm/worker/worker_base.py
+17
-3
@@ -1,12 +1,15 @@
1
+import datetime
2
import importlib
3
import os
4
+import tempfile
5
+import threading
6
from abc import ABC, abstractmethod
7
from typing import Dict, List, Set, Tuple
8
6
-from vllm.logger import init_logger
9
+from vllm.logger import enable_trace_function_call, init_logger
10
from vllm.lora.request import LoRARequest
11
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
-from vllm.utils import update_environment_variables
12
+from vllm.utils import get_vllm_instance_id, update_environment_variables
13
14
logger = init_logger(__name__)
15
@@ -115,9 +118,20 @@ class WorkerWrapperBase:
118
119
def init_worker(self, *args, **kwargs):
120
"""
118
- Actual initialization of the worker class.
121
+ Actual initialization of the worker class, and set up
122
+ function tracing if required.
123
Arguments are passed to the worker class constructor.
124
"""
125
+ if int(os.getenv("VLLM_TRACE_FUNCTION", "0")):
126
+ tmp_dir = tempfile.gettempdir()
127
+ filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
128
+ f"_thread_{threading.get_ident()}_"
129
+ f"at_{datetime.datetime.now()}.log").replace(" ", "_")
130
+ log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(),
131
+ filename)
132
+ os.makedirs(os.path.dirname(log_path), exist_ok=True)
133
+ enable_trace_function_call(log_path)
134
+
135
mod = importlib.import_module(self.worker_module_name)
136
worker_class = getattr(mod, self.worker_class_name)
137
self.worker = worker_class(*args, **kwargs)