@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8a7a3e4436
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core] add an option to log every function call to for debugging hang/crash in distributed inference (#4079)

Co-authored-by: Simon Mo <simon.mo@hey.com>

youkaichao committed Apr 18, 2024 at 16:15 UTC 8a7a3e4436d7284df4c0913f074d77d640a9c6c3
7 files changed +120 -8
.buildkite/test-pipeline.yaml
+1 -1
@@ -40,7 +40,7 @@ steps:
40 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf pytest -v -s test_chunked_prefill_distributed.py
41
42 - label: Engine Test
43 - command: pytest -v -s engine tokenization test_sequence.py test_config.py
43 + command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
44
45 - label: Entrypoints Test
46 commands:
.github/ISSUE_TEMPLATE/400-bug report.yml
+2
@@ -57,6 +57,8 @@ body:
57 If the code is too long (hopefully, it isn't), feel free to put it in a public gist and link it in the issue: https://gist.github.com.
58
59 Please also paste or describe the results you observe instead of the expected results. If you observe an error, please paste the error message including the **full** traceback of the exception. It may be relevant to wrap error messages in ```` ```triple quotes blocks``` ````.
60 +
61 + If you experienced crashes or hangs, it would be helpful to run vllm with `export VLLM_TRACE_FUNCTION=1` . All the function calls in vllm will be recorded. Inspect these log files, and tell which function crashes or hangs.
62 placeholder: |
63 A clear and concise description of what the bug is.
64
tests/test_logger.py new
+27
@@ -0,0 +1,27 @@
1 +import os
2 +import sys
3 +import tempfile
4 +
5 +from vllm.logger import enable_trace_function_call
6 +
7 +
8 +def f1(x):
9 + return f2(x)
10 +
11 +
12 +def f2(x):
13 + return x
14 +
15 +
16 +def test_trace_function_call():
17 + fd, path = tempfile.mkstemp()
18 + cur_dir = os.path.dirname(__file__)
19 + enable_trace_function_call(path, cur_dir)
20 + f1(1)
21 + with open(path, 'r') as f:
22 + content = f.read()
23 +
24 + assert "f1" in content
25 + assert "f2" in content
26 + sys.settrace(None)
27 + os.remove(path)
vllm/executor/ray_gpu_executor.py
+9 -3
@@ -10,7 +10,7 @@ from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
11 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
13 - make_async)
13 + get_vllm_instance_id, make_async)
14
15 if ray is not None:
16 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -133,12 +133,18 @@ class RayGPUExecutor(ExecutorBase):
133 for node_id, gpu_ids in node_gpus.items():
134 node_gpus[node_id] = sorted(gpu_ids)
135
136 - # Set CUDA_VISIBLE_DEVICES for the driver and workers.
136 + VLLM_INSTANCE_ID = get_vllm_instance_id()
137 +
138 + # Set environment variables for the driver and workers.
139 all_args_to_update_environment_variables = []
140 for (node_id, _) in worker_node_and_gpu_ids:
141 all_args_to_update_environment_variables.append([{
142 "CUDA_VISIBLE_DEVICES":
141 - ",".join(map(str, node_gpus[node_id]))
143 + ",".join(map(str, node_gpus[node_id])),
144 + "VLLM_INSTANCE_ID":
145 + VLLM_INSTANCE_ID,
146 + "VLLM_TRACE_FUNCTION":
147 + os.getenv("VLLM_TRACE_FUNCTION", "0"),
148 }])
149 self._run_workers("update_environment_variables",
150 all_args=all_args_to_update_environment_variables)
vllm/logger.py
+52
@@ -1,9 +1,11 @@
1 # Adapted from
2 # https://github.com/skypilot-org/skypilot/blob/86dc0f6283a335e4aa37b3c10716f90999f48ab6/sky/sky_logging.py
3 """Logging configuration for vLLM."""
4 +import datetime
5 import logging
6 import os
7 import sys
8 +from functools import partial
9 from typing import Optional
10
11 VLLM_CONFIGURE_LOGGING = int(os.getenv("VLLM_CONFIGURE_LOGGING", "1"))
@@ -65,3 +67,53 @@ def init_logger(name: str):
67 logger.addHandler(_default_handler)
68 logger.propagate = False
69 return logger
70 +
71 +
72 +logger = init_logger(__name__)
73 +
74 +
75 +def _trace_calls(log_path, root_dir, frame, event, arg=None):
76 + if event in ['call', 'return']:
77 + # Extract the filename, line number, function name, and the code object
78 + filename = frame.f_code.co_filename
79 + lineno = frame.f_lineno
80 + func_name = frame.f_code.co_name
81 + if not filename.startswith(root_dir):
82 + # only log the functions in the vllm root_dir
83 + return
84 + # Log every function call or return
85 + try:
86 + with open(log_path, 'a') as f:
87 + if event == 'call':
88 + f.write(f"{datetime.datetime.now()} Call to"
89 + f" {func_name} in {filename}:{lineno}\n")
90 + else:
91 + f.write(f"{datetime.datetime.now()} Return from"
92 + f" {func_name} in {filename}:{lineno}\n")
93 + except NameError:
94 + # modules are deleted during shutdown
95 + pass
96 + return partial(_trace_calls, log_path, root_dir)
97 +
98 +
99 +def enable_trace_function_call(log_file_path: str,
100 + root_dir: Optional[str] = None):
101 + """
102 + Enable tracing of every function call in code under `root_dir`.
103 + This is useful for debugging hangs or crashes.
104 + `log_file_path` is the path to the log file.
105 + `root_dir` is the root directory of the code to trace. If None, it is the
106 + vllm root directory.
107 +
108 + Note that this call is thread-level, any threads calling this function
109 + will have the trace enabled. Other threads will not be affected.
110 + """
111 + logger.warning(
112 + "VLLM_TRACE_FUNCTION is enabled. It will record every"
113 + " function executed by Python. This will slow down the code. It "
114 + "is suggested to be used for debugging hang or crashes only.")
115 + logger.info(f"Trace frame log is saved to {log_file_path}")
116 + if root_dir is None:
117 + # by default, this is the vllm root directory
118 + root_dir = os.path.dirname(os.path.dirname(__file__))
119 + sys.settrace(partial(_trace_calls, log_file_path, root_dir))
vllm/utils.py
+12 -1
@@ -163,6 +163,17 @@ def random_uuid() -> str:
163 return str(uuid.uuid4().hex)
164
165
166 +@lru_cache(maxsize=None)
167 +def get_vllm_instance_id():
168 + """
169 + If the environment variable VLLM_INSTANCE_ID is set, return it.
170 + Otherwise, return a random UUID.
171 + Instance id represents an instance of the VLLM. All processes in the same
172 + instance should have the same instance id.
173 + """
174 + return os.environ.get("VLLM_INSTANCE_ID", f"vllm-instance-{random_uuid()}")
175 +
176 +
177 @lru_cache(maxsize=None)
178 def in_wsl() -> bool:
179 # Reference: https://github.com/microsoft/WSL/issues/4071
@@ -274,7 +285,7 @@ def get_open_port() -> int:
285
286 def update_environment_variables(envs: Dict[str, str]):
287 for k, v in envs.items():
277 - if k in os.environ:
288 + if k in os.environ and os.environ[k] != v:
289 logger.warning(f"Overwriting environment variable {k} "
290 f"from '{os.environ[k]}' to '{v}'")
291 os.environ[k] = v
vllm/worker/worker_base.py
+17 -3
@@ -1,12 +1,15 @@
1 +import datetime
2 import importlib
3 import os
4 +import tempfile
5 +import threading
6 from abc import ABC, abstractmethod
7 from typing import Dict, List, Set, Tuple
8
6 -from vllm.logger import init_logger
9 +from vllm.logger import enable_trace_function_call, init_logger
10 from vllm.lora.request import LoRARequest
11 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9 -from vllm.utils import update_environment_variables
12 +from vllm.utils import get_vllm_instance_id, update_environment_variables
13
14 logger = init_logger(__name__)
15
@@ -115,9 +118,20 @@ class WorkerWrapperBase:
118
119 def init_worker(self, *args, **kwargs):
120 """
118 - Actual initialization of the worker class.
121 + Actual initialization of the worker class, and set up
122 + function tracing if required.
123 Arguments are passed to the worker class constructor.
124 """
125 + if int(os.getenv("VLLM_TRACE_FUNCTION", "0")):
126 + tmp_dir = tempfile.gettempdir()
127 + filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
128 + f"_thread_{threading.get_ident()}_"
129 + f"at_{datetime.datetime.now()}.log").replace(" ", "_")
130 + log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(),
131 + filename)
132 + os.makedirs(os.path.dirname(log_path), exist_ok=True)
133 + enable_trace_function_call(log_path)
134 +
135 mod = importlib.import_module(self.worker_module_name)
136 worker_class = getattr(mod, self.worker_class_name)
137 self.worker = worker_class(*args, **kwargs)