@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / efffb63f58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Move function tracing setup to util function (#4352)

Nick Hill committed Apr 25, 2024 at 16:45 UTC efffb63f584c1ce4fdcf4e7b7fd0bfc8b33a733a
2 files changed +24 -15
vllm/utils.py
+20 -1
@@ -1,10 +1,13 @@
1 import asyncio
2 +import datetime
3 import enum
4 import gc
5 import glob
6 import os
7 import socket
8 import subprocess
9 +import tempfile
10 +import threading
11 import uuid
12 import warnings
13 from collections import defaultdict
@@ -18,7 +21,7 @@ import psutil
21 import torch
22 from packaging.version import Version, parse
23
21 -from vllm.logger import init_logger
24 +from vllm.logger import enable_trace_function_call, init_logger
25
26 T = TypeVar("T")
27 logger = init_logger(__name__)
@@ -607,3 +610,19 @@ def find_nccl_library():
610 raise ValueError("NCCL only supports CUDA and ROCm backends.")
611 logger.info(f"Found nccl from library {so_file}")
612 return so_file
613 +
614 +
615 +def enable_trace_function_call_for_thread() -> None:
616 + """Set up function tracing for the current thread,
617 + if enabled via the VLLM_TRACE_FUNCTION environment variable
618 + """
619 +
620 + if int(os.getenv("VLLM_TRACE_FUNCTION", "0")):
621 + tmp_dir = tempfile.gettempdir()
622 + filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
623 + f"_thread_{threading.get_ident()}_"
624 + f"at_{datetime.datetime.now()}.log").replace(" ", "_")
625 + log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(),
626 + filename)
627 + os.makedirs(os.path.dirname(log_path), exist_ok=True)
628 + enable_trace_function_call(log_path)
vllm/worker/worker_base.py
+4 -14
@@ -1,15 +1,13 @@
1 -import datetime
1 import importlib
2 import os
4 -import tempfile
5 -import threading
3 from abc import ABC, abstractmethod
4 from typing import Dict, List, Set, Tuple
5
9 -from vllm.logger import enable_trace_function_call, init_logger
6 +from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
12 -from vllm.utils import get_vllm_instance_id, update_environment_variables
9 +from vllm.utils import (enable_trace_function_call_for_thread,
10 + update_environment_variables)
11
12 logger = init_logger(__name__)
13
@@ -128,15 +126,7 @@ class WorkerWrapperBase:
126 function tracing if required.
127 Arguments are passed to the worker class constructor.
128 """
131 - if int(os.getenv("VLLM_TRACE_FUNCTION", "0")):
132 - tmp_dir = tempfile.gettempdir()
133 - filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
134 - f"_thread_{threading.get_ident()}_"
135 - f"at_{datetime.datetime.now()}.log").replace(" ", "_")
136 - log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(),
137 - filename)
138 - os.makedirs(os.path.dirname(log_path), exist_ok=True)
139 - enable_trace_function_call(log_path)
129 + enable_trace_function_call_for_thread()
130
131 mod = importlib.import_module(self.worker_module_name)
132 worker_class = getattr(mod, self.worker_class_name)