@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b62745b1d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[core][executor] simplify instance id (#10976)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 7, 2024 at 09:33 UTC 1b62745b1d00153c5e99879edaf0c2d7ceb4e2c6
10 files changed +22 -55
vllm/config.py
+6 -1
@@ -27,7 +27,8 @@ from vllm.transformers_utils.config import (
27 get_hf_text_config, get_pooling_config,
28 get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
29 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
30 - print_warning_once, resolve_obj_by_qualname)
30 + print_warning_once, random_uuid,
31 + resolve_obj_by_qualname)
32
33 if TYPE_CHECKING:
34 from ray.util.placement_group import PlacementGroup
@@ -2408,6 +2409,7 @@ class VllmConfig:
2409 init=True) # type: ignore
2410 kv_transfer_config: KVTransferConfig = field(default=None,
2411 init=True) # type: ignore
2412 + instance_id: str = ""
2413
2414 @staticmethod
2415 def get_graph_batch_size(batch_size: int) -> int:
@@ -2573,6 +2575,9 @@ class VllmConfig:
2575
2576 current_platform.check_and_update_config(self)
2577
2578 + if not self.instance_id:
2579 + self.instance_id = random_uuid()[:5]
2580 +
2581 def __str__(self):
2582 return ("model=%r, speculative_config=%r, tokenizer=%r, "
2583 "skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
vllm/envs.py
-6
@@ -8,7 +8,6 @@ if TYPE_CHECKING:
8 VLLM_RPC_BASE_PATH: str = tempfile.gettempdir()
9 VLLM_USE_MODELSCOPE: bool = False
10 VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
11 - VLLM_INSTANCE_ID: Optional[str] = None
11 VLLM_NCCL_SO_PATH: Optional[str] = None
12 LD_LIBRARY_PATH: Optional[str] = None
13 VLLM_USE_TRITON_FLASH_ATTN: bool = False
@@ -175,11 +174,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
174 "VLLM_USE_MODELSCOPE":
175 lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true",
176
178 - # Instance id represents an instance of the VLLM. All processes in the same
179 - # instance should have the same instance id.
180 - "VLLM_INSTANCE_ID":
181 - lambda: os.environ.get("VLLM_INSTANCE_ID", None),
182 -
177 # Interval in seconds to log a warning message when the ring buffer is full
178 "VLLM_RINGBUFFER_WARNING_INTERVAL":
179 lambda: int(os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")),
vllm/executor/cpu_executor.py
+1 -5
@@ -10,8 +10,7 @@ from vllm.lora.request import LoRARequest
10 from vllm.model_executor.layers.sampler import SamplerOutput
11 from vllm.prompt_adapter.request import PromptAdapterRequest
12 from vllm.sequence import ExecuteModelRequest
13 -from vllm.utils import (get_distributed_init_method, get_open_port,
14 - get_vllm_instance_id, make_async)
13 +from vllm.utils import get_distributed_init_method, get_open_port, make_async
14 from vllm.worker.worker_base import WorkerWrapperBase
15
16 logger = init_logger(__name__)
@@ -31,9 +30,6 @@ class CPUExecutor(ExecutorBase):
30 # Environment variables for CPU executor
31 #
32
34 - # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
35 - os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
36 -
33 # Disable torch async compiling which won't work with daemonic processes
34 os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
35
vllm/executor/multiproc_gpu_executor.py
+1 -4
@@ -16,7 +16,7 @@ from vllm.sequence import ExecuteModelRequest
16 from vllm.triton_utils.importing import HAS_TRITON
17 from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18 cuda_is_initialized, get_distributed_init_method,
19 - get_open_port, get_vllm_instance_id, make_async,
19 + get_open_port, make_async,
20 update_environment_variables)
21
22 if HAS_TRITON:
@@ -37,9 +37,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
37 world_size = self.parallel_config.world_size
38 tensor_parallel_size = self.parallel_config.tensor_parallel_size
39
40 - # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
41 - os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
42 -
40 # Disable torch async compiling which won't work with daemonic processes
41 os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
42
vllm/executor/ray_gpu_executor.py
+1 -6
@@ -15,8 +15,7 @@ from vllm.logger import init_logger
15 from vllm.model_executor.layers.sampler import SamplerOutput
16 from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18 - get_ip, get_open_port, get_vllm_instance_id,
19 - make_async)
18 + get_ip, get_open_port, make_async)
19
20 if ray is not None:
21 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -220,14 +219,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
219 " environment variable, make sure it is unique for"
220 " each node.")
221
223 - VLLM_INSTANCE_ID = get_vllm_instance_id()
224 -
222 # Set environment variables for the driver and workers.
223 all_args_to_update_environment_variables = [({
224 "CUDA_VISIBLE_DEVICES":
225 ",".join(map(str, node_gpus[node_id])),
229 - "VLLM_INSTANCE_ID":
230 - VLLM_INSTANCE_ID,
226 "VLLM_TRACE_FUNCTION":
227 str(envs.VLLM_TRACE_FUNCTION),
228 **({
vllm/executor/ray_hpu_executor.py
+1 -6
@@ -15,8 +15,7 @@ from vllm.logger import init_logger
15 from vllm.model_executor.layers.sampler import SamplerOutput
16 from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18 - get_ip, get_open_port, get_vllm_instance_id,
19 - make_async)
18 + get_ip, get_open_port, make_async)
19
20 if ray is not None:
21 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -196,12 +195,8 @@ class RayHPUExecutor(DistributedGPUExecutor):
195 "environment variable, make sure it is unique for"
196 " each node.")
197
199 - VLLM_INSTANCE_ID = get_vllm_instance_id()
200 -
198 # Set environment variables for the driver and workers.
199 all_args_to_update_environment_variables = [({
203 - "VLLM_INSTANCE_ID":
204 - VLLM_INSTANCE_ID,
200 "VLLM_TRACE_FUNCTION":
201 str(envs.VLLM_TRACE_FUNCTION),
202 }, ) for (node_id, _) in worker_node_and_gpu_ids]
vllm/executor/ray_tpu_executor.py
+1 -5
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13 from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.sequence import ExecuteModelRequest
15 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16 - get_vllm_instance_id, make_async)
16 + make_async)
17
18 if ray is not None:
19 from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -144,12 +144,8 @@ class RayTPUExecutor(TPUExecutor):
144 for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
145 node_workers[node_id].append(i)
146
147 - VLLM_INSTANCE_ID = get_vllm_instance_id()
148 -
147 # Set environment variables for the driver and workers.
148 all_args_to_update_environment_variables = [({
151 - "VLLM_INSTANCE_ID":
152 - VLLM_INSTANCE_ID,
149 "VLLM_TRACE_FUNCTION":
150 str(envs.VLLM_TRACE_FUNCTION),
151 }, ) for _ in worker_node_and_gpu_ids]
vllm/executor/ray_xpu_executor.py
+1 -5
@@ -5,7 +5,7 @@ import vllm.envs as envs
5 from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
6 from vllm.executor.xpu_executor import XPUExecutor
7 from vllm.logger import init_logger
8 -from vllm.utils import get_vllm_instance_id, make_async
8 +from vllm.utils import make_async
9
10 logger = init_logger(__name__)
11
@@ -17,12 +17,8 @@ class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
17 worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
18 use_dummy_driver=True)
19
20 - VLLM_INSTANCE_ID = get_vllm_instance_id()
21 -
20 # Set environment variables for the driver and workers.
21 all_args_to_update_environment_variables = [({
24 - "VLLM_INSTANCE_ID":
25 - VLLM_INSTANCE_ID,
22 "VLLM_TRACE_FUNCTION":
23 str(envs.VLLM_TRACE_FUNCTION),
24 }, ) for (_, _) in worker_node_and_gpu_ids]
vllm/utils.py
+9 -16
@@ -24,9 +24,9 @@ from collections import UserDict, defaultdict
24 from collections.abc import Iterable, Mapping
25 from functools import lru_cache, partial, wraps
26 from platform import uname
27 -from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
28 - Hashable, List, Literal, Optional, OrderedDict, Set, Tuple,
29 - Type, TypeVar, Union, overload)
27 +from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
28 + Dict, Generic, Hashable, List, Literal, Optional,
29 + OrderedDict, Set, Tuple, Type, TypeVar, Union, overload)
30 from uuid import uuid4
31
32 import numpy as np
@@ -43,6 +43,9 @@ import vllm.envs as envs
43 from vllm.logger import enable_trace_function_call, init_logger
44 from vllm.platforms import current_platform
45
46 +if TYPE_CHECKING:
47 + from vllm.config import VllmConfig
48 +
49 logger = init_logger(__name__)
50
51 # Exception strings for non-implemented encoder/decoder scenarios
@@ -335,17 +338,6 @@ def random_uuid() -> str:
338 return str(uuid.uuid4().hex)
339
340
338 -@lru_cache(maxsize=None)
339 -def get_vllm_instance_id() -> str:
340 - """
341 - If the environment variable VLLM_INSTANCE_ID is set, return it.
342 - Otherwise, return a random UUID.
343 - Instance id represents an instance of the VLLM. All processes in the same
344 - instance should have the same instance id.
345 - """
346 - return envs.VLLM_INSTANCE_ID or f"vllm-instance-{random_uuid()}"
347 -
348 -
341 @lru_cache(maxsize=None)
342 def in_wsl() -> bool:
343 # Reference: https://github.com/microsoft/WSL/issues/4071
@@ -997,7 +989,7 @@ def find_nccl_library() -> str:
989 return so_file
990
991
1000 -def enable_trace_function_call_for_thread() -> None:
992 +def enable_trace_function_call_for_thread(vllm_config: "VllmConfig") -> None:
993 """Set up function tracing for the current thread,
994 if enabled via the VLLM_TRACE_FUNCTION environment variable
995 """
@@ -1009,7 +1001,8 @@ def enable_trace_function_call_for_thread() -> None:
1001 filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
1002 f"_thread_{threading.get_ident()}_"
1003 f"at_{datetime.datetime.now()}.log").replace(" ", "_")
1012 - log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(),
1004 + log_path = os.path.join(tmp_dir, "vllm",
1005 + f"vllm-instance-{vllm_config.instance_id}",
1006 filename)
1007 os.makedirs(os.path.dirname(log_path), exist_ok=True)
1008 enable_trace_function_call(log_path)
vllm/worker/worker_base.py
+1 -1
@@ -439,7 +439,7 @@ class WorkerWrapperBase:
439 Here we inject some common logic before initializing the worker.
440 Arguments are passed to the worker class constructor.
441 """
442 - enable_trace_function_call_for_thread()
442 + enable_trace_function_call_for_thread(self.vllm_config)
443
444 # see https://github.com/NVIDIA/nccl/issues/1234
445 os.environ['NCCL_CUMEM_ENABLE'] = '0'