A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[core][executor] simplify instance id (#10976)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 7, 2024 at 09:33 UTC
1b62745b1d00153c5e99879edaf0c2d7ceb4e2c6
10 files changed
+22
-55
vllm/config.py
+6
-1
@@ -27,7 +27,8 @@ from vllm.transformers_utils.config import (
27
get_hf_text_config, get_pooling_config,
28
get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
29
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
30
- print_warning_once, resolve_obj_by_qualname)
30
+ print_warning_once, random_uuid,
31
+ resolve_obj_by_qualname)
32
33
if TYPE_CHECKING:
34
from ray.util.placement_group import PlacementGroup
@@ -2408,6 +2409,7 @@ class VllmConfig:
2409
init=True) # type: ignore
2410
kv_transfer_config: KVTransferConfig = field(default=None,
2411
init=True) # type: ignore
2412
+ instance_id: str = ""
2413
2414
@staticmethod
2415
def get_graph_batch_size(batch_size: int) -> int:
@@ -2573,6 +2575,9 @@ class VllmConfig:
2575
2576
current_platform.check_and_update_config(self)
2577
2578
+ if not self.instance_id:
2579
+ self.instance_id = random_uuid()[:5]
2580
+
2581
def __str__(self):
2582
return ("model=%r, speculative_config=%r, tokenizer=%r, "
2583
"skip_tokenizer_init=%s, tokenizer_mode=%s, revision=%s, "
vllm/envs.py
-6
@@ -8,7 +8,6 @@ if TYPE_CHECKING:
8
VLLM_RPC_BASE_PATH: str = tempfile.gettempdir()
9
VLLM_USE_MODELSCOPE: bool = False
10
VLLM_RINGBUFFER_WARNING_INTERVAL: int = 60
11
- VLLM_INSTANCE_ID: Optional[str] = None
11
VLLM_NCCL_SO_PATH: Optional[str] = None
12
LD_LIBRARY_PATH: Optional[str] = None
13
VLLM_USE_TRITON_FLASH_ATTN: bool = False
@@ -175,11 +174,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
174
"VLLM_USE_MODELSCOPE":
175
lambda: os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true",
176
178
- # Instance id represents an instance of the VLLM. All processes in the same
179
- # instance should have the same instance id.
180
- "VLLM_INSTANCE_ID":
181
- lambda: os.environ.get("VLLM_INSTANCE_ID", None),
182
-
177
# Interval in seconds to log a warning message when the ring buffer is full
178
"VLLM_RINGBUFFER_WARNING_INTERVAL":
179
lambda: int(os.environ.get("VLLM_RINGBUFFER_WARNING_INTERVAL", "60")),
vllm/executor/cpu_executor.py
+1
-5
@@ -10,8 +10,7 @@ from vllm.lora.request import LoRARequest
10
from vllm.model_executor.layers.sampler import SamplerOutput
11
from vllm.prompt_adapter.request import PromptAdapterRequest
12
from vllm.sequence import ExecuteModelRequest
13
-from vllm.utils import (get_distributed_init_method, get_open_port,
14
- get_vllm_instance_id, make_async)
13
+from vllm.utils import get_distributed_init_method, get_open_port, make_async
14
from vllm.worker.worker_base import WorkerWrapperBase
15
16
logger = init_logger(__name__)
@@ -31,9 +30,6 @@ class CPUExecutor(ExecutorBase):
30
# Environment variables for CPU executor
31
#
32
34
- # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
35
- os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
36
-
33
# Disable torch async compiling which won't work with daemonic processes
34
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
35
vllm/executor/multiproc_gpu_executor.py
+1
-4
@@ -16,7 +16,7 @@ from vllm.sequence import ExecuteModelRequest
16
from vllm.triton_utils.importing import HAS_TRITON
17
from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18
cuda_is_initialized, get_distributed_init_method,
19
- get_open_port, get_vllm_instance_id, make_async,
19
+ get_open_port, make_async,
20
update_environment_variables)
21
22
if HAS_TRITON:
@@ -37,9 +37,6 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
37
world_size = self.parallel_config.world_size
38
tensor_parallel_size = self.parallel_config.tensor_parallel_size
39
40
- # Ensure that VLLM_INSTANCE_ID is set, to be inherited by workers
41
- os.environ["VLLM_INSTANCE_ID"] = get_vllm_instance_id()
42
-
40
# Disable torch async compiling which won't work with daemonic processes
41
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
42
vllm/executor/ray_gpu_executor.py
+1
-6
@@ -15,8 +15,7 @@ from vllm.logger import init_logger
15
from vllm.model_executor.layers.sampler import SamplerOutput
16
from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18
- get_ip, get_open_port, get_vllm_instance_id,
19
- make_async)
18
+ get_ip, get_open_port, make_async)
19
20
if ray is not None:
21
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -220,14 +219,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
219
" environment variable, make sure it is unique for"
220
" each node.")
221
223
- VLLM_INSTANCE_ID = get_vllm_instance_id()
224
-
222
# Set environment variables for the driver and workers.
223
all_args_to_update_environment_variables = [({
224
"CUDA_VISIBLE_DEVICES":
225
",".join(map(str, node_gpus[node_id])),
229
- "VLLM_INSTANCE_ID":
230
- VLLM_INSTANCE_ID,
226
"VLLM_TRACE_FUNCTION":
227
str(envs.VLLM_TRACE_FUNCTION),
228
**({
vllm/executor/ray_hpu_executor.py
+1
-6
@@ -15,8 +15,7 @@ from vllm.logger import init_logger
15
from vllm.model_executor.layers.sampler import SamplerOutput
16
from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18
- get_ip, get_open_port, get_vllm_instance_id,
19
- make_async)
18
+ get_ip, get_open_port, make_async)
19
20
if ray is not None:
21
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -196,12 +195,8 @@ class RayHPUExecutor(DistributedGPUExecutor):
195
"environment variable, make sure it is unique for"
196
" each node.")
197
199
- VLLM_INSTANCE_ID = get_vllm_instance_id()
200
-
198
# Set environment variables for the driver and workers.
199
all_args_to_update_environment_variables = [({
203
- "VLLM_INSTANCE_ID":
204
- VLLM_INSTANCE_ID,
200
"VLLM_TRACE_FUNCTION":
201
str(envs.VLLM_TRACE_FUNCTION),
202
}, ) for (node_id, _) in worker_node_and_gpu_ids]
vllm/executor/ray_tpu_executor.py
+1
-5
@@ -13,7 +13,7 @@ from vllm.logger import init_logger
13
from vllm.model_executor.layers.sampler import SamplerOutput
14
from vllm.sequence import ExecuteModelRequest
15
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16
- get_vllm_instance_id, make_async)
16
+ make_async)
17
18
if ray is not None:
19
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
@@ -144,12 +144,8 @@ class RayTPUExecutor(TPUExecutor):
144
for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
145
node_workers[node_id].append(i)
146
147
- VLLM_INSTANCE_ID = get_vllm_instance_id()
148
-
147
# Set environment variables for the driver and workers.
148
all_args_to_update_environment_variables = [({
151
- "VLLM_INSTANCE_ID":
152
- VLLM_INSTANCE_ID,
149
"VLLM_TRACE_FUNCTION":
150
str(envs.VLLM_TRACE_FUNCTION),
151
}, ) for _ in worker_node_and_gpu_ids]
vllm/executor/ray_xpu_executor.py
+1
-5
@@ -5,7 +5,7 @@ import vllm.envs as envs
5
from vllm.executor.ray_gpu_executor import RayGPUExecutor, RayGPUExecutorAsync
6
from vllm.executor.xpu_executor import XPUExecutor
7
from vllm.logger import init_logger
8
-from vllm.utils import get_vllm_instance_id, make_async
8
+from vllm.utils import make_async
9
10
logger = init_logger(__name__)
11
@@ -17,12 +17,8 @@ class RayXPUExecutor(RayGPUExecutor, XPUExecutor):
17
worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
18
use_dummy_driver=True)
19
20
- VLLM_INSTANCE_ID = get_vllm_instance_id()
21
-
20
# Set environment variables for the driver and workers.
21
all_args_to_update_environment_variables = [({
24
- "VLLM_INSTANCE_ID":
25
- VLLM_INSTANCE_ID,
22
"VLLM_TRACE_FUNCTION":
23
str(envs.VLLM_TRACE_FUNCTION),
24
}, ) for (_, _) in worker_node_and_gpu_ids]
vllm/utils.py
+9
-16
@@ -24,9 +24,9 @@ from collections import UserDict, defaultdict
24
from collections.abc import Iterable, Mapping
25
from functools import lru_cache, partial, wraps
26
from platform import uname
27
-from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, Generic,
28
- Hashable, List, Literal, Optional, OrderedDict, Set, Tuple,
29
- Type, TypeVar, Union, overload)
27
+from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
28
+ Dict, Generic, Hashable, List, Literal, Optional,
29
+ OrderedDict, Set, Tuple, Type, TypeVar, Union, overload)
30
from uuid import uuid4
31
32
import numpy as np
@@ -43,6 +43,9 @@ import vllm.envs as envs
43
from vllm.logger import enable_trace_function_call, init_logger
44
from vllm.platforms import current_platform
45
46
+if TYPE_CHECKING:
47
+ from vllm.config import VllmConfig
48
+
49
logger = init_logger(__name__)
50
51
# Exception strings for non-implemented encoder/decoder scenarios
@@ -335,17 +338,6 @@ def random_uuid() -> str:
338
return str(uuid.uuid4().hex)
339
340
338
-@lru_cache(maxsize=None)
339
-def get_vllm_instance_id() -> str:
340
- """
341
- If the environment variable VLLM_INSTANCE_ID is set, return it.
342
- Otherwise, return a random UUID.
343
- Instance id represents an instance of the VLLM. All processes in the same
344
- instance should have the same instance id.
345
- """
346
- return envs.VLLM_INSTANCE_ID or f"vllm-instance-{random_uuid()}"
347
-
348
-
341
@lru_cache(maxsize=None)
342
def in_wsl() -> bool:
343
# Reference: https://github.com/microsoft/WSL/issues/4071
@@ -997,7 +989,7 @@ def find_nccl_library() -> str:
989
return so_file
990
991
1000
-def enable_trace_function_call_for_thread() -> None:
992
+def enable_trace_function_call_for_thread(vllm_config: "VllmConfig") -> None:
993
"""Set up function tracing for the current thread,
994
if enabled via the VLLM_TRACE_FUNCTION environment variable
995
"""
@@ -1009,7 +1001,8 @@ def enable_trace_function_call_for_thread() -> None:
1001
filename = (f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
1002
f"_thread_{threading.get_ident()}_"
1003
f"at_{datetime.datetime.now()}.log").replace(" ", "_")
1012
- log_path = os.path.join(tmp_dir, "vllm", get_vllm_instance_id(),
1004
+ log_path = os.path.join(tmp_dir, "vllm",
1005
+ f"vllm-instance-{vllm_config.instance_id}",
1006
filename)
1007
os.makedirs(os.path.dirname(log_path), exist_ok=True)
1008
enable_trace_function_call(log_path)
vllm/worker/worker_base.py
+1
-1
@@ -439,7 +439,7 @@ class WorkerWrapperBase:
439
Here we inject some common logic before initializing the worker.
440
Arguments are passed to the worker class constructor.
441
"""
442
- enable_trace_function_call_for_thread()
442
+ enable_trace_function_call_for_thread(self.vllm_config)
443
444
# see https://github.com/NVIDIA/nccl/issues/1234
445
os.environ['NCCL_CUMEM_ENABLE'] = '0'