@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c320ca8edd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Core] Don't do platform detection at import time (#12933)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Feb 11, 2025 at 02:25 UTC c320ca8edd5c4c19e7581703e428dd566b068756
3 files changed +8 -8
vllm/executor/executor_base.py
+3 -3
@@ -8,11 +8,11 @@ from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
8 import torch.nn as nn
9 from typing_extensions import TypeVar
10
11 +import vllm.platforms
12 from vllm.config import VllmConfig
13 from vllm.logger import init_logger
14 from vllm.lora.request import LoRARequest
15 from vllm.model_executor.layers.sampler import SamplerOutput
15 -from vllm.platforms import current_platform
16 from vllm.prompt_adapter.request import PromptAdapterRequest
17 from vllm.sequence import ExecuteModelRequest, PoolerOutput
18 from vllm.utils import make_async
@@ -108,8 +108,8 @@ class ExecutorBase(ABC):
108 """
109 # NOTE: This is logged in the executor because there can be >1 workers.
110 logger.info("# %s blocks: %d, # CPU blocks: %d",
111 - current_platform.dispatch_key, num_gpu_blocks,
112 - num_cpu_blocks)
111 + vllm.platforms.current_platform.dispatch_key,
112 + num_gpu_blocks, num_cpu_blocks)
113 max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
114 self.model_config.max_model_len)
115 logger.info("Maximum concurrency for %s tokens per request: %.2fx",
vllm/executor/ray_utils.py
+3 -3
@@ -7,10 +7,10 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8 import msgspec
9
10 +import vllm.platforms
11 from vllm.config import ParallelConfig
12 from vllm.executor.msgspec_utils import decode_hook, encode_hook
13 from vllm.logger import init_logger
13 -from vllm.platforms import current_platform
14 from vllm.sequence import ExecuteModelRequest, IntermediateTensors
15 from vllm.utils import get_ip
16 from vllm.worker.worker_base import WorkerWrapperBase
@@ -54,10 +54,10 @@ try:
54
55 def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
56 node_id = ray.get_runtime_context().get_node_id()
57 - device_key = current_platform.ray_device_key
57 + device_key = vllm.platforms.current_platform.ray_device_key
58 if not device_key:
59 raise RuntimeError("current platform %s does not support ray.",
60 - current_platform.device_name)
60 + vllm.platforms.current_platform.device_name)
61 gpu_ids = ray.get_runtime_context().get_accelerator_ids(
62 )[device_key]
63 return node_id, gpu_ids
vllm/platforms/cuda.py
+2 -2
@@ -334,10 +334,10 @@ class NvmlCudaPlatform(CudaPlatformBase):
334 if (len(set(device_names)) > 1
335 and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"):
336 logger.warning(
337 - "Detected different devices in the system: \n%s\nPlease"
337 + "Detected different devices in the system: %s. Please"
338 " make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
339 "avoid unexpected behavior.",
340 - "\n".join(device_names),
340 + ", ".join(device_names),
341 )
342
343