A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Core] Don't do platform detection at import time (#12933)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Feb 11, 2025 at 02:25 UTC
c320ca8edd5c4c19e7581703e428dd566b068756
3 files changed
+8
-8
vllm/executor/executor_base.py
+3
-3
@@ -8,11 +8,11 @@ from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
8
import torch.nn as nn
9
from typing_extensions import TypeVar
10
11
+import vllm.platforms
12
from vllm.config import VllmConfig
13
from vllm.logger import init_logger
14
from vllm.lora.request import LoRARequest
15
from vllm.model_executor.layers.sampler import SamplerOutput
15
-from vllm.platforms import current_platform
16
from vllm.prompt_adapter.request import PromptAdapterRequest
17
from vllm.sequence import ExecuteModelRequest, PoolerOutput
18
from vllm.utils import make_async
@@ -108,8 +108,8 @@ class ExecutorBase(ABC):
108
"""
109
# NOTE: This is logged in the executor because there can be >1 workers.
110
logger.info("# %s blocks: %d, # CPU blocks: %d",
111
- current_platform.dispatch_key, num_gpu_blocks,
112
- num_cpu_blocks)
111
+ vllm.platforms.current_platform.dispatch_key,
112
+ num_gpu_blocks, num_cpu_blocks)
113
max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
114
self.model_config.max_model_len)
115
logger.info("Maximum concurrency for %s tokens per request: %.2fx",
vllm/executor/ray_utils.py
+3
-3
@@ -7,10 +7,10 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8
import msgspec
9
10
+import vllm.platforms
11
from vllm.config import ParallelConfig
12
from vllm.executor.msgspec_utils import decode_hook, encode_hook
13
from vllm.logger import init_logger
13
-from vllm.platforms import current_platform
14
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
15
from vllm.utils import get_ip
16
from vllm.worker.worker_base import WorkerWrapperBase
@@ -54,10 +54,10 @@ try:
54
55
def get_node_and_gpu_ids(self) -> Tuple[str, List[int]]:
56
node_id = ray.get_runtime_context().get_node_id()
57
- device_key = current_platform.ray_device_key
57
+ device_key = vllm.platforms.current_platform.ray_device_key
58
if not device_key:
59
raise RuntimeError("current platform %s does not support ray.",
60
- current_platform.device_name)
60
+ vllm.platforms.current_platform.device_name)
61
gpu_ids = ray.get_runtime_context().get_accelerator_ids(
62
)[device_key]
63
return node_id, gpu_ids
vllm/platforms/cuda.py
+2
-2
@@ -334,10 +334,10 @@ class NvmlCudaPlatform(CudaPlatformBase):
334
if (len(set(device_names)) > 1
335
and os.environ.get("CUDA_DEVICE_ORDER") != "PCI_BUS_ID"):
336
logger.warning(
337
- "Detected different devices in the system: \n%s\nPlease"
337
+ "Detected different devices in the system: %s. Please"
338
" make sure to set `CUDA_DEVICE_ORDER=PCI_BUS_ID` to "
339
"avoid unexpected behavior.",
340
- "\n".join(device_names),
340
+ ", ".join(device_names),
341
)
342
343