A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[MISC][V1] Handle exception of current_platform.get_device_name() in arg_utils (#14379)
Signed-off-by: Cody Yu <hao.yu.cody@gmail.com>
Cody Yu committed
Mar 10, 2025 at 17:42 UTC
485afdd3cb34357c0e83ece752710d5acecf1471
1 file changed
+13
-2
vllm/engine/arg_utils.py
+13
-2
@@ -1440,8 +1440,19 @@ class EngineArgs:
1440
# When no user override, set the default values based on the usage
1441
# context.
1442
# Use different default values for different hardware.
1443
- from vllm.platforms import current_platform
1444
- device_name = current_platform.get_device_name().lower()
1443
+
1444
+ # Try to query the device name on the current platform. If it fails,
1445
+ # it may be because the platform that imports vLLM is not the same
1446
+ # as the platform that vLLM is running on (e.g. the case of scaling
1447
+ # vLLM with Ray) and has no GPUs. In this case we use the default
1448
+ # values for non-H100/H200 GPUs.
1449
+ try:
1450
+ from vllm.platforms import current_platform
1451
+ device_name = current_platform.get_device_name().lower()
1452
+ except Exception:
1453
+ # This is only used to set default_max_num_batched_tokens
1454
+ device_name = "no-device"
1455
+
1456
if "h100" in device_name or "h200" in device_name:
1457
# For H100 and H200, we use larger default values.
1458
default_max_num_batched_tokens = {