A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Mapping physical device indices for e2e test utils (#8290)
shangmingc committed
Sep 13, 2024 at 11:06 UTC
40c396533d00b9b6efe08241525630dcf8d88c72
1 file changed
+11
tests/utils.py
+11
@@ -356,12 +356,23 @@ def error_on_warning():
356
yield
357
358
359
+def get_physical_device_indices(devices):
360
+ visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
361
+ if visible_devices is None:
362
+ return devices
363
+
364
+ visible_indices = [int(x) for x in visible_devices.split(",")]
365
+ index_mapping = {i: physical for i, physical in enumerate(visible_indices)}
366
+ return [index_mapping[i] for i in devices if i in index_mapping]
367
+
368
+
369
@_nvml()
370
def wait_for_gpu_memory_to_clear(devices: List[int],
371
threshold_bytes: int,
372
timeout_s: float = 120) -> None:
373
# Use nvml instead of pytorch to reduce measurement error from torch cuda
374
# context.
375
+ devices = get_physical_device_indices(devices)
376
start_time = time.time()
377
while True:
378
output: Dict[int, str] = {}