@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40c396533d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Mapping physical device indices for e2e test utils (#8290)

shangmingc committed Sep 13, 2024 at 11:06 UTC 40c396533d00b9b6efe08241525630dcf8d88c72
1 file changed +11
tests/utils.py
+11
@@ -356,12 +356,23 @@ def error_on_warning():
356 yield
357
358
359 +def get_physical_device_indices(devices):
360 + visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
361 + if visible_devices is None:
362 + return devices
363 +
364 + visible_indices = [int(x) for x in visible_devices.split(",")]
365 + index_mapping = {i: physical for i, physical in enumerate(visible_indices)}
366 + return [index_mapping[i] for i in devices if i in index_mapping]
367 +
368 +
369 @_nvml()
370 def wait_for_gpu_memory_to_clear(devices: List[int],
371 threshold_bytes: int,
372 timeout_s: float = 120) -> None:
373 # Use nvml instead of pytorch to reduce measurement error from torch cuda
374 # context.
375 + devices = get_physical_device_indices(devices)
376 start_time = time.time()
377 while True:
378 output: Dict[int, str] = {}