@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9cf4759493
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[executor] init `local_rank` as device index (#13027)

Signed-off-by: Mengqing Cao <cmq0113@163.com>

Mengqing Cao committed Feb 11, 2025 at 21:20 UTC 9cf4759493919580011f03812abf16387eafe18c
1 file changed +5
vllm/executor/uniproc_executor.py
+5
@@ -28,6 +28,11 @@ class UniProcExecutor(ExecutorBase):
28 distributed_init_method = get_distributed_init_method(
29 get_ip(), get_open_port())
30 local_rank = 0
31 + # set local rank as the device index if specified
32 + device_info = self.vllm_config.device_config.device.__str__().split(
33 + ":")
34 + if len(device_info) > 1:
35 + local_rank = int(device_info[1])
36 rank = 0
37 kwargs = dict(
38 vllm_config=self.vllm_config,