A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[executor] init `local_rank` as device index (#13027)
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Mengqing Cao committed
Feb 11, 2025 at 21:20 UTC
9cf4759493919580011f03812abf16387eafe18c
1 file changed
+5
vllm/executor/uniproc_executor.py
+5
@@ -28,6 +28,11 @@ class UniProcExecutor(ExecutorBase):
28
distributed_init_method = get_distributed_init_method(
29
get_ip(), get_open_port())
30
local_rank = 0
31
+ # set local rank as the device index if specified
32
+ device_info = self.vllm_config.device_config.device.__str__().split(
33
+ ":")
34
+ if len(device_info) > 1:
35
+ local_rank = int(device_info[1])
36
rank = 0
37
kwargs = dict(
38
vllm_config=self.vllm_config,