@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40bc242579
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Bugfix] Fix OpenVino/Neuron `driver_worker` init (#10779)

Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Nicolò Lucchesi committed Nov 30, 2024 at 05:07 UTC 40bc242579d260e6da7614e1494cbd80a6f985b2
2 files changed +6 -3
vllm/executor/neuron_executor.py
+4 -2
@@ -29,11 +29,13 @@ class NeuronExecutor(ExecutorBase):
29 wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
30 distributed_init_method = get_distributed_init_method(
31 get_ip(), get_open_port())
32 - self.driver_worker = wrapper.init_worker(
32 + wrapper.init_worker(
33 vllm_config=self.vllm_config,
34 local_rank=0,
35 rank=0,
36 - distributed_init_method=distributed_init_method)
36 + distributed_init_method=distributed_init_method,
37 + )
38 + self.driver_worker = wrapper.worker
39 self.driver_worker.init_device()
40 self.driver_worker.load_model()
41
vllm/executor/openvino_executor.py
+2 -1
@@ -36,7 +36,7 @@ class OpenVINOExecutor(ExecutorBase):
36
37 distributed_init_method = get_distributed_init_method(
38 get_ip(), get_open_port())
39 - self.driver_worker = wrapper.init_worker(
39 + wrapper.init_worker(
40 ov_core=ov.Core(),
41 vllm_config=self.vllm_config,
42 local_rank=0,
@@ -45,6 +45,7 @@ class OpenVINOExecutor(ExecutorBase):
45 kv_cache_dtype=self.cache_config.cache_dtype,
46 is_driver_worker=True,
47 )
48 + self.driver_worker = wrapper.worker
49 self.driver_worker.init_device()
50 self.driver_worker.load_model()
51