A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Bugfix] Fix OpenVino/Neuron `driver_worker` init (#10779)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Nicolò Lucchesi committed
Nov 30, 2024 at 05:07 UTC
40bc242579d260e6da7614e1494cbd80a6f985b2
2 files changed
+6
-3
vllm/executor/neuron_executor.py
+4
-2
@@ -29,11 +29,13 @@ class NeuronExecutor(ExecutorBase):
29
wrapper = WorkerWrapperBase(vllm_config=self.vllm_config)
30
distributed_init_method = get_distributed_init_method(
31
get_ip(), get_open_port())
32
- self.driver_worker = wrapper.init_worker(
32
+ wrapper.init_worker(
33
vllm_config=self.vllm_config,
34
local_rank=0,
35
rank=0,
36
- distributed_init_method=distributed_init_method)
36
+ distributed_init_method=distributed_init_method,
37
+ )
38
+ self.driver_worker = wrapper.worker
39
self.driver_worker.init_device()
40
self.driver_worker.load_model()
41
vllm/executor/openvino_executor.py
+2
-1
@@ -36,7 +36,7 @@ class OpenVINOExecutor(ExecutorBase):
36
37
distributed_init_method = get_distributed_init_method(
38
get_ip(), get_open_port())
39
- self.driver_worker = wrapper.init_worker(
39
+ wrapper.init_worker(
40
ov_core=ov.Core(),
41
vllm_config=self.vllm_config,
42
local_rank=0,
@@ -45,6 +45,7 @@ class OpenVINOExecutor(ExecutorBase):
45
kv_cache_dtype=self.cache_config.cache_dtype,
46
is_driver_worker=True,
47
)
48
+ self.driver_worker = wrapper.worker
49
self.driver_worker.init_device()
50
self.driver_worker.load_model()
51