A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] fix_small_bug_in_neuron_executor (#4051)
zspo committed
Apr 13, 2024 at 22:54 UTC
0a430b4ae2763c2f161e3bfb1529acf4685f7caa
1 file changed
+2
vllm/executor/neuron_executor.py
+2
@@ -25,6 +25,7 @@ class NeuronExecutor(ExecutorBase):
25
speculative_config: Optional[SpeculativeConfig],
26
) -> None:
27
self.model_config = model_config
28
+ self.cache_config = cache_config
29
assert lora_config is None, "LoRA is not supported for Neuron backend."
30
self.parallel_config = parallel_config
31
self.scheduler_config = scheduler_config
@@ -43,6 +44,7 @@ class NeuronExecutor(ExecutorBase):
44
self.parallel_config,
45
self.scheduler_config,
46
self.device_config,
47
+ self.cache_config,
48
)
49
self.driver_worker.init_device()
50
self.driver_worker.load_model()