@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a430b4ae2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] fix_small_bug_in_neuron_executor (#4051)

zspo committed Apr 13, 2024 at 22:54 UTC 0a430b4ae2763c2f161e3bfb1529acf4685f7caa
1 file changed +2
vllm/executor/neuron_executor.py
+2
@@ -25,6 +25,7 @@ class NeuronExecutor(ExecutorBase):
25 speculative_config: Optional[SpeculativeConfig],
26 ) -> None:
27 self.model_config = model_config
28 + self.cache_config = cache_config
29 assert lora_config is None, "LoRA is not supported for Neuron backend."
30 self.parallel_config = parallel_config
31 self.scheduler_config = scheduler_config
@@ -43,6 +44,7 @@ class NeuronExecutor(ExecutorBase):
44 self.parallel_config,
45 self.scheduler_config,
46 self.device_config,
47 + self.cache_config,
48 )
49 self.driver_worker.init_device()
50 self.driver_worker.load_model()