@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ce0539d47
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix Llava inference with Tensor Parallelism. (#3883)

Isotr0py committed Apr 7, 2024 at 22:54 UTC 0ce0539d4750f9ebcd9b19d7085ca3b934b9ec67
1 file changed +2
vllm/executor/ray_gpu_executor.py
+2
@@ -154,6 +154,7 @@ class RayGPUExecutor(ExecutorBase):
154 scheduler_config = copy.deepcopy(self.scheduler_config)
155 device_config = copy.deepcopy(self.device_config)
156 lora_config = copy.deepcopy(self.lora_config)
157 + vision_language_config = copy.deepcopy(self.vision_language_config)
158 kv_cache_dtype = self.cache_config.cache_dtype
159
160 # Initialize the actual workers with the Worker class.
@@ -172,6 +173,7 @@ class RayGPUExecutor(ExecutorBase):
173 rank,
174 distributed_init_method,
175 lora_config=lora_config,
176 + vision_language_config=vision_language_config,
177 kv_cache_dtype=kv_cache_dtype,
178 ))
179