A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix Llava inference with Tensor Parallelism. (#3883)
Isotr0py committed
Apr 7, 2024 at 22:54 UTC
0ce0539d4750f9ebcd9b19d7085ca3b934b9ec67
1 file changed
+2
vllm/executor/ray_gpu_executor.py
+2
@@ -154,6 +154,7 @@ class RayGPUExecutor(ExecutorBase):
154
scheduler_config = copy.deepcopy(self.scheduler_config)
155
device_config = copy.deepcopy(self.device_config)
156
lora_config = copy.deepcopy(self.lora_config)
157
+ vision_language_config = copy.deepcopy(self.vision_language_config)
158
kv_cache_dtype = self.cache_config.cache_dtype
159
160
# Initialize the actual workers with the Worker class.
@@ -172,6 +173,7 @@ class RayGPUExecutor(ExecutorBase):
173
rank,
174
distributed_init_method,
175
lora_config=lora_config,
176
+ vision_language_config=vision_language_config,
177
kv_cache_dtype=kv_cache_dtype,
178
))
179