A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Fix] Avoid pickling entire LLMEngine for Ray workers (#3207)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Nick Hill committed
Mar 5, 2024 at 16:17 UTC
2efce05dc3c7c1e367617465f8f661a058499e37
1 file changed
+14
-7
vllm/engine/llm_engine.py
+14
-7
@@ -158,6 +158,11 @@ class LLMEngine:
158
if USE_RAY_COMPILED_DAG:
159
self.forward_dag = self._compiled_ray_dag()
160
161
+ def __reduce__(self):
162
+ # This is to ensure that the LLMEngine is not referenced in
163
+ # the closure used to initialize Ray worker actors
164
+ raise RuntimeError("LLMEngine should not be pickled!")
165
+
166
def get_tokenizer_for_seq(self, sequence: Sequence):
167
return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
168
@@ -280,6 +285,8 @@ class LLMEngine:
285
parallel_config = copy.deepcopy(self.parallel_config)
286
scheduler_config = copy.deepcopy(self.scheduler_config)
287
device_config = copy.deepcopy(self.device_config)
288
+ lora_config = copy.deepcopy(self.lora_config)
289
+ kv_cache_dtype = self.cache_config.cache_dtype
290
291
for rank, (worker, (node_id,
292
_)) in enumerate(zip(self.workers,
@@ -295,22 +302,22 @@ class LLMEngine:
302
local_rank,
303
rank,
304
distributed_init_method,
298
- lora_config=self.lora_config,
299
- kv_cache_dtype=self.cache_config.cache_dtype,
305
+ lora_config=lora_config,
306
+ kv_cache_dtype=kv_cache_dtype,
307
))
308
309
driver_rank = 0
310
driver_local_rank = node_workers[driver_node_id].index(driver_rank)
311
self.driver_worker = Worker(
305
- model_config,
306
- parallel_config,
307
- scheduler_config,
308
- device_config,
312
+ self.model_config,
313
+ self.parallel_config,
314
+ self.scheduler_config,
315
+ self.device_config,
316
driver_local_rank,
317
driver_rank,
318
distributed_init_method,
319
lora_config=self.lora_config,
313
- kv_cache_dtype=self.cache_config.cache_dtype,
320
+ kv_cache_dtype=kv_cache_dtype,
321
is_driver_worker=True,
322
)
323