@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2efce05dc3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Fix] Avoid pickling entire LLMEngine for Ray workers (#3207)

Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>

Nick Hill committed Mar 5, 2024 at 16:17 UTC 2efce05dc3c7c1e367617465f8f661a058499e37
1 file changed +14 -7
vllm/engine/llm_engine.py
+14 -7
@@ -158,6 +158,11 @@ class LLMEngine:
158 if USE_RAY_COMPILED_DAG:
159 self.forward_dag = self._compiled_ray_dag()
160
161 + def __reduce__(self):
162 + # This is to ensure that the LLMEngine is not referenced in
163 + # the closure used to initialize Ray worker actors
164 + raise RuntimeError("LLMEngine should not be pickled!")
165 +
166 def get_tokenizer_for_seq(self, sequence: Sequence):
167 return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
168
@@ -280,6 +285,8 @@ class LLMEngine:
285 parallel_config = copy.deepcopy(self.parallel_config)
286 scheduler_config = copy.deepcopy(self.scheduler_config)
287 device_config = copy.deepcopy(self.device_config)
288 + lora_config = copy.deepcopy(self.lora_config)
289 + kv_cache_dtype = self.cache_config.cache_dtype
290
291 for rank, (worker, (node_id,
292 _)) in enumerate(zip(self.workers,
@@ -295,22 +302,22 @@ class LLMEngine:
302 local_rank,
303 rank,
304 distributed_init_method,
298 - lora_config=self.lora_config,
299 - kv_cache_dtype=self.cache_config.cache_dtype,
305 + lora_config=lora_config,
306 + kv_cache_dtype=kv_cache_dtype,
307 ))
308
309 driver_rank = 0
310 driver_local_rank = node_workers[driver_node_id].index(driver_rank)
311 self.driver_worker = Worker(
305 - model_config,
306 - parallel_config,
307 - scheduler_config,
308 - device_config,
312 + self.model_config,
313 + self.parallel_config,
314 + self.scheduler_config,
315 + self.device_config,
316 driver_local_rank,
317 driver_rank,
318 distributed_init_method,
319 lora_config=self.lora_config,
313 - kv_cache_dtype=self.cache_config.cache_dtype,
320 + kv_cache_dtype=kv_cache_dtype,
321 is_driver_worker=True,
322 )
323