@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 198d6a2898
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Shut down aDAG workers with clean async llm engine exit (#7224)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com>

Rui Qiao committed Aug 12, 2024 at 17:57 UTC 198d6a289849bc64a17555e0cbb15a197a1229cb
5 files changed +40 -25
tests/distributed/test_pipeline_parallel.py
+4 -8
@@ -34,9 +34,6 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
34 pytest.skip("Skipping multi-node pipeline parallel test for "
35 "multiprocessing distributed backend")
36
37 - USE_RAY_ADAG_NCCL = 0
38 - USE_RAY_ADAG = 0
39 -
37 pp_args = [
38 # use half precision for speed and memory savings in CI environment
39 "--dtype",
@@ -70,14 +67,13 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
67 pp_args.append("--enforce-eager")
68 tp_args.append("--enforce-eager")
69 pp_env = None
73 - if USE_RAY_ADAG:
74 - assert DIST_BACKEND == "ray", (
75 - "Ray ADAG is only supported with Ray distributed backend")
70 + if (DIST_BACKEND == "ray" and TP_SIZE == 2 and PP_SIZE == 2
71 + and CHUNKED_PREFILL):
72 + # Test Ray ADAG for a subset of the tests
73 pp_env = {
74 "VLLM_USE_RAY_COMPILED_DAG": "1",
75 "VLLM_USE_RAY_SPMD_WORKER": "1",
79 - "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
80 - str(int(USE_RAY_ADAG_NCCL)),
76 + "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": "1",
77 }
78
79 compare_two_settings(MODEL_NAME, pp_args, tp_args, pp_env)
vllm/engine/async_llm_engine.py
+14
@@ -661,6 +661,20 @@ class AsyncLLMEngine:
661 partial(_log_task_completion, error_callback=self._error_callback))
662 self.background_loop = asyncio.shield(self._background_loop_unshielded)
663
664 + def shutdown_background_loop(self) -> None:
665 + """
666 + Shut down the background loop.
667 +
668 + This method needs to be called during cleanup to remove
669 + references to `self` and properly GC the resources held
670 + by the async LLM engine (e.g., the executors as well as
671 + their resources).
672 + """
673 + if self._background_loop_unshielded is not None:
674 + self._background_loop_unshielded.cancel()
675 + self._background_loop_unshielded = None
676 + self.background_loop = None
677 +
678 def _init_engine(self, *args,
679 **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
680 if not self.engine_use_ray:
vllm/engine/llm_engine.py
+11 -6
@@ -245,9 +245,18 @@ class LLMEngine:
245 if not self.model_config.skip_tokenizer_init:
246 self.tokenizer = self._init_tokenizer()
247 self.detokenizer = Detokenizer(self.tokenizer)
248 + tokenizer_group = self.get_tokenizer_group()
249 else:
250 self.tokenizer = None
251 self.detokenizer = None
252 + tokenizer_group = None
253 +
254 + # Ensure that the function doesn't contain a reference to self,
255 + # to avoid engine GC issues
256 + def get_tokenizer_for_seq(sequence: Sequence) -> AnyTokenizer:
257 + assert tokenizer_group, ("tokenizer_group cannot be None, "
258 + "make sure skip_tokenizer_init is False")
259 + return tokenizer_group.get_lora_tokenizer(sequence.lora_request)
260
261 self.seq_counter = Counter()
262 self.generation_config_fields = _load_generation_config_dict(
@@ -356,10 +365,10 @@ class LLMEngine:
365 self.detokenizer,
366 self.scheduler,
367 self.seq_counter,
359 - self.get_tokenizer_for_seq,
368 + get_tokenizer_for_seq,
369 stop_checker=StopChecker(
370 self.scheduler_config.max_model_len,
362 - self.get_tokenizer_for_seq,
371 + get_tokenizer_for_seq,
372 ),
373 ))
374
@@ -491,10 +500,6 @@ class LLMEngine:
500 ) -> AnyTokenizer:
501 return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
502
494 - def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
495 - return self.get_tokenizer_group().get_lora_tokenizer(
496 - sequence.lora_request)
497 -
503 def _init_tokenizer(self) -> BaseTokenizerGroup:
504 return init_tokenizer_from_configs(
505 model_config=self.model_config,
vllm/entrypoints/openai/rpc/server.py
+1
@@ -36,6 +36,7 @@ class AsyncEngineRPCServer:
36 """Cleanup all resources."""
37 self.socket.close()
38 self.context.destroy()
39 + self.engine.shutdown_background_loop()
40
41 async def get_model_config(self, identity):
42 """Send the ModelConfig"""
vllm/executor/ray_gpu_executor.py
+10 -11
@@ -60,6 +60,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
60 # Create the parallel GPU workers.
61 self._init_workers_ray(placement_group)
62
63 + def shutdown(self) -> None:
64 + if hasattr(self, "forward_dag") and self.forward_dag is not None:
65 + self.forward_dag.teardown()
66 + import ray
67 + for worker in self.workers:
68 + ray.kill(worker)
69 + self.forward_dag = None
70 +
71 def _configure_ray_workers_use_nsight(self,
72 ray_remote_kwargs) -> Dict[str, Any]:
73 # If nsight profiling is enabled, we need to set the profiling
@@ -117,7 +125,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
125 logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
126 # Create the workers.
127 driver_ip = get_ip()
120 - logger.info("driver_ip: %s", driver_ip)
128 worker_wrapper_kwargs = self._get_worker_wrapper_args()
129 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
130 if not bundle.get("GPU", 0):
@@ -446,11 +453,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
453 return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
454
455 def __del__(self):
449 - if self.forward_dag is not None:
450 - self.forward_dag.teardown()
451 - import ray
452 - for worker in self.workers:
453 - ray.kill(worker)
456 + self.shutdown()
457
458
459 class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
@@ -523,8 +526,4 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
526 return await asyncio.gather(*coros)
527
528 def __del__(self):
526 - if self.forward_dag is not None:
527 - self.forward_dag.teardown()
528 - import ray
529 - for worker in self.workers:
530 - ray.kill(worker)
529 + self.shutdown()