A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Shut down aDAG workers with clean async llm engine exit (#7224)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Aug 12, 2024 at 17:57 UTC
198d6a289849bc64a17555e0cbb15a197a1229cb
5 files changed
+40
-25
tests/distributed/test_pipeline_parallel.py
+4
-8
@@ -34,9 +34,6 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
34
pytest.skip("Skipping multi-node pipeline parallel test for "
35
"multiprocessing distributed backend")
36
37
- USE_RAY_ADAG_NCCL = 0
38
- USE_RAY_ADAG = 0
39
-
37
pp_args = [
38
# use half precision for speed and memory savings in CI environment
39
"--dtype",
@@ -70,14 +67,13 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
67
pp_args.append("--enforce-eager")
68
tp_args.append("--enforce-eager")
69
pp_env = None
73
- if USE_RAY_ADAG:
74
- assert DIST_BACKEND == "ray", (
75
- "Ray ADAG is only supported with Ray distributed backend")
70
+ if (DIST_BACKEND == "ray" and TP_SIZE == 2 and PP_SIZE == 2
71
+ and CHUNKED_PREFILL):
72
+ # Test Ray ADAG for a subset of the tests
73
pp_env = {
74
"VLLM_USE_RAY_COMPILED_DAG": "1",
75
"VLLM_USE_RAY_SPMD_WORKER": "1",
79
- "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
80
- str(int(USE_RAY_ADAG_NCCL)),
76
+ "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": "1",
77
}
78
79
compare_two_settings(MODEL_NAME, pp_args, tp_args, pp_env)
vllm/engine/async_llm_engine.py
+14
@@ -661,6 +661,20 @@ class AsyncLLMEngine:
661
partial(_log_task_completion, error_callback=self._error_callback))
662
self.background_loop = asyncio.shield(self._background_loop_unshielded)
663
664
+ def shutdown_background_loop(self) -> None:
665
+ """
666
+ Shut down the background loop.
667
+
668
+ This method needs to be called during cleanup to remove
669
+ references to `self` and properly GC the resources held
670
+ by the async LLM engine (e.g., the executors as well as
671
+ their resources).
672
+ """
673
+ if self._background_loop_unshielded is not None:
674
+ self._background_loop_unshielded.cancel()
675
+ self._background_loop_unshielded = None
676
+ self.background_loop = None
677
+
678
def _init_engine(self, *args,
679
**kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
680
if not self.engine_use_ray:
vllm/engine/llm_engine.py
+11
-6
@@ -245,9 +245,18 @@ class LLMEngine:
245
if not self.model_config.skip_tokenizer_init:
246
self.tokenizer = self._init_tokenizer()
247
self.detokenizer = Detokenizer(self.tokenizer)
248
+ tokenizer_group = self.get_tokenizer_group()
249
else:
250
self.tokenizer = None
251
self.detokenizer = None
252
+ tokenizer_group = None
253
+
254
+ # Ensure that the function doesn't contain a reference to self,
255
+ # to avoid engine GC issues
256
+ def get_tokenizer_for_seq(sequence: Sequence) -> AnyTokenizer:
257
+ assert tokenizer_group, ("tokenizer_group cannot be None, "
258
+ "make sure skip_tokenizer_init is False")
259
+ return tokenizer_group.get_lora_tokenizer(sequence.lora_request)
260
261
self.seq_counter = Counter()
262
self.generation_config_fields = _load_generation_config_dict(
@@ -356,10 +365,10 @@ class LLMEngine:
365
self.detokenizer,
366
self.scheduler,
367
self.seq_counter,
359
- self.get_tokenizer_for_seq,
368
+ get_tokenizer_for_seq,
369
stop_checker=StopChecker(
370
self.scheduler_config.max_model_len,
362
- self.get_tokenizer_for_seq,
371
+ get_tokenizer_for_seq,
372
),
373
))
374
@@ -491,10 +500,6 @@ class LLMEngine:
500
) -> AnyTokenizer:
501
return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
502
494
- def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
495
- return self.get_tokenizer_group().get_lora_tokenizer(
496
- sequence.lora_request)
497
-
503
def _init_tokenizer(self) -> BaseTokenizerGroup:
504
return init_tokenizer_from_configs(
505
model_config=self.model_config,
vllm/entrypoints/openai/rpc/server.py
+1
@@ -36,6 +36,7 @@ class AsyncEngineRPCServer:
36
"""Cleanup all resources."""
37
self.socket.close()
38
self.context.destroy()
39
+ self.engine.shutdown_background_loop()
40
41
async def get_model_config(self, identity):
42
"""Send the ModelConfig"""
vllm/executor/ray_gpu_executor.py
+10
-11
@@ -60,6 +60,14 @@ class RayGPUExecutor(DistributedGPUExecutor):
60
# Create the parallel GPU workers.
61
self._init_workers_ray(placement_group)
62
63
+ def shutdown(self) -> None:
64
+ if hasattr(self, "forward_dag") and self.forward_dag is not None:
65
+ self.forward_dag.teardown()
66
+ import ray
67
+ for worker in self.workers:
68
+ ray.kill(worker)
69
+ self.forward_dag = None
70
+
71
def _configure_ray_workers_use_nsight(self,
72
ray_remote_kwargs) -> Dict[str, Any]:
73
# If nsight profiling is enabled, we need to set the profiling
@@ -117,7 +125,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
125
logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
126
# Create the workers.
127
driver_ip = get_ip()
120
- logger.info("driver_ip: %s", driver_ip)
128
worker_wrapper_kwargs = self._get_worker_wrapper_args()
129
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
130
if not bundle.get("GPU", 0):
@@ -446,11 +453,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
453
return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
454
455
def __del__(self):
449
- if self.forward_dag is not None:
450
- self.forward_dag.teardown()
451
- import ray
452
- for worker in self.workers:
453
- ray.kill(worker)
456
+ self.shutdown()
457
458
459
class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
@@ -523,8 +526,4 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
526
return await asyncio.gather(*coros)
527
528
def __del__(self):
526
- if self.forward_dag is not None:
527
- self.forward_dag.teardown()
528
- import ray
529
- for worker in self.workers:
530
- ray.kill(worker)
529
+ self.shutdown()