@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d11bf435a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[MISC] Consolidate cleanup() and refactor offline_inference_with_prefix.py (#9510)

Cody Yu committed Oct 18, 2024 at 14:30 UTC d11bf435a0bfdefece204aa6a725e849dc00d8cb
20 files changed +84 -105
examples/offline_inference_with_prefix.py
+12 -7
@@ -1,4 +1,5 @@
1 from vllm import LLM, SamplingParams
2 +from vllm.distributed import cleanup_dist_env_and_memory
3
4 # NOTE: This is just a running example. For benchmarking purpose,
5 # please see benchmarks/benchmark_prefix_caching.py
@@ -28,14 +29,9 @@ generating_prompts = [prefix + prompt for prompt in prompts]
29 # Create a sampling params object.
30 sampling_params = SamplingParams(temperature=0.0)
31
31 -# Create an LLM.
32 -regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3)
32 +# Create an LLM without prefix caching as a baseline.
33 +regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
34
34 -# The second LLM needs to request a higher gpu_memory_utilization because
35 -# the first LLM has already allocated a full 30% of the gpu memory.
36 -prefix_cached_llm = LLM(model="facebook/opt-125m",
37 - enable_prefix_caching=True,
38 - gpu_memory_utilization=0.6)
35 print("Results without `enable_prefix_caching`")
36
37 # Generate texts from the prompts. The output is a list of RequestOutput objects
@@ -52,6 +48,15 @@ for output in outputs:
48
49 print("-" * 80)
50
51 +# Destroy the LLM object and free up the GPU memory.
52 +del regular_llm
53 +cleanup_dist_env_and_memory()
54 +
55 +# Create an LLM with prefix caching enabled.
56 +prefix_cached_llm = LLM(model="facebook/opt-125m",
57 + enable_prefix_caching=True,
58 + gpu_memory_utilization=0.4)
59 +
60 # Warmup so that the shared prompt's KV cache is computed.
61 prefix_cached_llm.generate(generating_prompts[0], sampling_params)
62
tests/async_engine/test_async_llm_engine.py
+2 -2
@@ -12,11 +12,11 @@ import torch
12
13 from vllm import SamplingParams
14 from vllm.config import ParallelConfig
15 +from vllm.distributed import cleanup_dist_env_and_memory
16 from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
17 from vllm.outputs import RequestOutput as RealRequestOutput
18 from vllm.sampling_params import RequestOutputKind
19
19 -from ..conftest import cleanup
20 from ..utils import wait_for_gpu_memory_to_clear
21
22
@@ -157,7 +157,7 @@ async def async_engine():
157 engine.shutdown_background_loop()
158 del engine
159 await asyncio.sleep(0.1)
160 - cleanup()
160 + cleanup_dist_env_and_memory()
161
162
163 @pytest.fixture()
tests/conftest.py
+5 -18
@@ -1,5 +1,3 @@
1 -import contextlib
2 -import gc
1 import json
2 import os
3 import sys
@@ -27,8 +25,7 @@ from vllm.assets.image import ImageAsset
25 from vllm.assets.video import VideoAsset
26 from vllm.config import TaskOption, TokenizerPoolConfig
27 from vllm.connections import global_http_connection
30 -from vllm.distributed import (destroy_distributed_environment,
31 - destroy_model_parallel,
28 +from vllm.distributed import (cleanup_dist_env_and_memory,
29 init_distributed_environment,
30 initialize_model_parallel)
31 from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
@@ -140,17 +137,7 @@ def dist_init():
137 )
138 initialize_model_parallel(1, 1)
139 yield
143 - cleanup()
144 -
145 -
146 -def cleanup():
147 - destroy_model_parallel()
148 - destroy_distributed_environment()
149 - with contextlib.suppress(AssertionError):
150 - torch.distributed.destroy_process_group()
151 - gc.collect()
152 - if not is_cpu():
153 - torch.cuda.empty_cache()
140 + cleanup_dist_env_and_memory()
141
142
143 @pytest.fixture()
@@ -167,7 +154,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
154 def cleanup_fixture(should_do_global_cleanup_after_test: bool):
155 yield
156 if should_do_global_cleanup_after_test:
170 - cleanup()
157 + cleanup_dist_env_and_memory()
158
159
160 @pytest.fixture(autouse=True)
@@ -606,7 +593,7 @@ class HfRunner:
593
594 def __exit__(self, exc_type, exc_value, traceback):
595 del self.model
609 - cleanup()
596 + cleanup_dist_env_and_memory()
597
598
599 @pytest.fixture(scope="session")
@@ -861,7 +848,7 @@ class VllmRunner:
848
849 def __exit__(self, exc_type, exc_value, traceback):
850 del self.model
864 - cleanup()
851 + cleanup_dist_env_and_memory()
852
853
854 @pytest.fixture(scope="session")
tests/core/block/e2e/conftest.py
+2 -3
@@ -3,10 +3,9 @@ from typing import Callable, Iterable, Optional
3 import pytest
4
5 from vllm import LLM
6 +from vllm.distributed import cleanup_dist_env_and_memory
7 from vllm.model_executor.utils import set_random_seed
8
8 -from ....conftest import cleanup
9 -
9
10 @pytest.fixture
11 def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
@@ -37,7 +36,7 @@ def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
36
37 yield llm
38 del llm
40 - cleanup()
39 + cleanup_dist_env_and_memory()
40
41 for llm in generator_inner():
42 yield llm
tests/entrypoints/llm/test_encode.py
+2 -3
@@ -4,8 +4,7 @@ from typing import List
4 import pytest
5
6 from vllm import LLM, EmbeddingRequestOutput, PoolingParams
7 -
8 -from ...conftest import cleanup
7 +from vllm.distributed import cleanup_dist_env_and_memory
8
9 MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
10
@@ -41,7 +40,7 @@ def llm():
40
41 del llm
42
44 - cleanup()
43 + cleanup_dist_env_and_memory()
44
45
46 def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
tests/entrypoints/llm/test_generate.py
+2 -3
@@ -4,8 +4,7 @@ from typing import List
4 import pytest
5
6 from vllm import LLM, RequestOutput, SamplingParams
7 -
8 -from ...conftest import cleanup
7 +from vllm.distributed import cleanup_dist_env_and_memory
8
9 MODEL_NAME = "facebook/opt-125m"
10
@@ -39,7 +38,7 @@ def llm():
38
39 del llm
40
42 - cleanup()
41 + cleanup_dist_env_and_memory()
42
43
44 def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
tests/entrypoints/llm/test_generate_multiple_loras.py
+2 -3
@@ -5,10 +5,9 @@ import pytest
5 from huggingface_hub import snapshot_download
6
7 from vllm import LLM
8 +from vllm.distributed import cleanup_dist_env_and_memory
9 from vllm.lora.request import LoRARequest
10
10 -from ...conftest import cleanup
11 -
11 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
12
13 PROMPTS = [
@@ -39,7 +38,7 @@ def llm():
38
39 del llm
40
42 - cleanup()
41 + cleanup_dist_env_and_memory()
42
43
44 @pytest.fixture(scope="module")
tests/entrypoints/llm/test_guided_generate.py
+2 -3
@@ -5,12 +5,11 @@ import weakref
5 import jsonschema
6 import pytest
7
8 +from vllm.distributed import cleanup_dist_env_and_memory
9 from vllm.entrypoints.llm import LLM
10 from vllm.outputs import RequestOutput
11 from vllm.sampling_params import GuidedDecodingParams, SamplingParams
12
12 -from ...conftest import cleanup
13 -
13 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
15
@@ -23,7 +22,7 @@ def llm():
22 with llm.deprecate_legacy_api():
23 yield weakref.proxy(llm)
24 del llm
26 - cleanup()
25 + cleanup_dist_env_and_memory()
26
27
28 @pytest.mark.skip_global_cleanup
tests/entrypoints/llm/test_lazy_outlines.py
+7 -2
@@ -1,6 +1,7 @@
1 import sys
2
3 from vllm import LLM, SamplingParams
4 +from vllm.distributed import cleanup_dist_env_and_memory
5
6
7 def test_lazy_outlines(sample_regex):
@@ -14,6 +15,7 @@ def test_lazy_outlines(sample_regex):
15 ]
16 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
18 + # Create an LLM without guided decoding as a baseline.
19 llm = LLM(model="facebook/opt-125m",
20 enforce_eager=True,
21 gpu_memory_utilization=0.3)
@@ -26,8 +28,11 @@ def test_lazy_outlines(sample_regex):
28 # make sure outlines is not imported
29 assert 'outlines' not in sys.modules
30
29 - # The second LLM needs to request a higher gpu_memory_utilization because
30 - # the first LLM has already allocated a full 30% of the gpu memory.
31 + # Destroy the LLM object and free up the GPU memory.
32 + del llm
33 + cleanup_dist_env_and_memory()
34 +
35 + # Create an LLM with guided decoding enabled.
36 llm = LLM(model="facebook/opt-125m",
37 enforce_eager=True,
38 guided_decoding_backend="lm-format-enforcer",
tests/entrypoints/offline_mode/test_offline_mode.py
+2 -3
@@ -6,8 +6,7 @@ import weakref
6 import pytest
7
8 from vllm import LLM
9 -
10 -from ...conftest import cleanup
9 +from vllm.distributed import cleanup_dist_env_and_memory
10
11 MODEL_NAME = "facebook/opt-125m"
12
@@ -27,7 +26,7 @@ def llm():
26
27 del llm
28
30 - cleanup()
29 + cleanup_dist_env_and_memory()
30
31
32 @pytest.mark.skip_global_cleanup
tests/lora/conftest.py
+6 -20
@@ -1,20 +1,16 @@
1 -import contextlib
2 -import gc
1 import tempfile
2 from collections import OrderedDict
3 from typing import Dict, List, TypedDict
4 from unittest.mock import MagicMock, patch
5
6 import pytest
9 -import ray
7 import torch
8 import torch.nn as nn
9 from huggingface_hub import snapshot_download
10
11 import vllm
12 from vllm.config import LoRAConfig
16 -from vllm.distributed import (destroy_distributed_environment,
17 - destroy_model_parallel,
13 +from vllm.distributed import (cleanup_dist_env_and_memory,
14 init_distributed_environment,
15 initialize_model_parallel)
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -48,16 +44,6 @@ LONG_LORA_INFOS: List[ContextIDInfo] = [{
44 }]
45
46
51 -def cleanup():
52 - destroy_model_parallel()
53 - destroy_distributed_environment()
54 - with contextlib.suppress(AssertionError):
55 - torch.distributed.destroy_process_group()
56 - gc.collect()
57 - torch.cuda.empty_cache()
58 - ray.shutdown()
59 -
60 -
47 @pytest.fixture()
48 def should_do_global_cleanup_after_test(request) -> bool:
49 """Allow subdirectories to skip global cleanup by overriding this fixture.
@@ -72,7 +58,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
58 def cleanup_fixture(should_do_global_cleanup_after_test: bool):
59 yield
60 if should_do_global_cleanup_after_test:
75 - cleanup()
61 + cleanup_dist_env_and_memory(shutdown_ray=True)
62
63
64 @pytest.fixture
@@ -87,7 +73,7 @@ def dist_init():
73 )
74 initialize_model_parallel(1, 1)
75 yield
90 - cleanup()
76 + cleanup_dist_env_and_memory(shutdown_ray=True)
77
78
79 @pytest.fixture
@@ -238,7 +224,7 @@ def long_context_lora_files_32k():
224 def long_context_infos(long_context_lora_files_16k_1,
225 long_context_lora_files_16k_2,
226 long_context_lora_files_32k):
241 - cleanup()
227 + cleanup_dist_env_and_memory(shutdown_ray=True)
228 infos: Dict[int, ContextInfo] = {}
229 for lora_checkpoint_info in LONG_LORA_INFOS:
230 lora_id = lora_checkpoint_info["lora_id"]
@@ -259,7 +245,7 @@ def long_context_infos(long_context_lora_files_16k_1,
245
246 @pytest.fixture
247 def llama_2_7b_engine_extra_embeddings():
262 - cleanup()
248 + cleanup_dist_env_and_memory(shutdown_ray=True)
249 get_model_old = get_model
250
251 def get_model_patched(*, model_config, device_config, **kwargs):
@@ -272,7 +258,7 @@ def llama_2_7b_engine_extra_embeddings():
258 engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
259 yield engine.llm_engine
260 del engine
275 - cleanup()
261 + cleanup_dist_env_and_memory(shutdown_ray=True)
262
263
264 @pytest.fixture
tests/lora/test_baichuan.py
+4 -5
@@ -3,10 +3,9 @@ from typing import List
3 import pytest
4
5 import vllm
6 +from vllm.distributed import cleanup_dist_env_and_memory
7 from vllm.lora.request import LoRARequest
8
8 -from .conftest import cleanup
9 -
9 MODEL_PATH = "baichuan-inc/Baichuan-7B"
10
11 PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
@@ -80,7 +79,7 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
79 output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
80
81 del llm_tp1
83 - cleanup()
82 + cleanup_dist_env_and_memory()
83
84 llm_tp2 = vllm.LLM(MODEL_PATH,
85 enable_lora=True,
@@ -93,7 +92,7 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
92 output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
93
94 del llm_tp2
96 - cleanup()
95 + cleanup_dist_env_and_memory()
96
97 assert output_tp1 == output_tp2
98
@@ -108,6 +107,6 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
107 output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
108
109 del llm_tp4
111 - cleanup()
110 + cleanup_dist_env_and_memory()
111
112 assert output_tp1 == output_tp4
tests/lora/test_llama.py
+4 -5
@@ -4,10 +4,9 @@ import pytest
4 import ray
5
6 import vllm
7 +from vllm.distributed import cleanup_dist_env_and_memory
8 from vllm.lora.request import LoRARequest
9
9 -from .conftest import cleanup
10 -
10 MODEL_PATH = "meta-llama/Llama-2-7b-hf"
11
12
@@ -93,7 +92,7 @@ def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
92 output_tp1 = do_sample(llm_tp1, sql_lora_files, lora_id=1)
93
94 del llm_tp1
96 - cleanup()
95 + cleanup_dist_env_and_memory()
96
97 llm_tp2 = vllm.LLM(MODEL_PATH,
98 enable_lora=True,
@@ -103,7 +102,7 @@ def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
102 output_tp2 = do_sample(llm_tp2, sql_lora_files, lora_id=1)
103
104 del llm_tp2
106 - cleanup()
105 + cleanup_dist_env_and_memory()
106
107 assert output_tp1 == output_tp2
108
@@ -115,7 +114,7 @@ def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
114 output_tp4 = do_sample(llm_tp4, sql_lora_files, lora_id=1)
115
116 del llm_tp4
118 - cleanup()
117 + cleanup_dist_env_and_memory()
118
119 assert output_tp1 == output_tp4
120
tests/lora/test_quant_model.py
+4 -5
@@ -6,11 +6,10 @@ from typing import List
6 import pytest
7
8 import vllm
9 +from vllm.distributed import cleanup_dist_env_and_memory
10 from vllm.lora.request import LoRARequest
11 from vllm.utils import is_hip
12
12 -from .conftest import cleanup
13 -
13
14 @dataclass
15 class ModelWithQuantization:
@@ -160,7 +159,7 @@ def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
159 print("removing lora")
160
161 del llm
163 - cleanup()
162 + cleanup_dist_env_and_memory()
163
164
165 @pytest.mark.parametrize("model", MODELS)
@@ -181,7 +180,7 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
180 output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
181
182 del llm_tp1
184 - cleanup()
183 + cleanup_dist_env_and_memory()
184
185 llm_tp2 = vllm.LLM(
186 model=model.model_path,
@@ -194,6 +193,6 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
193 output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
194
195 del llm_tp2
197 - cleanup()
196 + cleanup_dist_env_and_memory()
197
198 assert output_tp1 == output_tp2
tests/metrics/test_metrics.py
+2 -3
@@ -6,13 +6,12 @@ import ray
6 from prometheus_client import REGISTRY
7
8 from vllm import EngineArgs, LLMEngine
9 +from vllm.distributed import cleanup_dist_env_and_memory
10 from vllm.engine.arg_utils import AsyncEngineArgs
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.engine.metrics import RayPrometheusStatLogger
13 from vllm.sampling_params import SamplingParams
14
14 -from ..conftest import cleanup
15 -
15 MODELS = [
16 "facebook/opt-125m",
17 ]
@@ -307,7 +306,7 @@ def test_metric_spec_decode_interval(
306
307 finally:
308 del engine
310 - cleanup()
309 + cleanup_dist_env_and_memory()
310
311
312 def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
tests/models/decoder_only/vision_language/test_intern_vit.py
+4 -3
@@ -6,7 +6,7 @@ import torch.nn as nn
6 from huggingface_hub import snapshot_download
7 from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8
9 -from ....conftest import _ImageAssets, cleanup
9 +from ....conftest import _ImageAssets
10
11 # we use snapshot_download to prevent conflicts between
12 # dynamic_module and trust_remote_code for hf_runner
@@ -45,12 +45,13 @@ def run_intern_vit_test(
45 for pixel_value in pixel_values
46 ]
47
48 + from vllm.distributed import cleanup_dist_env_and_memory
49 from vllm.model_executor.models.intern_vit import InternVisionModel
50 vllm_model = InternVisionModel(config)
51 vllm_model.load_weights(hf_model.state_dict().items())
52
53 del hf_model
53 - cleanup()
54 + cleanup_dist_env_and_memory()
55
56 vllm_model = vllm_model.to("cuda", dtype)
57 vllm_outputs_per_image = [
@@ -58,7 +59,7 @@ def run_intern_vit_test(
59 for pixel_value in pixel_values
60 ]
61 del vllm_model
61 - cleanup()
62 + cleanup_dist_env_and_memory()
63
64 cos_similar = nn.CosineSimilarity(dim=-1)
65 for vllm_output, hf_output in zip(vllm_outputs_per_image,
tests/prefix_caching/test_disable_sliding_window.py
+3 -3
@@ -4,8 +4,8 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4 """
5 import pytest
6
7 -from tests.conftest import cleanup
7 from vllm import LLM
8 +from vllm.distributed import cleanup_dist_env_and_memory
9
10 MODEL_LEN_LEN = [
11 # Example models with sliding window.
@@ -31,7 +31,7 @@ def test_disable_sliding_window(model_len_len, ):
31 model_config.max_model_len)
32
33 del vllm_disabled_model
34 - cleanup()
34 + cleanup_dist_env_and_memory()
35
36 vllm_enabled_model = LLM(model, disable_sliding_window=False)
37 vllm_enabled_model.generate("Hi my name is")
@@ -41,4 +41,4 @@ def test_disable_sliding_window(model_len_len, ):
41 model_config.max_model_len)
42
43 del vllm_enabled_model
44 - cleanup()
44 + cleanup_dist_env_and_memory()
tests/spec_decode/e2e/conftest.py
+2 -2
@@ -4,10 +4,10 @@ from typing import List, Optional, Sequence, Tuple, Union
4 import pytest
5
6 from vllm import LLM, SamplingParams
7 +from vllm.distributed import cleanup_dist_env_and_memory
8 from vllm.model_executor.utils import set_random_seed
9 from vllm.sequence import PromptLogprobs, SampleLogprobs
10
10 -from ...conftest import cleanup
11 from ...models.utils import (TokensTextLogprobs,
12 TokensTextLogprobsPromptLogprobs,
13 check_logprobs_close, check_outputs_equal)
@@ -44,7 +44,7 @@ def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
44 yield llm
45
46 del llm
47 - cleanup()
47 + cleanup_dist_env_and_memory()
48
49 return generate
50
tests/tensorizer_loader/conftest.py
+2 -11
@@ -1,27 +1,18 @@
1 -import contextlib
1 import functools
2 import gc
3 from typing import Callable, TypeVar
4
5 import pytest
7 -import ray
6 import torch
7 from typing_extensions import ParamSpec
8
11 -from vllm.distributed import (destroy_distributed_environment,
12 - destroy_model_parallel)
9 +from vllm.distributed import cleanup_dist_env_and_memory
10 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
11
12
13 @pytest.fixture(autouse=True)
14 def cleanup():
18 - destroy_model_parallel()
19 - destroy_distributed_environment()
20 - with contextlib.suppress(AssertionError):
21 - torch.distributed.destroy_process_group()
22 - ray.shutdown()
23 - gc.collect()
24 - torch.cuda.empty_cache()
15 + cleanup_dist_env_and_memory(shutdown_ray=True)
16
17
18 _P = ParamSpec("_P")
vllm/distributed/parallel_state.py
+15 -1
@@ -20,6 +20,7 @@ If you only need to use the distributed environment without model/pipeline
20 steps.
21 """
22 import contextlib
23 +import gc
24 import pickle
25 import weakref
26 from collections import namedtuple
@@ -36,7 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37 import vllm.envs as envs
38 from vllm.logger import init_logger
39 from vllm.platforms import current_platform
39 -from vllm.utils import supports_custom_op
40 +from vllm.utils import is_cpu, supports_custom_op
41
42
43 @dataclass
@@ -1129,6 +1130,19 @@ def destroy_distributed_environment():
1130 torch.distributed.destroy_process_group()
1131
1132
1133 +def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1134 + destroy_model_parallel()
1135 + destroy_distributed_environment()
1136 + with contextlib.suppress(AssertionError):
1137 + torch.distributed.destroy_process_group()
1138 + if shutdown_ray:
1139 + import ray # Lazy import Ray
1140 + ray.shutdown()
1141 + gc.collect()
1142 + if not is_cpu():
1143 + torch.cuda.empty_cache()
1144 +
1145 +
1146 def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1147 """
1148 This is a collective operation that returns if each rank is in the same node