A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[MISC] Consolidate cleanup() and refactor offline_inference_with_prefix.py (#9510)
Cody Yu committed
Oct 18, 2024 at 14:30 UTC
d11bf435a0bfdefece204aa6a725e849dc00d8cb
20 files changed
+84
-105
examples/offline_inference_with_prefix.py
+12
-7
@@ -1,4 +1,5 @@
1
from vllm import LLM, SamplingParams
2
+from vllm.distributed import cleanup_dist_env_and_memory
3
4
# NOTE: This is just a running example. For benchmarking purpose,
5
# please see benchmarks/benchmark_prefix_caching.py
@@ -28,14 +29,9 @@ generating_prompts = [prefix + prompt for prompt in prompts]
29
# Create a sampling params object.
30
sampling_params = SamplingParams(temperature=0.0)
31
31
-# Create an LLM.
32
-regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.3)
32
+# Create an LLM without prefix caching as a baseline.
33
+regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
34
34
-# The second LLM needs to request a higher gpu_memory_utilization because
35
-# the first LLM has already allocated a full 30% of the gpu memory.
36
-prefix_cached_llm = LLM(model="facebook/opt-125m",
37
- enable_prefix_caching=True,
38
- gpu_memory_utilization=0.6)
35
print("Results without `enable_prefix_caching`")
36
37
# Generate texts from the prompts. The output is a list of RequestOutput objects
@@ -52,6 +48,15 @@ for output in outputs:
48
49
print("-" * 80)
50
51
+# Destroy the LLM object and free up the GPU memory.
52
+del regular_llm
53
+cleanup_dist_env_and_memory()
54
+
55
+# Create an LLM with prefix caching enabled.
56
+prefix_cached_llm = LLM(model="facebook/opt-125m",
57
+ enable_prefix_caching=True,
58
+ gpu_memory_utilization=0.4)
59
+
60
# Warmup so that the shared prompt's KV cache is computed.
61
prefix_cached_llm.generate(generating_prompts[0], sampling_params)
62
tests/async_engine/test_async_llm_engine.py
+2
-2
@@ -12,11 +12,11 @@ import torch
12
13
from vllm import SamplingParams
14
from vllm.config import ParallelConfig
15
+from vllm.distributed import cleanup_dist_env_and_memory
16
from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
17
from vllm.outputs import RequestOutput as RealRequestOutput
18
from vllm.sampling_params import RequestOutputKind
19
19
-from ..conftest import cleanup
20
from ..utils import wait_for_gpu_memory_to_clear
21
22
@@ -157,7 +157,7 @@ async def async_engine():
157
engine.shutdown_background_loop()
158
del engine
159
await asyncio.sleep(0.1)
160
- cleanup()
160
+ cleanup_dist_env_and_memory()
161
162
163
@pytest.fixture()
tests/conftest.py
+5
-18
@@ -1,5 +1,3 @@
1
-import contextlib
2
-import gc
1
import json
2
import os
3
import sys
@@ -27,8 +25,7 @@ from vllm.assets.image import ImageAsset
25
from vllm.assets.video import VideoAsset
26
from vllm.config import TaskOption, TokenizerPoolConfig
27
from vllm.connections import global_http_connection
30
-from vllm.distributed import (destroy_distributed_environment,
31
- destroy_model_parallel,
28
+from vllm.distributed import (cleanup_dist_env_and_memory,
29
init_distributed_environment,
30
initialize_model_parallel)
31
from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
@@ -140,17 +137,7 @@ def dist_init():
137
)
138
initialize_model_parallel(1, 1)
139
yield
143
- cleanup()
144
-
145
-
146
-def cleanup():
147
- destroy_model_parallel()
148
- destroy_distributed_environment()
149
- with contextlib.suppress(AssertionError):
150
- torch.distributed.destroy_process_group()
151
- gc.collect()
152
- if not is_cpu():
153
- torch.cuda.empty_cache()
140
+ cleanup_dist_env_and_memory()
141
142
143
@pytest.fixture()
@@ -167,7 +154,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
154
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
155
yield
156
if should_do_global_cleanup_after_test:
170
- cleanup()
157
+ cleanup_dist_env_and_memory()
158
159
160
@pytest.fixture(autouse=True)
@@ -606,7 +593,7 @@ class HfRunner:
593
594
def __exit__(self, exc_type, exc_value, traceback):
595
del self.model
609
- cleanup()
596
+ cleanup_dist_env_and_memory()
597
598
599
@pytest.fixture(scope="session")
@@ -861,7 +848,7 @@ class VllmRunner:
848
849
def __exit__(self, exc_type, exc_value, traceback):
850
del self.model
864
- cleanup()
851
+ cleanup_dist_env_and_memory()
852
853
854
@pytest.fixture(scope="session")
tests/core/block/e2e/conftest.py
+2
-3
@@ -3,10 +3,9 @@ from typing import Callable, Iterable, Optional
3
import pytest
4
5
from vllm import LLM
6
+from vllm.distributed import cleanup_dist_env_and_memory
7
from vllm.model_executor.utils import set_random_seed
8
8
-from ....conftest import cleanup
9
-
9
10
@pytest.fixture
11
def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
@@ -37,7 +36,7 @@ def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
36
37
yield llm
38
del llm
40
- cleanup()
39
+ cleanup_dist_env_and_memory()
40
41
for llm in generator_inner():
42
yield llm
tests/entrypoints/llm/test_encode.py
+2
-3
@@ -4,8 +4,7 @@ from typing import List
4
import pytest
5
6
from vllm import LLM, EmbeddingRequestOutput, PoolingParams
7
-
8
-from ...conftest import cleanup
7
+from vllm.distributed import cleanup_dist_env_and_memory
8
9
MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
10
@@ -41,7 +40,7 @@ def llm():
40
41
del llm
42
44
- cleanup()
43
+ cleanup_dist_env_and_memory()
44
45
46
def assert_outputs_equal(o1: List[EmbeddingRequestOutput],
tests/entrypoints/llm/test_generate.py
+2
-3
@@ -4,8 +4,7 @@ from typing import List
4
import pytest
5
6
from vllm import LLM, RequestOutput, SamplingParams
7
-
8
-from ...conftest import cleanup
7
+from vllm.distributed import cleanup_dist_env_and_memory
8
9
MODEL_NAME = "facebook/opt-125m"
10
@@ -39,7 +38,7 @@ def llm():
38
39
del llm
40
42
- cleanup()
41
+ cleanup_dist_env_and_memory()
42
43
44
def assert_outputs_equal(o1: List[RequestOutput], o2: List[RequestOutput]):
tests/entrypoints/llm/test_generate_multiple_loras.py
+2
-3
@@ -5,10 +5,9 @@ import pytest
5
from huggingface_hub import snapshot_download
6
7
from vllm import LLM
8
+from vllm.distributed import cleanup_dist_env_and_memory
9
from vllm.lora.request import LoRARequest
10
10
-from ...conftest import cleanup
11
-
11
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
12
13
PROMPTS = [
@@ -39,7 +38,7 @@ def llm():
38
39
del llm
40
42
- cleanup()
41
+ cleanup_dist_env_and_memory()
42
43
44
@pytest.fixture(scope="module")
tests/entrypoints/llm/test_guided_generate.py
+2
-3
@@ -5,12 +5,11 @@ import weakref
5
import jsonschema
6
import pytest
7
8
+from vllm.distributed import cleanup_dist_env_and_memory
9
from vllm.entrypoints.llm import LLM
10
from vllm.outputs import RequestOutput
11
from vllm.sampling_params import GuidedDecodingParams, SamplingParams
12
12
-from ...conftest import cleanup
13
-
13
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
14
15
@@ -23,7 +22,7 @@ def llm():
22
with llm.deprecate_legacy_api():
23
yield weakref.proxy(llm)
24
del llm
26
- cleanup()
25
+ cleanup_dist_env_and_memory()
26
27
28
@pytest.mark.skip_global_cleanup
tests/entrypoints/llm/test_lazy_outlines.py
+7
-2
@@ -1,6 +1,7 @@
1
import sys
2
3
from vllm import LLM, SamplingParams
4
+from vllm.distributed import cleanup_dist_env_and_memory
5
6
7
def test_lazy_outlines(sample_regex):
@@ -14,6 +15,7 @@ def test_lazy_outlines(sample_regex):
15
]
16
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
17
18
+ # Create an LLM without guided decoding as a baseline.
19
llm = LLM(model="facebook/opt-125m",
20
enforce_eager=True,
21
gpu_memory_utilization=0.3)
@@ -26,8 +28,11 @@ def test_lazy_outlines(sample_regex):
28
# make sure outlines is not imported
29
assert 'outlines' not in sys.modules
30
29
- # The second LLM needs to request a higher gpu_memory_utilization because
30
- # the first LLM has already allocated a full 30% of the gpu memory.
31
+ # Destroy the LLM object and free up the GPU memory.
32
+ del llm
33
+ cleanup_dist_env_and_memory()
34
+
35
+ # Create an LLM with guided decoding enabled.
36
llm = LLM(model="facebook/opt-125m",
37
enforce_eager=True,
38
guided_decoding_backend="lm-format-enforcer",
tests/entrypoints/offline_mode/test_offline_mode.py
+2
-3
@@ -6,8 +6,7 @@ import weakref
6
import pytest
7
8
from vllm import LLM
9
-
10
-from ...conftest import cleanup
9
+from vllm.distributed import cleanup_dist_env_and_memory
10
11
MODEL_NAME = "facebook/opt-125m"
12
@@ -27,7 +26,7 @@ def llm():
26
27
del llm
28
30
- cleanup()
29
+ cleanup_dist_env_and_memory()
30
31
32
@pytest.mark.skip_global_cleanup
tests/lora/conftest.py
+6
-20
@@ -1,20 +1,16 @@
1
-import contextlib
2
-import gc
1
import tempfile
2
from collections import OrderedDict
3
from typing import Dict, List, TypedDict
4
from unittest.mock import MagicMock, patch
5
6
import pytest
9
-import ray
7
import torch
8
import torch.nn as nn
9
from huggingface_hub import snapshot_download
10
11
import vllm
12
from vllm.config import LoRAConfig
16
-from vllm.distributed import (destroy_distributed_environment,
17
- destroy_model_parallel,
13
+from vllm.distributed import (cleanup_dist_env_and_memory,
14
init_distributed_environment,
15
initialize_model_parallel)
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -48,16 +44,6 @@ LONG_LORA_INFOS: List[ContextIDInfo] = [{
44
}]
45
46
51
-def cleanup():
52
- destroy_model_parallel()
53
- destroy_distributed_environment()
54
- with contextlib.suppress(AssertionError):
55
- torch.distributed.destroy_process_group()
56
- gc.collect()
57
- torch.cuda.empty_cache()
58
- ray.shutdown()
59
-
60
-
47
@pytest.fixture()
48
def should_do_global_cleanup_after_test(request) -> bool:
49
"""Allow subdirectories to skip global cleanup by overriding this fixture.
@@ -72,7 +58,7 @@ def should_do_global_cleanup_after_test(request) -> bool:
58
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
59
yield
60
if should_do_global_cleanup_after_test:
75
- cleanup()
61
+ cleanup_dist_env_and_memory(shutdown_ray=True)
62
63
64
@pytest.fixture
@@ -87,7 +73,7 @@ def dist_init():
73
)
74
initialize_model_parallel(1, 1)
75
yield
90
- cleanup()
76
+ cleanup_dist_env_and_memory(shutdown_ray=True)
77
78
79
@pytest.fixture
@@ -238,7 +224,7 @@ def long_context_lora_files_32k():
224
def long_context_infos(long_context_lora_files_16k_1,
225
long_context_lora_files_16k_2,
226
long_context_lora_files_32k):
241
- cleanup()
227
+ cleanup_dist_env_and_memory(shutdown_ray=True)
228
infos: Dict[int, ContextInfo] = {}
229
for lora_checkpoint_info in LONG_LORA_INFOS:
230
lora_id = lora_checkpoint_info["lora_id"]
@@ -259,7 +245,7 @@ def long_context_infos(long_context_lora_files_16k_1,
245
246
@pytest.fixture
247
def llama_2_7b_engine_extra_embeddings():
262
- cleanup()
248
+ cleanup_dist_env_and_memory(shutdown_ray=True)
249
get_model_old = get_model
250
251
def get_model_patched(*, model_config, device_config, **kwargs):
@@ -272,7 +258,7 @@ def llama_2_7b_engine_extra_embeddings():
258
engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
259
yield engine.llm_engine
260
del engine
275
- cleanup()
261
+ cleanup_dist_env_and_memory(shutdown_ray=True)
262
263
264
@pytest.fixture
tests/lora/test_baichuan.py
+4
-5
@@ -3,10 +3,9 @@ from typing import List
3
import pytest
4
5
import vllm
6
+from vllm.distributed import cleanup_dist_env_and_memory
7
from vllm.lora.request import LoRARequest
8
8
-from .conftest import cleanup
9
-
9
MODEL_PATH = "baichuan-inc/Baichuan-7B"
10
11
PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
@@ -80,7 +79,7 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
79
output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
80
81
del llm_tp1
83
- cleanup()
82
+ cleanup_dist_env_and_memory()
83
84
llm_tp2 = vllm.LLM(MODEL_PATH,
85
enable_lora=True,
@@ -93,7 +92,7 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
92
output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
93
94
del llm_tp2
96
- cleanup()
95
+ cleanup_dist_env_and_memory()
96
97
assert output_tp1 == output_tp2
98
@@ -108,6 +107,6 @@ def test_baichuan_tensor_parallel_equality(baichuan_lora_files,
107
output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
108
109
del llm_tp4
111
- cleanup()
110
+ cleanup_dist_env_and_memory()
111
112
assert output_tp1 == output_tp4
tests/lora/test_llama.py
+4
-5
@@ -4,10 +4,9 @@ import pytest
4
import ray
5
6
import vllm
7
+from vllm.distributed import cleanup_dist_env_and_memory
8
from vllm.lora.request import LoRARequest
9
9
-from .conftest import cleanup
10
-
10
MODEL_PATH = "meta-llama/Llama-2-7b-hf"
11
12
@@ -93,7 +92,7 @@ def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
92
output_tp1 = do_sample(llm_tp1, sql_lora_files, lora_id=1)
93
94
del llm_tp1
96
- cleanup()
95
+ cleanup_dist_env_and_memory()
96
97
llm_tp2 = vllm.LLM(MODEL_PATH,
98
enable_lora=True,
@@ -103,7 +102,7 @@ def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
102
output_tp2 = do_sample(llm_tp2, sql_lora_files, lora_id=1)
103
104
del llm_tp2
106
- cleanup()
105
+ cleanup_dist_env_and_memory()
106
107
assert output_tp1 == output_tp2
108
@@ -115,7 +114,7 @@ def test_llama_tensor_parallel_equality(sql_lora_files, num_gpus_available):
114
output_tp4 = do_sample(llm_tp4, sql_lora_files, lora_id=1)
115
116
del llm_tp4
118
- cleanup()
117
+ cleanup_dist_env_and_memory()
118
119
assert output_tp1 == output_tp4
120
tests/lora/test_quant_model.py
+4
-5
@@ -6,11 +6,10 @@ from typing import List
6
import pytest
7
8
import vllm
9
+from vllm.distributed import cleanup_dist_env_and_memory
10
from vllm.lora.request import LoRARequest
11
from vllm.utils import is_hip
12
12
-from .conftest import cleanup
13
-
13
14
@dataclass
15
class ModelWithQuantization:
@@ -160,7 +159,7 @@ def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,
159
print("removing lora")
160
161
del llm
163
- cleanup()
162
+ cleanup_dist_env_and_memory()
163
164
165
@pytest.mark.parametrize("model", MODELS)
@@ -181,7 +180,7 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
180
output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
181
182
del llm_tp1
184
- cleanup()
183
+ cleanup_dist_env_and_memory()
184
185
llm_tp2 = vllm.LLM(
186
model=model.model_path,
@@ -194,6 +193,6 @@ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available,
193
output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
194
195
del llm_tp2
197
- cleanup()
196
+ cleanup_dist_env_and_memory()
197
198
assert output_tp1 == output_tp2
tests/metrics/test_metrics.py
+2
-3
@@ -6,13 +6,12 @@ import ray
6
from prometheus_client import REGISTRY
7
8
from vllm import EngineArgs, LLMEngine
9
+from vllm.distributed import cleanup_dist_env_and_memory
10
from vllm.engine.arg_utils import AsyncEngineArgs
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.engine.metrics import RayPrometheusStatLogger
13
from vllm.sampling_params import SamplingParams
14
14
-from ..conftest import cleanup
15
-
15
MODELS = [
16
"facebook/opt-125m",
17
]
@@ -307,7 +306,7 @@ def test_metric_spec_decode_interval(
306
307
finally:
308
del engine
310
- cleanup()
309
+ cleanup_dist_env_and_memory()
310
311
312
def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
tests/models/decoder_only/vision_language/test_intern_vit.py
+4
-3
@@ -6,7 +6,7 @@ import torch.nn as nn
6
from huggingface_hub import snapshot_download
7
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8
9
-from ....conftest import _ImageAssets, cleanup
9
+from ....conftest import _ImageAssets
10
11
# we use snapshot_download to prevent conflicts between
12
# dynamic_module and trust_remote_code for hf_runner
@@ -45,12 +45,13 @@ def run_intern_vit_test(
45
for pixel_value in pixel_values
46
]
47
48
+ from vllm.distributed import cleanup_dist_env_and_memory
49
from vllm.model_executor.models.intern_vit import InternVisionModel
50
vllm_model = InternVisionModel(config)
51
vllm_model.load_weights(hf_model.state_dict().items())
52
53
del hf_model
53
- cleanup()
54
+ cleanup_dist_env_and_memory()
55
56
vllm_model = vllm_model.to("cuda", dtype)
57
vllm_outputs_per_image = [
@@ -58,7 +59,7 @@ def run_intern_vit_test(
59
for pixel_value in pixel_values
60
]
61
del vllm_model
61
- cleanup()
62
+ cleanup_dist_env_and_memory()
63
64
cos_similar = nn.CosineSimilarity(dim=-1)
65
for vllm_output, hf_output in zip(vllm_outputs_per_image,
tests/prefix_caching/test_disable_sliding_window.py
+3
-3
@@ -4,8 +4,8 @@ Run `pytest tests/prefix_caching/test_prefix_caching.py`.
4
"""
5
import pytest
6
7
-from tests.conftest import cleanup
7
from vllm import LLM
8
+from vllm.distributed import cleanup_dist_env_and_memory
9
10
MODEL_LEN_LEN = [
11
# Example models with sliding window.
@@ -31,7 +31,7 @@ def test_disable_sliding_window(model_len_len, ):
31
model_config.max_model_len)
32
33
del vllm_disabled_model
34
- cleanup()
34
+ cleanup_dist_env_and_memory()
35
36
vllm_enabled_model = LLM(model, disable_sliding_window=False)
37
vllm_enabled_model.generate("Hi my name is")
@@ -41,4 +41,4 @@ def test_disable_sliding_window(model_len_len, ):
41
model_config.max_model_len)
42
43
del vllm_enabled_model
44
- cleanup()
44
+ cleanup_dist_env_and_memory()
tests/spec_decode/e2e/conftest.py
+2
-2
@@ -4,10 +4,10 @@ from typing import List, Optional, Sequence, Tuple, Union
4
import pytest
5
6
from vllm import LLM, SamplingParams
7
+from vllm.distributed import cleanup_dist_env_and_memory
8
from vllm.model_executor.utils import set_random_seed
9
from vllm.sequence import PromptLogprobs, SampleLogprobs
10
10
-from ...conftest import cleanup
11
from ...models.utils import (TokensTextLogprobs,
12
TokensTextLogprobsPromptLogprobs,
13
check_logprobs_close, check_outputs_equal)
@@ -44,7 +44,7 @@ def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
44
yield llm
45
46
del llm
47
- cleanup()
47
+ cleanup_dist_env_and_memory()
48
49
return generate
50
tests/tensorizer_loader/conftest.py
+2
-11
@@ -1,27 +1,18 @@
1
-import contextlib
1
import functools
2
import gc
3
from typing import Callable, TypeVar
4
5
import pytest
7
-import ray
6
import torch
7
from typing_extensions import ParamSpec
8
11
-from vllm.distributed import (destroy_distributed_environment,
12
- destroy_model_parallel)
9
+from vllm.distributed import cleanup_dist_env_and_memory
10
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
11
12
13
@pytest.fixture(autouse=True)
14
def cleanup():
18
- destroy_model_parallel()
19
- destroy_distributed_environment()
20
- with contextlib.suppress(AssertionError):
21
- torch.distributed.destroy_process_group()
22
- ray.shutdown()
23
- gc.collect()
24
- torch.cuda.empty_cache()
15
+ cleanup_dist_env_and_memory(shutdown_ray=True)
16
17
18
_P = ParamSpec("_P")
vllm/distributed/parallel_state.py
+15
-1
@@ -20,6 +20,7 @@ If you only need to use the distributed environment without model/pipeline
20
steps.
21
"""
22
import contextlib
23
+import gc
24
import pickle
25
import weakref
26
from collections import namedtuple
@@ -36,7 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37
import vllm.envs as envs
38
from vllm.logger import init_logger
39
from vllm.platforms import current_platform
39
-from vllm.utils import supports_custom_op
40
+from vllm.utils import is_cpu, supports_custom_op
41
42
43
@dataclass
@@ -1129,6 +1130,19 @@ def destroy_distributed_environment():
1130
torch.distributed.destroy_process_group()
1131
1132
1133
+def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1134
+ destroy_model_parallel()
1135
+ destroy_distributed_environment()
1136
+ with contextlib.suppress(AssertionError):
1137
+ torch.distributed.destroy_process_group()
1138
+ if shutdown_ray:
1139
+ import ray # Lazy import Ray
1140
+ ray.shutdown()
1141
+ gc.collect()
1142
+ if not is_cpu():
1143
+ torch.cuda.empty_cache()
1144
+
1145
+
1146
def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]:
1147
"""
1148
This is a collective operation that returns if each rank is in the same node