Stabilize GPU memory teardown between ROCm CI tests (#49242)
Signed-off-by: aarushjain29 <Aarushi.Jain2@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
Aarushi Jain committed
Jul 24, 2026 at 23:21 UTC
0ba2aa35a81dcc3246b26291368b53fa2389c7d7
3 files changed
+126
-93
tests/conftest.py
+26
-9
@@ -1728,33 +1728,50 @@ def disable_deepgemm_ue8m0(monkeypatch):
1728
is_deep_gemm_e8m0_used.cache_clear()
1729
1730
1731
+def _should_clean_gpu_memory_between_tests() -> bool:
1732
+ setting = os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY")
1733
+ if setting == "1":
1734
+ return True
1735
+ if setting == "0":
1736
+ return False
1737
+ # ROCm reclaims VRAM lazily; default to waiting between tests on ROCm CI.
1738
+ return current_platform.is_rocm()
1739
+
1740
+
1741
@pytest.fixture(autouse=True)
1742
def clean_gpu_memory_between_tests():
1733
- if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1":
1743
+ if not _should_clean_gpu_memory_between_tests():
1744
yield
1745
return
1746
1737
- # Wait for GPU memory to be cleared before starting the test
1747
import gc
1748
1740
- from tests.utils import wait_for_gpu_memory_to_clear
1749
+ from tests.utils import wait_for_gpu_memory_to_clear, wait_for_rocm_memory_to_settle
1750
1751
num_gpus = torch.accelerator.device_count()
1743
- if num_gpus > 0:
1752
+
1753
+ def _wait_for_settled_gpu_memory() -> None:
1754
+ if num_gpus <= 0:
1755
+ return
1756
try:
1745
- wait_for_gpu_memory_to_clear(
1746
- devices=list(range(num_gpus)),
1747
- threshold_ratio=0.1,
1748
- )
1757
+ if current_platform.is_rocm():
1758
+ wait_for_rocm_memory_to_settle()
1759
+ else:
1760
+ wait_for_gpu_memory_to_clear(
1761
+ devices=list(range(num_gpus)),
1762
+ threshold_ratio=0.1,
1763
+ )
1764
except ValueError as e:
1765
logger.info("Failed to clean GPU memory: %s", e)
1766
1767
+ _wait_for_settled_gpu_memory()
1768
+
1769
yield
1770
1754
- # Clean up GPU memory after the test
1771
if torch.cuda.is_available():
1772
torch.accelerator.empty_cache()
1773
gc.collect()
1774
+ _wait_for_settled_gpu_memory()
1775
1776
1777
@pytest.fixture
tests/lora/test_qwenvl.py
+87
-81
@@ -7,6 +7,7 @@ from packaging.version import Version
7
from transformers import __version__ as TRANSFORMERS_VERSION
8
9
import vllm
10
+from tests.conftest import VllmRunner
11
from vllm.assets.image import ImageAsset
12
from vllm.lora.request import LoRARequest
13
from vllm.platforms import current_platform
@@ -56,24 +57,29 @@ class Qwen2VLTester:
57
58
def __init__(self, config: TestConfig):
59
self.config = config
59
- self.llm = self._initialize_llm()
60
-
61
- def _initialize_llm(self) -> vllm.LLM:
62
- """Initialize the LLM with given configuration"""
63
- return vllm.LLM(
64
- model=self.config.model_path,
65
- max_num_seqs=self.config.max_num_seqs,
60
+ self._runner = VllmRunner(
61
+ model_name=config.model_path,
62
+ max_num_seqs=config.max_num_seqs,
63
enable_lora=True,
67
- max_loras=self.config.max_loras,
68
- max_lora_rank=self.config.max_lora_rank,
69
- enable_tower_connector_lora=self.config.enable_tower_connector_lora,
70
- trust_remote_code=True,
71
- gpu_memory_utilization=self.config.gpu_memory_utilization,
72
- mm_processor_kwargs=self.config.mm_processor_kwargs,
73
- mm_processor_cache_gb=self.config.mm_processor_cache_gb,
74
- max_model_len=self.config.max_model_len,
64
+ max_loras=config.max_loras,
65
+ max_lora_rank=config.max_lora_rank,
66
+ enable_tower_connector_lora=config.enable_tower_connector_lora,
67
+ gpu_memory_utilization=config.gpu_memory_utilization,
68
+ mm_processor_kwargs=config.mm_processor_kwargs,
69
+ mm_processor_cache_gb=config.mm_processor_cache_gb,
70
+ max_model_len=config.max_model_len,
71
)
72
73
+ @property
74
+ def llm(self) -> vllm.LLM:
75
+ return self._runner.get_llm()
76
+
77
+ def __enter__(self) -> "Qwen2VLTester":
78
+ return self
79
+
80
+ def __exit__(self, exc_type, exc_value, traceback) -> None:
81
+ self._runner.__exit__(exc_type, exc_value, traceback)
82
+
83
def run_test(
84
self,
85
images: list[ImageAsset],
@@ -183,29 +189,29 @@ QWEN3VL_MODEL_PATH = "Qwen/Qwen3-VL-4B-Instruct"
189
def test_qwen2vl_lora(qwen2vl_lora_files):
190
"""Test Qwen 2.0 VL model with LoRA"""
191
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
186
- tester = Qwen2VLTester(config)
187
-
188
- # Test with different LoRA IDs
189
- for lora_id in [1, 2]:
190
- tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
192
+ with Qwen2VLTester(config) as tester:
193
+ # Test with different LoRA IDs
194
+ for lora_id in [1, 2]:
195
+ tester.run_test(
196
+ TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id
197
+ )
198
199
200
def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
201
"""Test Qwen 2.0 VL model with LoRA through beam search."""
202
config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
196
- tester = Qwen2VLTester(config)
197
-
198
- # Test with different LoRA IDs
199
- for lora_id in [1, 2]:
200
- # NOTE currently, we only test cherry blossom since stop sign
201
- # output is slightly different for v1; - the root cause is likely
202
- # independent of the intent of this test, which is to ensure beam
203
- # search passes through lora through correctly.
204
- tester.run_beam_search_test(
205
- [ImageAsset("cherry_blossom")],
206
- expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
207
- lora_id=lora_id,
208
- )
203
+ with Qwen2VLTester(config) as tester:
204
+ # Test with different LoRA IDs
205
+ for lora_id in [1, 2]:
206
+ # NOTE currently, we only test cherry blossom since stop sign
207
+ # output is slightly different for v1; - the root cause is likely
208
+ # independent of the intent of this test, which is to ensure beam
209
+ # search passes through lora through correctly.
210
+ tester.run_beam_search_test(
211
+ [ImageAsset("cherry_blossom")],
212
+ expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
213
+ lora_id=lora_id,
214
+ )
215
216
217
@pytest.mark.skipif(
@@ -214,11 +220,12 @@ def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
220
def test_qwen25vl_lora(qwen25vl_lora_files):
221
"""Test Qwen 2.5 VL model with LoRA"""
222
config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files)
217
- tester = Qwen2VLTester(config)
218
-
219
- # Test with different LoRA IDs
220
- for lora_id in [1, 2]:
221
- tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
223
+ with Qwen2VLTester(config) as tester:
224
+ # Test with different LoRA IDs
225
+ for lora_id in [1, 2]:
226
+ tester.run_test(
227
+ TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id
228
+ )
229
230
231
@pytest.mark.skipif(
@@ -234,13 +241,13 @@ def test_qwen25vl_vision_lora(qwen25vl_vision_lora_files):
241
mm_processor_cache_gb=0,
242
enable_tower_connector_lora=True,
243
)
237
- tester = Qwen2VLTester(config)
238
- for lora_id in [1, 2]:
239
- tester.run_test(
240
- TEST_IMAGES,
241
- expected_outputs=EXPECTED_OUTPUTS,
242
- lora_id=lora_id,
243
- )
244
+ with Qwen2VLTester(config) as tester:
245
+ for lora_id in [1, 2]:
246
+ tester.run_test(
247
+ TEST_IMAGES,
248
+ expected_outputs=EXPECTED_OUTPUTS,
249
+ lora_id=lora_id,
250
+ )
251
252
253
def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
@@ -253,13 +260,13 @@ def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
260
mm_processor_cache_gb=0,
261
enable_tower_connector_lora=True,
262
)
256
- tester = Qwen2VLTester(config)
257
- for lora_id in [1, 2]:
258
- tester.run_test(
259
- TEST_IMAGES,
260
- expected_outputs=EXPECTED_OUTPUTS,
261
- lora_id=lora_id,
262
- )
263
+ with Qwen2VLTester(config) as tester:
264
+ for lora_id in [1, 2]:
265
+ tester.run_test(
266
+ TEST_IMAGES,
267
+ expected_outputs=EXPECTED_OUTPUTS,
268
+ lora_id=lora_id,
269
+ )
270
271
272
def test_qwen2vl_multiple_lora_types(
@@ -287,34 +294,33 @@ def test_qwen2vl_multiple_lora_types(
294
mm_processor_cache_gb=0,
295
enable_tower_connector_lora=True,
296
)
290
- tester = Qwen2VLTester(config)
291
-
292
- # Test 1: Language-only LoRA adapter
293
- tester.config.lora_path = qwen2vl_language_lora_files
294
- for lora_id in [1, 2]:
295
- tester.run_test(
296
- TEST_IMAGES,
297
- expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
298
- lora_id=lora_id,
299
- lora_name="language_only",
300
- )
297
+ with Qwen2VLTester(config) as tester:
298
+ # Test 1: Language-only LoRA adapter
299
+ tester.config.lora_path = qwen2vl_language_lora_files
300
+ for lora_id in [1, 2]:
301
+ tester.run_test(
302
+ TEST_IMAGES,
303
+ expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
304
+ lora_id=lora_id,
305
+ lora_name="language_only",
306
+ )
307
302
- # Test 2: Vision tower + connector LoRA adapter
303
- tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
304
- for lora_id in [3, 4]:
305
- tester.run_test(
306
- TEST_IMAGES,
307
- expected_outputs=EXPECTED_OUTPUTS_VISION,
308
- lora_id=lora_id,
309
- lora_name="vision_tower_connector",
310
- )
308
+ # Test 2: Vision tower + connector LoRA adapter
309
+ tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
310
+ for lora_id in [3, 4]:
311
+ tester.run_test(
312
+ TEST_IMAGES,
313
+ expected_outputs=EXPECTED_OUTPUTS_VISION,
314
+ lora_id=lora_id,
315
+ lora_name="vision_tower_connector",
316
+ )
317
312
- # Test 3: Vision tower only LoRA adapter (no connector)
313
- tester.config.lora_path = qwen2vl_vision_tower_lora_files
314
- for lora_id in [5, 6]:
315
- tester.run_test(
316
- TEST_IMAGES,
317
- expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
318
- lora_id=lora_id,
319
- lora_name="vision_tower",
320
- )
318
+ # Test 3: Vision tower only LoRA adapter (no connector)
319
+ tester.config.lora_path = qwen2vl_vision_tower_lora_files
320
+ for lora_id in [5, 6]:
321
+ tester.run_test(
322
+ TEST_IMAGES,
323
+ expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
324
+ lora_id=lora_id,
325
+ lora_name="vision_tower",
326
+ )
tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh
+13
-3
@@ -98,7 +98,7 @@ GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
98
SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "")
99
100
# Trap the SIGINT signal (triggered by Ctrl+C)
101
-trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
101
+trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
102
103
# Waits for vLLM to start.
104
wait_for_server() {
@@ -110,10 +110,20 @@ wait_for_server() {
110
}
111
112
# Function to clean up previous instances
113
+wait_for_gpu_memory_release() {
114
+ if [[ "$SMI_BIN" == *"rocm"* ]]; then
115
+ PYTHONPATH="${GIT_ROOT}" python3 -c "from tests.utils import wait_for_rocm_memory_to_settle; wait_for_rocm_memory_to_settle()"
116
+ fi
117
+}
118
+
119
cleanup_instances() {
120
echo "Cleaning up any running vLLM instances..."
115
- pkill -f "vllm serve" || true
121
+ pkill -f "toy_proxy_server.py" || true
122
+ pkill -TERM -f "vllm serve" || true
123
+ sleep 3
124
+ pkill -9 -f "vllm serve" || true
125
sleep 2
126
+ wait_for_gpu_memory_release
127
}
128
129
get_num_gpus() {
@@ -132,6 +142,7 @@ get_num_gpus() {
142
# Function to run tests for a specific model
143
run_tests_for_model() {
144
local model_name=$1
145
+ cleanup_instances
146
echo "================================"
147
echo "Testing model: $model_name"
148
echo "================================"
@@ -303,7 +314,6 @@ run_tests_for_model() {
314
315
# Clean up before running next model
316
cleanup_instances
306
- sleep 3
317
}
318
319
# Run tests for each model