@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 91eea72330
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 09:45 UTC

[Tests] Add Qwen3-VL multimodal memory leak check (#39268)

Signed-off-by: Lalit Laxminarayan Bangad <lalitbangad@gmail.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

lalit10 committed Apr 9, 2026 at 04:54 UTC 91eea72330ef507c839596afc59f37732b045820
3 files changed +193 -1
tests/models/multimodal/generation/test_memory_leak.py new
+182
@@ -0,0 +1,182 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import gc
5 +import random
6 +import string
7 +import sys
8 +import weakref
9 +
10 +import pytest
11 +import torch
12 +
13 +from tests.models.registry import HF_EXAMPLE_MODELS
14 +from vllm import LLM, SamplingParams
15 +from vllm.distributed import cleanup_dist_env_and_memory
16 +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
17 +from vllm.platforms import current_platform
18 +from vllm.utils.mem_utils import KiB_bytes, MiB_bytes, format_mib
19 +
20 +MODEL_NAME = "Qwen/Qwen3-VL-4B-Instruct"
21 +RANDOM_PREFIX_LEN = 100
22 +TEST_IMAGE_NAMES = [
23 + "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
24 + "Grayscale_8bits_palette_sample_image.png",
25 +]
26 +MAX_MODEL_LEN = 8192
27 +REQUESTS_PER_ROUND = 4
28 +WARMUP_ROUNDS = 1
29 +MEASURED_ROUNDS = 16
30 +GPU_GROWTH_THRESHOLD_MIB = 0
31 +CPU_PEAK_GROWTH_THRESHOLD_MIB = 0
32 +
33 +SAMPLING_PARAMS = SamplingParams(
34 + temperature=0.0,
35 + max_tokens=16,
36 +)
37 +
38 +
39 +def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
40 + # Avoid obscuring memory leaks because of prefix caching
41 + random_text = "".join(random.choices(string.ascii_uppercase, k=RANDOM_PREFIX_LEN))
42 +
43 + return [
44 + {
45 + "role": "user",
46 + "content": [
47 + {
48 + "type": "text",
49 + "text": f"Ignore this random string: {random_text}",
50 + },
51 + {"type": "image_url", "image_url": {"url": image_url}},
52 + {
53 + "type": "text",
54 + "text": "Describe this image in one short sentence.",
55 + },
56 + ],
57 + }
58 + ]
59 +
60 +
61 +def _build_request_batch(
62 + image_urls: list[str],
63 +) -> list[list[ChatCompletionMessageParam]]:
64 + return [
65 + _make_messages(image_urls[i % len(image_urls)])
66 + for i in range(REQUESTS_PER_ROUND)
67 + ]
68 +
69 +
70 +def _ru_maxrss_bytes() -> int | None:
71 + try:
72 + import resource
73 + except ImportError:
74 + return None
75 +
76 + rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
77 + if rss <= 0:
78 + return 0
79 +
80 + # Linux reports kilobytes, macOS reports bytes.
81 + return rss if sys.platform == "darwin" else rss * KiB_bytes
82 +
83 +
84 +def _gpu_used_bytes() -> int:
85 + torch.accelerator.synchronize()
86 + free_bytes, total_bytes = current_platform.mem_get_info()
87 + return int(total_bytes - free_bytes)
88 +
89 +
90 +def _format_mib(num_bytes: int | None) -> str:
91 + if num_bytes is None:
92 + return "n/a"
93 +
94 + return f"{format_mib(num_bytes)} MiB"
95 +
96 +
97 +@pytest.fixture(scope="function")
98 +def llm(monkeypatch):
99 + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
100 +
101 + # pytest caches the fixture so we use weakref.proxy to
102 + # enable garbage collection
103 + llm_kwargs = dict(
104 + model=MODEL_NAME,
105 + enforce_eager=True,
106 + max_model_len=MAX_MODEL_LEN,
107 + max_num_seqs=REQUESTS_PER_ROUND,
108 + limit_mm_per_prompt={"image": 1},
109 + seed=0,
110 + disable_log_stats=True,
111 + gpu_memory_utilization=0.8,
112 + )
113 + if current_platform.is_rocm():
114 + llm_kwargs["attention_backend"] = "TRITON_ATTN"
115 +
116 + llm = LLM(**llm_kwargs)
117 +
118 + yield weakref.proxy(llm)
119 +
120 + del llm
121 +
122 + cleanup_dist_env_and_memory()
123 +
124 +
125 +@pytest.mark.core_model
126 +@pytest.mark.parametrize("image_urls", [TEST_IMAGE_NAMES], indirect=True)
127 +def test_qwen3_vl_no_memory_leak(llm, image_urls: list[str]) -> None:
128 + model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME)
129 + model_info.check_available_online(on_fail="skip")
130 + model_info.check_transformers_version(on_fail="skip")
131 +
132 + request_batch = _build_request_batch(image_urls)
133 +
134 + # Establish a warmup baseline after model load and the first multimodal
135 + # requests complete. Later rounds should remain near this steady state.
136 + for _ in range(WARMUP_ROUNDS):
137 + outputs = llm.chat(request_batch, sampling_params=SAMPLING_PARAMS)
138 + assert len(outputs) == len(request_batch)
139 + assert llm.llm_engine.get_num_unfinished_requests() == 0
140 + del outputs
141 +
142 + gc.collect()
143 + warmup_gpu = _gpu_used_bytes()
144 + warmup_cpu_peak = _ru_maxrss_bytes()
145 +
146 + post_warmup_gpu_samples: list[int] = []
147 + post_warmup_cpu_peak_samples: list[int] = []
148 +
149 + for _ in range(MEASURED_ROUNDS):
150 + outputs = llm.chat(request_batch, sampling_params=SAMPLING_PARAMS)
151 + assert len(outputs) == len(request_batch)
152 + assert llm.llm_engine.get_num_unfinished_requests() == 0
153 + del outputs
154 +
155 + gc.collect()
156 + post_warmup_gpu_samples.append(_gpu_used_bytes())
157 + cpu_peak = _ru_maxrss_bytes()
158 + if cpu_peak is not None:
159 + post_warmup_cpu_peak_samples.append(cpu_peak)
160 +
161 + gpu_growth = max(post_warmup_gpu_samples) - warmup_gpu
162 + gpu_threshold = GPU_GROWTH_THRESHOLD_MIB * MiB_bytes
163 +
164 + assert gpu_growth <= gpu_threshold, (
165 + "Qwen3-VL GPU memory kept growing after warmup. "
166 + f"warmup_baseline={_format_mib(warmup_gpu)}, "
167 + f"post_warmup_samples={[_format_mib(x) for x in post_warmup_gpu_samples]}, "
168 + f"gpu_growth={_format_mib(gpu_growth)}, "
169 + f"gpu_threshold={GPU_GROWTH_THRESHOLD_MIB} MiB"
170 + )
171 +
172 + if warmup_cpu_peak is not None and post_warmup_cpu_peak_samples:
173 + cpu_peak_growth = max(post_warmup_cpu_peak_samples) - warmup_cpu_peak
174 + cpu_threshold = CPU_PEAK_GROWTH_THRESHOLD_MIB * MiB_bytes
175 +
176 + assert cpu_peak_growth <= cpu_threshold, (
177 + "Qwen3-VL CPU peak RSS kept growing after warmup. "
178 + f"warmup_ru_maxrss={_format_mib(warmup_cpu_peak)}, "
179 + f"post_warmup_ru_maxrss={[_format_mib(x) for x in post_warmup_cpu_peak_samples]}, " # noqa: E501
180 + f"cpu_peak_growth={_format_mib(cpu_peak_growth)}, "
181 + f"cpu_peak_threshold={CPU_PEAK_GROWTH_THRESHOLD_MIB} MiB"
182 + )
vllm/utils/mem_constants.py
+6
@@ -1,5 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +KB_bytes = 1_000
4 +"""The number of bytes in one kilobyte (KB)."""
5 +
6 +KiB_bytes = 1 << 10
7 +"""The number of bytes in one kibibyte (KiB)."""
8 +
9 MB_bytes = 1_000_000
10 """The number of bytes in one megabyte (MB)."""
11
vllm/utils/mem_utils.py
+5 -1
@@ -13,7 +13,11 @@ import torch.types
13
14 from vllm.platforms import current_platform
15
16 -from .mem_constants import GiB_bytes, MiB_bytes
16 +from .mem_constants import GiB_bytes, KiB_bytes, MiB_bytes
17 +
18 +
19 +def format_kib(b: int) -> str:
20 + return f"{round(b / KiB_bytes, 2)}"
21
22
23 def format_mib(b: int) -> str: