[CI] Reuse ColQwen3 models across pooling tests (#55889)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Andreas Karatzas committed
Sep 8, 2026 at 16:03 UTC
cb222346875b1cec3152b9c882781115accc57e5
1 file changed
+81
-162
tests/models/multimodal/pooling/test_colqwen3.py
+81
-162
@@ -50,6 +50,20 @@ DTYPE = "half"
50
GPU_MEMORY_UTILIZATION = 0.8
51
52
53
+@pytest.fixture(scope="module", params=MODELS)
54
+def colqwen3_model(request, vllm_runner):
55
+ model = request.param
56
+ with vllm_runner(
57
+ model,
58
+ runner="pooling",
59
+ dtype=DTYPE,
60
+ max_model_len=4096,
61
+ enforce_eager=True,
62
+ gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
63
+ ) as vllm_model:
64
+ yield model, vllm_model
65
+
66
+
67
def _make_base64_image(
68
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
69
) -> str:
@@ -87,75 +101,51 @@ def _make_text_mm_param(text: str) -> ScoreMultiModalParam:
101
102
103
def _run_token_embed_test(
90
- vllm_runner: type[VllmRunner],
104
+ vllm_model: VllmRunner,
105
model: str,
92
- *,
93
- dtype: str,
106
) -> None:
107
"""Verify per-token embedding shape and L2 normalization."""
96
- with vllm_runner(
97
- model,
98
- runner="pooling",
99
- dtype=dtype,
100
- max_model_len=4096,
101
- enforce_eager=True,
102
- gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
103
- ) as vllm_model:
104
- outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
105
-
106
- assert len(outputs) == 1
107
- emb = torch.tensor(outputs[0])
108
- # Token embeddings should be 2D: [num_tokens, embed_dim]
109
- assert emb.dim() == 2
110
- assert emb.shape[1] == EMBED_DIMS[model]
111
- assert emb.shape[0] > 1
112
-
113
- # Verify L2 normalization
114
- norms = torch.norm(emb, p=2, dim=-1)
115
- torch.testing.assert_close(
116
- norms,
117
- torch.ones_like(norms),
118
- rtol=1e-2,
119
- atol=1e-2,
120
- )
108
+ outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
109
+
110
+ assert len(outputs) == 1
111
+ emb = torch.tensor(outputs[0])
112
+ # Token embeddings should be 2D: [num_tokens, embed_dim]
113
+ assert emb.dim() == 2
114
+ assert emb.shape[1] == EMBED_DIMS[model]
115
+ assert emb.shape[0] > 1
116
+
117
+ # Verify L2 normalization
118
+ norms = torch.norm(emb, p=2, dim=-1)
119
+ torch.testing.assert_close(
120
+ norms,
121
+ torch.ones_like(norms),
122
+ rtol=1e-2,
123
+ atol=1e-2,
124
+ )
125
126
127
def _run_late_interaction_test(
124
- vllm_runner: type[VllmRunner],
125
- model: str,
126
- *,
127
- dtype: str,
128
+ vllm_model: VllmRunner,
129
) -> None:
130
"""Verify MaxSim scoring matches manual computation."""
131
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
132
132
- with vllm_runner(
133
- model,
134
- runner="pooling",
135
- dtype=dtype,
136
- max_model_len=4096,
137
- enforce_eager=True,
138
- gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
139
- ) as vllm_model:
140
- q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
141
- d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
133
+ q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
134
+ d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
135
143
- q_emb = torch.tensor(q_outputs[0])
144
- d_emb = torch.tensor(d_outputs[0])
136
+ q_emb = torch.tensor(q_outputs[0])
137
+ d_emb = torch.tensor(d_outputs[0])
138
146
- manual_score = compute_maxsim_score(q_emb, d_emb).item()
139
+ manual_score = compute_maxsim_score(q_emb, d_emb).item()
140
148
- vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
141
+ vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
142
150
- assert len(vllm_scores) == 1
151
- assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
143
+ assert len(vllm_scores) == 1
144
+ assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
145
146
147
def _run_relevance_test(
155
- vllm_runner: type[VllmRunner],
156
- model: str,
157
- *,
158
- dtype: str,
148
+ vllm_model: VllmRunner,
149
) -> None:
150
"""Verify that relevant documents score higher than irrelevant ones."""
151
query = "What is machine learning?"
@@ -165,59 +155,33 @@ def _run_relevance_test(
155
"Deep learning uses neural networks for complex tasks.",
156
]
157
168
- with vllm_runner(
169
- model,
170
- runner="pooling",
171
- dtype=dtype,
172
- max_model_len=4096,
173
- enforce_eager=True,
174
- gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
175
- ) as vllm_model:
176
- scores = vllm_model.score(query, documents)
158
+ scores = vllm_model.score(query, documents)
159
178
- assert len(scores) == 3
179
- assert scores[0] > scores[1], "ML doc should score higher than weather doc"
180
- assert scores[2] > scores[1], "DL doc should score higher than weather doc"
160
+ assert len(scores) == 3
161
+ assert scores[0] > scores[1], "ML doc should score higher than weather doc"
162
+ assert scores[2] > scores[1], "DL doc should score higher than weather doc"
163
164
183
-@pytest.mark.parametrize("model", MODELS)
184
-@pytest.mark.parametrize("dtype", [DTYPE])
185
-def test_colqwen3_token_embed(
186
- vllm_runner,
187
- model: str,
188
- dtype: str,
189
-) -> None:
190
- _run_token_embed_test(vllm_runner, model, dtype=dtype)
165
+def test_colqwen3_token_embed(colqwen3_model) -> None:
166
+ model, vllm_model = colqwen3_model
167
+ _run_token_embed_test(vllm_model, model)
168
169
193
-@pytest.mark.parametrize("model", MODELS)
194
-@pytest.mark.parametrize("dtype", [DTYPE])
195
-def test_colqwen3_late_interaction_scoring(
196
- vllm_runner,
197
- model: str,
198
- dtype: str,
199
-) -> None:
200
- _run_late_interaction_test(vllm_runner, model, dtype=dtype)
170
+def test_colqwen3_late_interaction_scoring(colqwen3_model) -> None:
171
+ _, vllm_model = colqwen3_model
172
+ _run_late_interaction_test(vllm_model)
173
174
203
-@pytest.mark.parametrize("model", MODELS)
204
-@pytest.mark.parametrize("dtype", [DTYPE])
205
-def test_colqwen3_relevance_ordering(
206
- vllm_runner,
207
- model: str,
208
- dtype: str,
209
-) -> None:
210
- _run_relevance_test(vllm_runner, model, dtype=dtype)
175
+def test_colqwen3_relevance_ordering(colqwen3_model) -> None:
176
+ _, vllm_model = colqwen3_model
177
+ _run_relevance_test(vllm_model)
178
179
180
# ── Multimodal scoring tests ────────────────────────────────
181
182
183
def _run_multimodal_text_query_image_docs_test(
217
- vllm_runner: type[VllmRunner],
218
- model: str,
219
- *,
220
- dtype: str,
184
+ vllm_model: VllmRunner,
185
) -> None:
186
"""Score a text query against image documents via the multimodal path.
187
@@ -233,26 +197,15 @@ def _run_multimodal_text_query_image_docs_test(
197
_make_image_mm_param(blue_image),
198
]
199
236
- with vllm_runner(
237
- model,
238
- runner="pooling",
239
- dtype=dtype,
240
- max_model_len=4096,
241
- enforce_eager=True,
242
- gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
243
- ) as vllm_model:
244
- scores = vllm_model.llm.score(query, image_docs)
200
+ scores = vllm_model.llm.score(query, image_docs)
201
246
- assert len(scores) == 2
247
- for s in scores:
248
- assert isinstance(s.outputs.score, float)
202
+ assert len(scores) == 2
203
+ for s in scores:
204
+ assert isinstance(s.outputs.score, float)
205
206
207
def _run_multimodal_mixed_docs_test(
252
- vllm_runner: type[VllmRunner],
253
- model: str,
254
- *,
255
- dtype: str,
208
+ vllm_model: VllmRunner,
209
) -> None:
210
"""Score a text query against a mix of text and image documents.
211
@@ -268,28 +221,17 @@ def _run_multimodal_mixed_docs_test(
221
_make_image_mm_param(red_image),
222
]
223
271
- with vllm_runner(
272
- model,
273
- runner="pooling",
274
- dtype=dtype,
275
- max_model_len=4096,
276
- enforce_eager=True,
277
- gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
278
- ) as vllm_model:
279
- scores = vllm_model.llm.score(query, documents)
224
+ scores = vllm_model.llm.score(query, documents)
225
281
- assert len(scores) == 2
282
- for s in scores:
283
- assert isinstance(s.outputs.score, float)
284
- # Text document about France should score higher than a random image
285
- assert scores[0].outputs.score > scores[1].outputs.score
226
+ assert len(scores) == 2
227
+ for s in scores:
228
+ assert isinstance(s.outputs.score, float)
229
+ # Text document about France should score higher than a random image
230
+ assert scores[0].outputs.score > scores[1].outputs.score
231
232
233
def _run_multimodal_image_query_text_docs_test(
289
- vllm_runner: type[VllmRunner],
290
- model: str,
291
- *,
292
- dtype: str,
234
+ vllm_model: VllmRunner,
235
) -> None:
236
"""Score an image query against text documents.
237
@@ -304,46 +246,23 @@ def _run_multimodal_image_query_text_docs_test(
246
"The weather forecast shows rain tomorrow.",
247
]
248
307
- with vllm_runner(
308
- model,
309
- runner="pooling",
310
- dtype=dtype,
311
- max_model_len=4096,
312
- enforce_eager=True,
313
- gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
314
- ) as vllm_model:
315
- scores = vllm_model.llm.score(image_query, documents)
249
+ scores = vllm_model.llm.score(image_query, documents)
250
317
- assert len(scores) == 2
318
- for s in scores:
319
- assert isinstance(s.outputs.score, float)
251
+ assert len(scores) == 2
252
+ for s in scores:
253
+ assert isinstance(s.outputs.score, float)
254
255
322
-@pytest.mark.parametrize("model", MODELS)
323
-@pytest.mark.parametrize("dtype", [DTYPE])
324
-def test_colqwen3_multimodal_text_query_image_docs(
325
- vllm_runner,
326
- model: str,
327
- dtype: str,
328
-) -> None:
329
- _run_multimodal_text_query_image_docs_test(vllm_runner, model, dtype=dtype)
256
+def test_colqwen3_multimodal_text_query_image_docs(colqwen3_model) -> None:
257
+ _, vllm_model = colqwen3_model
258
+ _run_multimodal_text_query_image_docs_test(vllm_model)
259
260
332
-@pytest.mark.parametrize("model", MODELS)
333
-@pytest.mark.parametrize("dtype", [DTYPE])
334
-def test_colqwen3_multimodal_mixed_docs(
335
- vllm_runner,
336
- model: str,
337
- dtype: str,
338
-) -> None:
339
- _run_multimodal_mixed_docs_test(vllm_runner, model, dtype=dtype)
261
+def test_colqwen3_multimodal_mixed_docs(colqwen3_model) -> None:
262
+ _, vllm_model = colqwen3_model
263
+ _run_multimodal_mixed_docs_test(vllm_model)
264
265
342
-@pytest.mark.parametrize("model", MODELS)
343
-@pytest.mark.parametrize("dtype", [DTYPE])
344
-def test_colqwen3_multimodal_image_query_text_docs(
345
- vllm_runner,
346
- model: str,
347
- dtype: str,
348
-) -> None:
349
- _run_multimodal_image_query_text_docs_test(vllm_runner, model, dtype=dtype)
266
+def test_colqwen3_multimodal_image_query_text_docs(colqwen3_model) -> None:
267
+ _, vllm_model = colqwen3_model
268
+ _run_multimodal_image_query_text_docs_test(vllm_model)