A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 17:00 UTC
Fix failing test for CRadio (#27738)
Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: wang.yuqi <noooop@126.com>
Eugene Khvedchenya committed
Nov 7, 2025 at 01:32 UTC
827e4237bc28a8f5128639b6d8c99b485035de19
1 file changed
+12
-1
tests/models/multimodal/pooling/test_radio.py
+12
-1
@@ -42,6 +42,10 @@ def run_radio_test(
42
43
config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
44
45
+ # RADIO model on HF does not properly handle torch_dtype argument
46
+ # And relies on args["dtype"] which we have to patch manually:
47
+ config.args["dtype"] = torch_dtype
48
+
49
hf_model = AutoModel.from_pretrained(
50
model_id,
51
config=config,
@@ -50,6 +54,13 @@ def run_radio_test(
54
).to("cuda")
55
hf_model.eval()
56
57
+ # A HF model has image normalization as a part of model's forward
58
+ # However in vLLM we don't make normalization a part of the model
59
+ # forward step since mean/std stored as model's parameters and
60
+ # subject to precision loss (when using fp16/bf16) which negatively
61
+ # affects evaluation benchmarks.
62
+ hf_model.make_preprocessor_external()
63
+
64
hf_outputs_per_image = [
65
hf_model(pixel_value.to("cuda")).features for pixel_value in pixel_values
66
]
@@ -78,7 +89,7 @@ def run_radio_test(
89
"nvidia/C-RADIOv2-H",
90
],
91
)
81
-@pytest.mark.parametrize("dtype", ["half"])
92
+@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
93
def test_radio(dist_init, image_assets, model_id, dtype: str) -> None:
94
run_radio_test(
95
image_assets,