A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
[Bugfix] Fix Nemotron Parse loading (#37407)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 19, 2026 at 17:55 UTC
765e4610651b02fefe9ebe3b3d322fc398038af6
4 files changed
+49
-19
tests/models/multimodal/generation/test_keye.py
+1
-5
@@ -24,12 +24,8 @@ class ModelRequestData(NamedTuple):
24
sampling_params: SamplingParams | None = None
25
26
27
-@pytest.mark.core_model
27
@pytest.mark.parametrize("question", [QUESTION])
29
-def test_keye_vl(
30
- image_assets,
31
- question: str,
32
-):
28
+def test_keye_vl(image_assets, question: str):
29
images = [asset.pil_image for asset in image_assets]
30
image_urls = [encode_image_url(image) for image in images]
31
tests/models/multimodal/generation/test_nemotron_parse.py
+44
-11
@@ -1,21 +1,53 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from collections.abc import Sequence
3
+from collections.abc import Iterable, Sequence
4
5
import pytest
6
+import regex as re
7
from transformers import AutoModel
8
9
from tests.models.utils import check_logprobs_close
10
from vllm.assets.image import ImageAsset
11
+from vllm.logprobs import Logprob, SampleLogprobs
12
+from vllm.tokenizers import TokenizerLike
13
14
from ....conftest import HfRunner, PromptImageInput, VllmRunner
13
-from ....utils import create_new_process_for_each_test
15
16
IMAGE = ImageAsset("paper-11").pil_image_ext(ext="png").convert("RGB")
17
PROMPT = "</s><s><predict_bbox><predict_classes><output_markdown>"
18
19
20
+class DummyLogprobs(dict[int, Logprob]):
21
+ def __init__(self, vocab_ids: Iterable[int]):
22
+ super().__init__(dict.fromkeys(vocab_ids, Logprob(0.0)))
23
+
24
+ def __repr__(self):
25
+ return "DummyLogprobs()"
26
+
27
+
28
+def mask_bbox_tokens(
29
+ output: tuple[list[int], str, SampleLogprobs],
30
+ tokenizer: TokenizerLike,
31
+) -> tuple[list[int], str, SampleLogprobs]:
32
+ """
33
+ Always pass check_logprobs_close check for bounding box tokens
34
+ because it is reasonable for them to differ slightly.
35
+ """
36
+ ignore_pattern = r"<[xy]_[\d.]+>"
37
+ vocab = tokenizer.get_vocab()
38
+
39
+ output_ids, output_str, out_logprobs = output
40
+
41
+ masked_logprobs = list[dict[int, Logprob]]()
42
+ for token, logprobs in zip(output_ids, out_logprobs):
43
+ if re.match(ignore_pattern, tokenizer.decode(token)):
44
+ masked_logprobs.append(DummyLogprobs(vocab.values()))
45
+ else:
46
+ masked_logprobs.append(logprobs)
47
+
48
+ return output_ids, output_str, masked_logprobs
49
+
50
+
51
def run_test(
52
hf_runner: type[HfRunner],
53
vllm_runner: type[VllmRunner],
@@ -44,6 +76,8 @@ def run_test(
76
for prompts, images in inputs
77
]
78
79
+ tokenizer = vllm_model.llm.get_tokenizer()
80
+
81
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
82
hf_outputs_per_case = [
83
hf_model.generate_greedy_logprobs_limit(
@@ -58,18 +92,20 @@ def run_test(
92
93
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
94
check_logprobs_close(
61
- outputs_0_lst=hf_outputs,
62
- outputs_1_lst=vllm_outputs,
95
+ outputs_0_lst=[
96
+ mask_bbox_tokens(output, tokenizer) for output in hf_outputs
97
+ ],
98
+ outputs_1_lst=[
99
+ mask_bbox_tokens(output, tokenizer) for output in vllm_outputs
100
+ ],
101
name_0="hf",
102
name_1="vllm",
103
)
104
105
68
-@pytest.mark.core_model
106
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.1"])
107
@pytest.mark.parametrize("dtype", ["bfloat16"])
108
@pytest.mark.parametrize("num_logprobs", [5])
72
-@create_new_process_for_each_test("spawn")
109
def test_models(
110
hf_runner, vllm_runner, model: str, dtype: str, num_logprobs: int
111
) -> None:
@@ -77,10 +113,7 @@ def test_models(
113
hf_runner,
114
vllm_runner,
115
inputs=[
80
- (
81
- [PROMPT] * 10,
82
- [IMAGE] * 10,
83
- ),
116
+ ([PROMPT] * 10, [IMAGE] * 10),
117
],
118
model=model,
119
dtype=dtype,
tests/models/test_terratorch.py
+1
-1
@@ -8,7 +8,7 @@ from tests.conftest import VllmRunner
8
from tests.utils import create_new_process_for_each_test
9
10
11
-@create_new_process_for_each_test() # Memory is not cleaned up properly otherwise
11
+@create_new_process_for_each_test() # Hangs otherwise
12
@pytest.mark.parametrize(
13
"model",
14
[
vllm/model_executor/models/nemotron_parse.py
+3
-2
@@ -319,8 +319,9 @@ class MBartDecoderNoPos(nn.Module):
319
(".self_attn.qkv_proj", ".self_attn.q_proj", "q"),
320
(".self_attn.qkv_proj", ".self_attn.k_proj", "k"),
321
(".self_attn.qkv_proj", ".self_attn.v_proj", "v"),
322
- (".encoder_attn.kv_proj", ".encoder_attn.k_proj", "k"),
323
- (".encoder_attn.kv_proj", ".encoder_attn.v_proj", "v"),
322
+ # MergedColumnParallelLinear uses integer indices (0, 1)
323
+ (".encoder_attn.kv_proj", ".encoder_attn.k_proj", 0),
324
+ (".encoder_attn.kv_proj", ".encoder_attn.v_proj", 1),
325
]
326
params_dict = dict(self.named_parameters())
327
loaded_params: set[str] = set()