A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
feature/issac 0.2 (#31550)
Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io>
Akshat Shrivastava committed
Jan 9, 2026 at 19:18 UTC
e45946bd91946e257d809d910c10d770e251e703
4 files changed
+43
-3
tests/models/multimodal/generation/test_common.py
+4
-1
@@ -526,7 +526,10 @@ VLM_TEST_SETTINGS = {
526
auto_cls=AutoModelForImageTextToText,
527
),
528
"isaac": VLMTestInfo(
529
- models=["PerceptronAI/Isaac-0.1"],
529
+ models=[
530
+ "PerceptronAI/Isaac-0.1",
531
+ "PerceptronAI/Isaac-0.2-2B-Preview",
532
+ ],
533
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
534
prompt_formatter=lambda img_prompt: (
535
f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n"
tests/models/registry.py
+1
@@ -708,6 +708,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
708
"IsaacForConditionalGeneration": _HfExamplesInfo(
709
"PerceptronAI/Isaac-0.1",
710
trust_remote_code=True,
711
+ extras={"0.2-2B-Preview": "PerceptronAI/Isaac-0.2-2B-Preview"},
712
),
713
"InternS1ForConditionalGeneration": _HfExamplesInfo(
714
"internlm/Intern-S1", trust_remote_code=True
vllm/model_executor/models/isaac.py
+23
-1
@@ -65,6 +65,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
65
from vllm.sequence import IntermediateTensors
66
from vllm.tokenizers import get_tokenizer
67
from vllm.tokenizers.hf import get_cached_tokenizer
68
+from vllm.transformers_utils.config import patch_rope_parameters
69
from vllm.transformers_utils.configs import (
70
IsaacConfig,
71
PixelShuffleSiglip2VisionConfig,
@@ -1284,11 +1285,14 @@ class IsaacForConditionalGeneration(
1285
hf_to_vllm_mapper = WeightsMapper(
1286
orig_to_new_prefix={
1287
"lm_head.": "language_model.lm_head.",
1288
+ "model.text_model.lm_head.": "language_model.lm_head.",
1289
+ "model.text_model.": "language_model.model.",
1290
"model.vision_embedding.0": "vision_embedding.transformer",
1291
"model.vision_embedding.1": "vision_embedding.linear_fc1",
1292
"model.vision_embedding.2": "vision_embedding.act",
1293
"model.vision_embedding.3": "vision_embedding.linear_fc2",
1294
"model.vision_embedding.": "vision_embedding.",
1295
+ "model.lm_head.": "language_model.lm_head.",
1296
"model.": "language_model.model.",
1297
}
1298
)
@@ -1319,7 +1323,25 @@ class IsaacForConditionalGeneration(
1323
)
1324
config.image_token_id = self.vision_token_id
1325
1322
- config.rope_scaling["mrope_section"] = calculated_mrope_section
1326
+ text_cfg = getattr(config, "text_config", None)
1327
+ target_cfg = (
1328
+ text_cfg
1329
+ if text_cfg is not None and not isinstance(text_cfg, dict)
1330
+ else config
1331
+ )
1332
+
1333
+ rope_scaling = getattr(target_cfg, "rope_scaling", None)
1334
+ if rope_scaling is None and target_cfg is config:
1335
+ rope_scaling = getattr(config, "_rope_scaling", None)
1336
+
1337
+ patch_rope_parameters(target_cfg)
1338
+ rope_parameters = target_cfg.rope_parameters
1339
+ rope_parameters["mrope_section"] = calculated_mrope_section
1340
+ if rope_scaling is not None and "mrope_interleaved" in rope_scaling:
1341
+ rope_parameters.setdefault(
1342
+ "mrope_interleaved", rope_scaling["mrope_interleaved"]
1343
+ )
1344
+ target_cfg.rope_parameters = rope_parameters
1345
self.language_model = init_vllm_registered_model(
1346
vllm_config=vllm_config,
1347
architectures=["Qwen3ForCausalLM"],
vllm/transformers_utils/configs/isaac.py
+15
-1
@@ -32,10 +32,14 @@ class IsaacConfig(Qwen3Config):
32
"""Configuration class for Isaac multimodal model."""
33
34
model_type = "isaac"
35
- sub_configs = {"vision_config": PixelShuffleSiglip2VisionConfig}
35
+ sub_configs = {
36
+ "vision_config": PixelShuffleSiglip2VisionConfig,
37
+ "text_config": Qwen3Config,
38
+ }
39
40
def __init__(
41
self,
42
+ text_config=None,
43
vision_config=None,
44
vision_patch_size: int = 16,
45
vision_max_num_patches: int = 256,
@@ -48,6 +52,16 @@ class IsaacConfig(Qwen3Config):
52
):
53
super().__init__(**kwargs)
54
55
+ if isinstance(text_config, dict):
56
+ # from HF config
57
+ self.text_config = self.sub_configs["text_config"](**text_config)
58
+ elif text_config is None:
59
+ # For BC use all kwargs to init text config.
60
+ self.text_config = self.sub_configs["text_config"](**kwargs)
61
+ else:
62
+ # from Qwen3Config
63
+ self.text_config = text_config
64
+
65
# EventStreamProcessor parameters (for backward compatibility)
66
self.video_patch_size = vision_patch_size
67
self.vision_max_num_patches = vision_max_num_patches