A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 08:30 UTC
[CI/Build] Enable Step3p7ForConditionalGeneration testing (#43956)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Jee Jee Li committed
May 31, 2026 at 13:16 UTC
6bdabbad5bce747865fd3a249658518a4269cc22
2 files changed
+9
-1
tests/models/multimodal/processing/test_tensor_schema.py
+1
@@ -180,6 +180,7 @@ def test_model_tensor_schema(model_id: str):
180
dummy_hf_overrides,
181
model_arch=model_arch,
182
exist_overrides=model_info.hf_overrides,
183
+ use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
184
)
185
186
# ROCm: Detect if model uses AWQ quantization and set appropriate dtype
tests/models/registry.py
+8
-1
@@ -1372,7 +1372,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1372
"stepfun-ai/Step3-VL-10B", trust_remote_code=True
1373
),
1374
"Step3p7ForConditionalGeneration": _HfExamplesInfo(
1375
- "stepfun-ai/Step-3.7-Flash", is_available_online=False, trust_remote_code=True
1375
+ "stepfun-ai/Step-3.7-Flash",
1376
+ trust_remote_code=True,
1377
+ use_original_num_layers=True,
1378
+ # The MoE config lives in the nested ``text_config``, so the overrides
1379
+ # must be nested too. Use 4 layers to initialize at least one MoE layer
1380
+ # and shrink ``moe_num_experts`` (a non-standard key not handled by
1381
+ # ``dummy_hf_overrides``) to avoid OOM during init.
1382
+ hf_overrides={"text_config": {"num_hidden_layers": 4, "moe_num_experts": 8}},
1383
),
1384
"UltravoxModel": _HfExamplesInfo(
1385
"fixie-ai/ultravox-v0_5-llama-3_2-1b",