A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
Fix `ExaoneMoeMTP` test that never ran in Transformers v4 (#36792)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 11, 2026 at 17:10 UTC
5efa206a8cc5501563a79f667a5ae2f87dba2108
4 files changed
+17
tests/distributed/test_pipeline_parallel.py
+3
@@ -247,6 +247,7 @@ def _compare_tp(
247
hf_config = get_config(model_id, trust_remote_code)
248
require_embed_inputs = model_info.require_embed_inputs
249
max_num_seqs = model_info.max_num_seqs
250
+ enable_prefix_caching = model_info.enable_prefix_caching
251
252
dtype = "float16"
253
if hf_config.model_type in _FLOAT16_NOT_SUPPORTED_MODELS:
@@ -300,6 +301,8 @@ def _compare_tp(
301
common_args.extend(["--load-format", load_format])
302
if hf_overrides:
303
common_args.extend(["--hf-overrides", json.dumps(hf_overrides)])
304
+ if not enable_prefix_caching:
305
+ common_args.append("--no-enable-prefix-caching")
306
if require_embed_inputs:
307
common_args.extend(
308
[
tests/models/multimodal/generation/vlm_utils/core.py
+2
@@ -74,6 +74,8 @@ def run_test(
74
if model_info.require_embed_inputs:
75
for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
76
vllm_runner_kwargs_[k] = model_info.require_embed_inputs
77
+ if not model_info.enable_prefix_caching:
78
+ vllm_runner_kwargs_["enable_prefix_caching"] = False
79
80
if vllm_runner_kwargs:
81
vllm_runner_kwargs_.update(vllm_runner_kwargs)
tests/models/registry.py
+7
@@ -72,6 +72,12 @@ class _HfExamplesInfo:
72
If False, we will use CUDA graph and eager execution in hybrid.
73
"""
74
75
+ enable_prefix_caching: bool = True
76
+ """
77
+ Whether to enable prefix caching for the model. If True, we will test the model with
78
+ prefix caching enabled. If False, we will test the model without prefix caching.
79
+ """
80
+
81
is_available_online: bool = True
82
"""
83
Set this to `False` if the name of this architecture no longer exists on
@@ -1206,6 +1212,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
1212
"LGAI-EXAONE/K-EXAONE-236B-A23B",
1213
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
1214
min_transformers_version="5.1.0",
1215
+ enable_prefix_caching=False,
1216
),
1217
"ExtractHiddenStatesModel": _HfExamplesInfo(
1218
"Qwen/Qwen3-8B",
tests/models/test_initialization.py
+5
@@ -136,6 +136,10 @@ def can_initialize(
136
if model_arch == "WhisperForConditionalGeneration":
137
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
138
139
+ kwargs = {}
140
+ if not model_info.enable_prefix_caching:
141
+ kwargs["enable_prefix_caching"] = False
142
+
143
LLM(
144
model_info.default,
145
tokenizer=model_info.tokenizer,
@@ -165,6 +169,7 @@ def can_initialize(
169
hf_overrides=hf_overrides_fn,
170
max_num_seqs=model_info.max_num_seqs,
171
attention_config=attention_config,
172
+ **kwargs,
173
)
174
175