A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 14:04 UTC
[Bugfix] DFlash FP8 KV-Cache (#42692)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Benjamin Chislett committed
May 15, 2026 at 10:29 UTC
0fe7550254674d843d645c4e01c5ceaf6bf87e81
2 files changed
+4
-2
vllm/model_executor/models/qwen3_dflash.py
+3
-1
@@ -247,8 +247,10 @@ class DFlashQwen3Model(nn.Module):
247
[
248
DFlashQwen3DecoderLayer(
249
current_vllm_config,
250
- prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
250
config=self.config,
251
+ cache_config=current_vllm_config.cache_config,
252
+ quant_config=self.quant_config,
253
+ prefix=maybe_prefix(prefix, f"layers.{layer_idx + start_layer_id}"),
254
)
255
for layer_idx in range(self.config.num_hidden_layers)
256
]
vllm/v1/spec_decode/llm_base_proposer.py
+1
-1
@@ -116,7 +116,7 @@ class SpecDecodeBaseProposer:
116
117
self.max_batch_size = vllm_config.scheduler_config.max_num_seqs
118
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
119
- self.token_arange_np = np.arange(self.max_num_tokens)
119
+ self.token_arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
120
121
# Can be specialized by methods like DFlash to reduce the limit
122
self.max_query_tokens = self.max_num_tokens