A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
[MRV2] Use fp32 for draft logits (#37526)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Mar 19, 2026 at 08:41 UTC
40b8363b45a9c59984907603b00b736e41d25065
2 files changed
+1
-3
vllm/v1/worker/gpu/model_runner.py
-1
@@ -195,7 +195,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
195
num_speculative_steps=self.num_speculative_steps,
196
vocab_size=self.vocab_size,
197
device=self.device,
198
- model_dtype=self.dtype,
198
cache_draft_logits=not use_strict_rejection_sampling,
199
)
200
self.input_buffers = InputBuffers(
vllm/v1/worker/gpu/states.py
+1
-2
@@ -15,7 +15,6 @@ class RequestState:
15
num_speculative_steps: int,
16
vocab_size: int,
17
device: torch.device,
18
- model_dtype: torch.dtype,
18
cache_draft_logits: bool,
19
):
20
self.max_num_reqs = max_num_reqs
@@ -81,7 +80,7 @@ class RequestState:
80
self.max_num_reqs,
81
self.num_speculative_steps,
82
self.vocab_size,
84
- dtype=model_dtype,
83
+ dtype=torch.float32,
84
device=device,
85
)
86