A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
Update sampling_metadata.py (#21937)
Signed-off-by: Aviad Rossmann <aviadr@neureality.ai>
Aviad Rossmann committed
Aug 1, 2025 at 09:23 UTC
53d7c39271aeb0568afcae337396a972e1848586
1 file changed
+6
-6
vllm/model_executor/sampling_metadata.py
+6
-6
@@ -539,37 +539,37 @@ class SamplingTensors:
539
temperatures_t = torch.tensor(
540
temperatures,
541
device="cpu",
542
- dtype=dtype,
542
+ dtype=torch.float32,
543
pin_memory=pin_memory,
544
)
545
top_ps_t = torch.tensor(
546
top_ps,
547
device="cpu",
548
- dtype=dtype,
548
+ dtype=torch.float32,
549
pin_memory=pin_memory,
550
)
551
min_ps_t = torch.tensor(
552
min_ps,
553
device="cpu",
554
- dtype=dtype,
554
+ dtype=torch.float32,
555
pin_memory=pin_memory,
556
)
557
presence_penalties_t = torch.tensor(
558
presence_penalties,
559
device="cpu",
560
- dtype=dtype,
560
+ dtype=torch.float32,
561
pin_memory=pin_memory,
562
)
563
frequency_penalties_t = torch.tensor(
564
frequency_penalties,
565
device="cpu",
566
- dtype=dtype,
566
+ dtype=torch.float32,
567
pin_memory=pin_memory,
568
)
569
repetition_penalties_t = torch.tensor(
570
repetition_penalties,
571
device="cpu",
572
- dtype=dtype,
572
+ dtype=torch.float32,
573
pin_memory=pin_memory,
574
)
575
top_ks_t = torch.tensor(