A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
Revert "Update sampling_metadata.py (#21937)" (#22088)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Aug 1, 2025 at 13:24 UTC
87c94bc87943818ad039d5c916df793fbd081e6a
1 file changed
+6
-6
vllm/model_executor/sampling_metadata.py
+6
-6
@@ -539,37 +539,37 @@ class SamplingTensors:
539
temperatures_t = torch.tensor(
540
temperatures,
541
device="cpu",
542
- dtype=torch.float32,
542
+ dtype=dtype,
543
pin_memory=pin_memory,
544
)
545
top_ps_t = torch.tensor(
546
top_ps,
547
device="cpu",
548
- dtype=torch.float32,
548
+ dtype=dtype,
549
pin_memory=pin_memory,
550
)
551
min_ps_t = torch.tensor(
552
min_ps,
553
device="cpu",
554
- dtype=torch.float32,
554
+ dtype=dtype,
555
pin_memory=pin_memory,
556
)
557
presence_penalties_t = torch.tensor(
558
presence_penalties,
559
device="cpu",
560
- dtype=torch.float32,
560
+ dtype=dtype,
561
pin_memory=pin_memory,
562
)
563
frequency_penalties_t = torch.tensor(
564
frequency_penalties,
565
device="cpu",
566
- dtype=torch.float32,
566
+ dtype=dtype,
567
pin_memory=pin_memory,
568
)
569
repetition_penalties_t = torch.tensor(
570
repetition_penalties,
571
device="cpu",
572
- dtype=torch.float32,
572
+ dtype=dtype,
573
pin_memory=pin_memory,
574
)
575
top_ks_t = torch.tensor(