A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Sampler] Don't apply temp for greedy-only (#13311)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Feb 14, 2025 at 18:10 UTC
6a854c7a2bb5b8a2015bbd83d94d311b991ac45d
1 file changed
+15
-9
vllm/v1/sample/sampler.py
+15
-9
@@ -41,8 +41,6 @@ class Sampler(nn.Module):
41
logits = self.apply_logits_bias(logits, sampling_metadata)
42
# Apply penalties (e.g., min_tokens, freq_penalties).
43
logits = self.apply_penalties(logits, sampling_metadata)
44
- # Apply temperature.
45
- logits = self.apply_temperature(logits, sampling_metadata.temperature)
44
# Sample the next token.
45
sampled = self.sample(logits, sampling_metadata)
46
@@ -82,9 +80,21 @@ class Sampler(nn.Module):
80
) -> torch.Tensor:
81
assert not (sampling_metadata.all_greedy
82
and sampling_metadata.all_random)
85
- if sampling_metadata.all_greedy:
86
- return self.greedy_sample(logits)
83
+ if sampling_metadata.all_random:
84
+ greedy_sampled = None
85
+ else:
86
+ greedy_sampled = self.greedy_sample(logits)
87
+ if sampling_metadata.all_greedy:
88
+ return greedy_sampled
89
90
+ # Apply temperature.
91
+ logits = self.apply_temperature(logits, sampling_metadata.temperature)
92
+
93
+ # Apply min_p.
94
+ if not sampling_metadata.no_min_p:
95
+ logits = self.apply_min_p(logits, sampling_metadata.min_p)
96
+
97
+ # Apply top_k and/or top_p.
98
random_sampled = self.topk_topp_sampler(
99
logits,
100
sampling_metadata.generators,
@@ -94,13 +104,9 @@ class Sampler(nn.Module):
104
sampling_metadata.top_p,
105
)
106
97
- if not sampling_metadata.no_min_p:
98
- logits = self.apply_min_p(logits, sampling_metadata.min_p)
99
-
100
- if sampling_metadata.all_random:
107
+ if greedy_sampled is None:
108
return random_sampled
109
103
- greedy_sampled = self.greedy_sample(logits)
110
sampled = torch.where(
111
sampling_metadata.temperature < _SAMPLING_EPS,
112
greedy_sampled,