@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6a854c7a2b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][Sampler] Don't apply temp for greedy-only (#13311)

Signed-off-by: Nick Hill <nhill@redhat.com>

Nick Hill committed Feb 14, 2025 at 18:10 UTC 6a854c7a2bb5b8a2015bbd83d94d311b991ac45d
1 file changed +15 -9
vllm/v1/sample/sampler.py
+15 -9
@@ -41,8 +41,6 @@ class Sampler(nn.Module):
41 logits = self.apply_logits_bias(logits, sampling_metadata)
42 # Apply penalties (e.g., min_tokens, freq_penalties).
43 logits = self.apply_penalties(logits, sampling_metadata)
44 - # Apply temperature.
45 - logits = self.apply_temperature(logits, sampling_metadata.temperature)
44 # Sample the next token.
45 sampled = self.sample(logits, sampling_metadata)
46
@@ -82,9 +80,21 @@ class Sampler(nn.Module):
80 ) -> torch.Tensor:
81 assert not (sampling_metadata.all_greedy
82 and sampling_metadata.all_random)
85 - if sampling_metadata.all_greedy:
86 - return self.greedy_sample(logits)
83 + if sampling_metadata.all_random:
84 + greedy_sampled = None
85 + else:
86 + greedy_sampled = self.greedy_sample(logits)
87 + if sampling_metadata.all_greedy:
88 + return greedy_sampled
89
90 + # Apply temperature.
91 + logits = self.apply_temperature(logits, sampling_metadata.temperature)
92 +
93 + # Apply min_p.
94 + if not sampling_metadata.no_min_p:
95 + logits = self.apply_min_p(logits, sampling_metadata.min_p)
96 +
97 + # Apply top_k and/or top_p.
98 random_sampled = self.topk_topp_sampler(
99 logits,
100 sampling_metadata.generators,
@@ -94,13 +104,9 @@ class Sampler(nn.Module):
104 sampling_metadata.top_p,
105 )
106
97 - if not sampling_metadata.no_min_p:
98 - logits = self.apply_min_p(logits, sampling_metadata.min_p)
99 -
100 - if sampling_metadata.all_random:
107 + if greedy_sampled is None:
108 return random_sampled
109
103 - greedy_sampled = self.greedy_sample(logits)
110 sampled = torch.where(
111 sampling_metadata.temperature < _SAMPLING_EPS,
112 greedy_sampled,