A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Change `top_k` to be disabled with `0` (still accept `-1` for now) (#17773)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 9, 2025 at 11:01 UTC
c6798baa9c8c484eb3f8862822f5eb0608927998
6 files changed
+14
-13
tests/samplers/test_sampler.py
+1
-1
@@ -478,7 +478,7 @@ def test_sampler_mixed(seed: int, device: str):
478
sampling_params = SamplingParams(
479
temperature=random.random() + 0.1,
480
top_p=min(random.random() + 0.1, 1),
481
- top_k=random.randint(0, 10) or -1,
481
+ top_k=random.randint(0, 10),
482
n=n,
483
presence_penalty=random.randint(0, 1),
484
)
vllm/entrypoints/openai/protocol.py
+3
-3
@@ -409,7 +409,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
409
"repetition_penalty": 1.0,
410
"temperature": 1.0,
411
"top_p": 1.0,
412
- "top_k": -1,
412
+ "top_k": 0,
413
"min_p": 0.0,
414
}
415
@@ -853,7 +853,7 @@ class CompletionRequest(OpenAIBaseModel):
853
"repetition_penalty": 1.0,
854
"temperature": 1.0,
855
"top_p": 1.0,
856
- "top_k": -1,
856
+ "top_k": 0,
857
"min_p": 0.0,
858
}
859
@@ -1679,7 +1679,7 @@ class TranscriptionRequest(OpenAIBaseModel):
1679
"repetition_penalty": 1.0,
1680
"temperature": 1.0,
1681
"top_p": 1.0,
1682
- "top_k": -1,
1682
+ "top_k": 0,
1683
"min_p": 0.0,
1684
}
1685
vllm/model_executor/sampling_metadata.py
+1
-1
@@ -416,7 +416,7 @@ class SamplingTensors:
416
417
# k should not be greater than the vocab size.
418
top_k = min(sampling_params.top_k, vocab_size)
419
- top_k = vocab_size if top_k == -1 else top_k
419
+ top_k = vocab_size if top_k < 1 else top_k
420
if temperature < _SAMPLING_EPS:
421
# NOTE: Zero temperature means deterministic sampling
422
# (i.e., greedy sampling or beam search).
vllm/sampling_params.py
+7
-6
@@ -149,7 +149,7 @@ class SamplingParams(
149
top_p: Float that controls the cumulative probability of the top tokens
150
to consider. Must be in (0, 1]. Set to 1 to consider all tokens.
151
top_k: Integer that controls the number of top tokens to consider. Set
152
- to -1 to consider all tokens.
152
+ to 0 (or -1) to consider all tokens.
153
min_p: Float that represents the minimum probability for a token to be
154
considered, relative to the probability of the most likely token.
155
Must be in [0, 1]. Set to 0 to disable this.
@@ -209,7 +209,7 @@ class SamplingParams(
209
repetition_penalty: float = 1.0
210
temperature: float = 1.0
211
top_p: float = 1.0
212
- top_k: int = -1
212
+ top_k: int = 0
213
min_p: float = 0.0
214
seed: Optional[int] = None
215
stop: Optional[Union[str, list[str]]] = None
@@ -256,7 +256,7 @@ class SamplingParams(
256
repetition_penalty: Optional[float] = 1.0,
257
temperature: Optional[float] = 1.0,
258
top_p: Optional[float] = 1.0,
259
- top_k: int = -1,
259
+ top_k: int = 0,
260
min_p: float = 0.0,
261
seed: Optional[int] = None,
262
stop: Optional[Union[str, list[str]]] = None,
@@ -376,7 +376,7 @@ class SamplingParams(
376
if self.temperature < _SAMPLING_EPS:
377
# Zero temperature means greedy sampling.
378
self.top_p = 1.0
379
- self.top_k = -1
379
+ self.top_k = 0
380
self.min_p = 0.0
381
self._verify_greedy_sampling()
382
@@ -404,8 +404,9 @@ class SamplingParams(
404
f"temperature must be non-negative, got {self.temperature}.")
405
if not 0.0 < self.top_p <= 1.0:
406
raise ValueError(f"top_p must be in (0, 1], got {self.top_p}.")
407
- if self.top_k < -1 or self.top_k == 0:
408
- raise ValueError(f"top_k must be -1 (disable), or at least 1, "
407
+ # quietly accept -1 as disabled, but prefer 0
408
+ if self.top_k < -1:
409
+ raise ValueError(f"top_k must be 0 (disable), or at least 1, "
410
f"got {self.top_k}.")
411
if not isinstance(self.top_k, int):
412
raise TypeError(
vllm/worker/neuron_model_runner.py
+1
-1
@@ -348,7 +348,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
348
if temperature == 0.0:
349
# Enable greedy sampling on zero temperature
350
return (1, 1.0, 1.0)
351
- if top_k < 0 or top_k > self._MAX_NEURON_SAMPLING_TOP_K:
351
+ if top_k < 1 or top_k > self._MAX_NEURON_SAMPLING_TOP_K:
352
top_k = self._MAX_NEURON_SAMPLING_TOP_K
353
354
return (top_k, top_p, temperature)
vllm/worker/tpu_model_runner.py
+1
-1
@@ -525,7 +525,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
525
"Top-p sampling is currently disabled for the TPU backend "
526
"due to performance issues.")
527
p.append(sampling_params.top_p)
528
- if sampling_params.top_k != -1:
528
+ if sampling_params.top_k > 0:
529
raise NotImplementedError(
530
"Top-k sampling is currently disabled for the TPU backend "
531
"due to performance issues.")