@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7eea5a520
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][CI] Fix failed v1-test because of min_p (#13316)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Feb 14, 2025 at 17:29 UTC e7eea5a5205ada4d01d2430e4bcce48cfe3e2ec8
1 file changed +4 -1
tests/v1/worker/test_gpu_input_batch.py
+4 -1
@@ -62,6 +62,7 @@ def _construct_expected_sampling_metadata(
62 repetition_penalties = [1.0 for _ in range(num_reqs)]
63 top_k = [0 for _ in range(num_reqs)]
64 top_p = [0.0 for _ in range(num_reqs)]
65 + min_p = [0.0 for _ in range(num_reqs)]
66 temperature = [0.0 for _ in range(num_reqs)]
67 stop_token_ids: List[Set[int]] = [set() for _ in range(num_reqs)]
68 min_tokens = [0 for _ in range(num_reqs)]
@@ -80,12 +81,12 @@ def _construct_expected_sampling_metadata(
81 req.sampling_params.repetition_penalty)
82 top_k[index_in_input_batch] = req.sampling_params.top_k
83 top_p[index_in_input_batch] = req.sampling_params.top_p
84 + min_p[index_in_input_batch] = req.sampling_params.min_p
85 temperature[index_in_input_batch] = req.sampling_params.temperature
86 stop_token_ids[
87 index_in_input_batch] = req.sampling_params.all_stop_token_ids
88 min_tokens[index_in_input_batch] = req.sampling_params.min_tokens
89 logit_bias[index_in_input_batch] = req.sampling_params.logit_bias
88 -
90 return SamplingMetadata(
91 temperature=torch.tensor(temperature, dtype=torch.float,
92 device=device),
@@ -95,6 +96,8 @@ def _construct_expected_sampling_metadata(
96 top_k=torch.tensor(top_k, dtype=torch.int, device=device),
97 no_top_p=all(x == 1.0 for x in top_p),
98 no_top_k=all(x == 0 for x in top_k),
99 + min_p=torch.tensor(min_p, dtype=torch.float, device=device),
100 + no_min_p=all(x == 0.0 for x in min_p),
101 generators={},
102 max_num_logprobs=0,
103 prompt_token_ids=make_tensor_with_pad(