@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dcb1a944d4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1] Adding min tokens/repetition/presence/frequence penalties to V1 sampler (#10681)

Signed-off-by: Sourashis Roy <sroy@roblox.com> Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

sroy745 committed Dec 26, 2024 at 02:02 UTC dcb1a944d4cf95b4a7b3522ddf970e6d3a28b8b5
11 files changed +879 -49
tests/v1/engine/test_engine_core.py
+38
@@ -139,3 +139,41 @@ def test_engine_core(monkeypatch):
139 engine_core.abort_requests([req2.request_id, req0.request_id])
140 assert len(engine_core.scheduler.waiting) == 0
141 assert len(engine_core.scheduler.running) == 0
142 +
143 +
144 +def test_engine_core_advanced_sampling(monkeypatch):
145 + """
146 + A basic end-to-end test to verify that the engine functions correctly
147 + when additional sampling parameters, such as min_tokens and
148 + presence_penalty, are set.
149 + """
150 + with monkeypatch.context() as m:
151 + m.setenv("VLLM_USE_V1", "1")
152 + """Setup the EngineCore."""
153 + engine_args = EngineArgs(model=MODEL_NAME)
154 + vllm_config = engine_args.create_engine_config(
155 + usage_context=UsageContext.UNKNOWN_CONTEXT)
156 + executor_class = AsyncLLM._get_executor_cls(vllm_config)
157 +
158 + engine_core = EngineCore(vllm_config=vllm_config,
159 + executor_class=executor_class,
160 + usage_context=UsageContext.UNKNOWN_CONTEXT)
161 + """Test basic request lifecycle."""
162 + # First request.
163 + request: EngineCoreRequest = make_request()
164 + request.sampling_params = SamplingParams(
165 + min_tokens=4,
166 + presence_penalty=1.0,
167 + frequency_penalty=1.0,
168 + repetition_penalty=0.1,
169 + stop_token_ids=[1001, 1002],
170 + )
171 + engine_core.add_request(request)
172 + assert len(engine_core.scheduler.waiting) == 1
173 + assert len(engine_core.scheduler.running) == 0
174 + # Loop through until they are all done.
175 + while len(engine_core.step()) > 0:
176 + pass
177 +
178 + assert len(engine_core.scheduler.waiting) == 0
179 + assert len(engine_core.scheduler.running) == 0
tests/v1/sample/__init__.py
tests/v1/sample/test_sampler.py new
+331
@@ -0,0 +1,331 @@
1 +from typing import List, Set, Tuple
2 +
3 +import numpy as np
4 +import pytest
5 +import torch
6 +
7 +from vllm.utils import make_tensor_with_pad
8 +from vllm.v1.sample.metadata import SamplingMetadata
9 +from vllm.v1.sample.sampler import Sampler
10 +
11 +VOCAB_SIZE = 1024
12 +NUM_OUTPUT_TOKENS = 20
13 +CUDA_DEVICES = [
14 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15 +]
16 +MAX_NUM_PROMPT_TOKENS = 64
17 +
18 +
19 +def _create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor:
20 + fake_logits = torch.full((batch_size, vocab_size), 1e-2, dtype=torch.float)
21 + return fake_logits
22 +
23 +
24 +def _create_penalty_tensor(batch_size: int, penalty_value: float,
25 + device: torch.device) -> torch.Tensor:
26 + return torch.full((batch_size, ),
27 + fill_value=penalty_value,
28 + dtype=torch.float,
29 + device=device)
30 +
31 +
32 +def _create_prompt_tokens_tensor(
33 + prompt_token_ids: List[List[int]],
34 + vocab_size: int,
35 + device: torch.device,
36 +) -> torch.Tensor:
37 + return make_tensor_with_pad(
38 + prompt_token_ids,
39 + pad=vocab_size,
40 + device=device,
41 + dtype=torch.int64,
42 + pin_memory=False,
43 + )
44 +
45 +
46 +def _create_default_sampling_metadata(
47 + num_output_tokens: int,
48 + batch_size: int,
49 + vocab_size: int,
50 + device: torch.device,
51 +) -> SamplingMetadata:
52 + output_token_ids: List[List[int]] = []
53 + prompt_token_ids: List[List[int]] = []
54 + for _ in range(batch_size):
55 + output_token_ids.append(
56 + np.random.randint(0, vocab_size, size=num_output_tokens).tolist())
57 + prompt_token_ids.append(
58 + np.random.randint(0,
59 + vocab_size,
60 + size=np.random.randint(
61 + 1, MAX_NUM_PROMPT_TOKENS)).tolist())
62 + fake_sampling_metadata = SamplingMetadata(
63 + temperature=torch.full((batch_size, ), 0.0),
64 + all_greedy=True,
65 + all_random=False,
66 + top_p=torch.empty(batch_size, ),
67 + top_k=torch.empty(batch_size, ),
68 + no_top_p=True,
69 + no_top_k=True,
70 + generators={},
71 + max_num_logprobs=VOCAB_SIZE,
72 + prompt_token_ids=_create_prompt_tokens_tensor(prompt_token_ids,
73 + vocab_size, device),
74 + output_token_ids=output_token_ids,
75 + frequency_penalties=_create_penalty_tensor(batch_size, 0.0, device),
76 + presence_penalties=_create_penalty_tensor(batch_size, 0.0, device),
77 + repetition_penalties=_create_penalty_tensor(batch_size, 1.0, device),
78 + no_penalties=True,
79 + min_tokens=[],
80 + stop_token_ids=[],
81 + )
82 + return fake_sampling_metadata
83 +
84 +
85 +def _generate_min_token_penalties_and_stop_tokens(
86 + num_output_tokens: int, batch_size: int, vocab_size: int,
87 + batch_indices_for_min_token_penalty: List[int]
88 +) -> Tuple[List[int], List[Set[int]]]:
89 + """
90 + Generates and returns a list of minimum token penalties (`min_tokens`)
91 + and a corresponding list of stop token IDs (`stop_token_ids`) for each
92 + batch.
93 +
94 + If a batch index is included in `batch_indices_for_min_token_penalty`,
95 + a higher `min_tokens` value is assigned (within a randomized range),
96 + and a random set of stop token IDs is created. Otherwise, a lower
97 + `min_tokens` value is assigned, and the stop token IDs set is empty.
98 + """
99 + stop_token_ids: List[Set[int]] = []
100 + min_tokens: List[int] = []
101 + for index in range(batch_size):
102 + if index in batch_indices_for_min_token_penalty:
103 + min_tokens.append(
104 + np.random.randint(num_output_tokens + 1,
105 + 2 * num_output_tokens))
106 + stop_token_ids.append(
107 + set(
108 + np.random.randint(0, vocab_size - 1)
109 + for _ in range(np.random.randint(0, vocab_size))))
110 +
111 + else:
112 + min_tokens.append(np.random.randint(0, num_output_tokens))
113 + stop_token_ids.append(set())
114 + return (min_tokens, stop_token_ids)
115 +
116 +
117 +def _create_weighted_output_token_list(
118 + batch_size: int,
119 + vocab_size: int) -> Tuple[List[List[int]], List[List[int]]]:
120 + """
121 + Creates an output token list where each token occurs a distinct
122 + number of times.
123 +
124 + For each batch, a random subset of token IDs is selected from the
125 + vocabulary. The selected tokens are then added to the output token
126 + list, each with a different frequency.
127 +
128 + Returns:
129 + Tuple[List[List[int]], List[List[int]]]:
130 + - The first element is the output token list, where each sublist
131 + corresponds to a batch and contains tokens with weighted
132 + frequencies.
133 + - The second element is a list of distinct token IDs for each
134 + batch, ordered by their frequency in the corresponding output
135 + list.
136 + """
137 + output_token_ids: List[List[int]] = []
138 + sorted_token_ids_in_output: List[List[int]] = []
139 + for _ in range(batch_size):
140 + distinct_token_ids = np.random.choice(vocab_size,
141 + size=np.random.randint(1, 10),
142 + replace=False).tolist()
143 + sorted_token_ids_in_output.append(distinct_token_ids)
144 + output_token_ids_for_batch = []
145 + for index, token_id in enumerate(distinct_token_ids):
146 + output_token_ids_for_batch.extend(
147 + [token_id for _ in range(index + 1)])
148 + output_token_ids.append(output_token_ids_for_batch)
149 + return (output_token_ids, sorted_token_ids_in_output)
150 +
151 +
152 +@pytest.mark.parametrize("device", CUDA_DEVICES)
153 +@pytest.mark.parametrize("batch_size", [1, 2, 32])
154 +def test_sampler_min_tokens_penalty(device: str, batch_size: int):
155 + """
156 + Tests that if the number of output tokens is less than
157 + SamplingParams.min_tokens then we will set the logits for
158 + the stop token ids to -inf.
159 + """
160 + torch.set_default_device(device)
161 + fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
162 + sampling_metadata = _create_default_sampling_metadata(
163 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
164 + batch_indices_for_min_token_penalty = np.random.randint(
165 + 0, batch_size - 1, size=np.random.randint(0, batch_size)).tolist()
166 + min_tokens, stop_token_ids = _generate_min_token_penalties_and_stop_tokens(
167 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE,
168 + batch_indices_for_min_token_penalty)
169 + sampling_metadata.min_tokens = min_tokens
170 + sampling_metadata.stop_token_ids = stop_token_ids
171 + sampler = Sampler()
172 + sampler_output = sampler(fake_logits, sampling_metadata)
173 + for batch_idx in range(batch_size):
174 + for vocab in range(VOCAB_SIZE):
175 + # Verify that the logprobs for stop token ids is set
176 + # to -inf.
177 + logprob_index = torch.where(
178 + sampler_output.logprob_token_ids[batch_idx] ==
179 + vocab)[0].item()
180 + if vocab in stop_token_ids[batch_idx]:
181 + assert sampler_output.logprobs[batch_idx][
182 + logprob_index] == -float("inf")
183 + else:
184 + assert sampler_output.logprobs[batch_idx][
185 + logprob_index] != -float("inf")
186 +
187 +
188 +@pytest.mark.parametrize("device", CUDA_DEVICES)
189 +@pytest.mark.parametrize("batch_size", [1, 2, 32])
190 +@pytest.mark.parametrize("presence_penalty", [-2.0, 2.0])
191 +def test_sampler_presence_penalty(device: str, batch_size: int,
192 + presence_penalty: float):
193 + """
194 + Test to verify that if presence penalty is enabled then tokens
195 + are penalized as per their presence in the existing output.
196 + """
197 + torch.set_default_device(device)
198 + # Create fake logits where each token is assigned the same
199 + # logit value.
200 + fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
201 + sampling_metadata = _create_default_sampling_metadata(
202 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
203 + output_token_ids = sampling_metadata.output_token_ids
204 + sampling_metadata.presence_penalties = _create_penalty_tensor(
205 + batch_size, presence_penalty, torch.device(device))
206 + sampling_metadata.no_penalties = False
207 + sampler = Sampler()
208 + sampler_output = sampler(fake_logits, sampling_metadata)
209 + for batch_idx in range(batch_size):
210 + # The logprobs in the SamplerOutput are arranged in descending order.
211 + # Since all tokens initially have the same logprobs, the non-penalized
212 + # tokens will appear at the beginning, while the penalized tokens
213 + # will appear at the end of the list.
214 + penalized_token_id = sampler_output.logprob_token_ids[batch_idx][
215 + VOCAB_SIZE - 1]
216 + penalized_log_prod = sampler_output.logprobs[batch_idx][VOCAB_SIZE - 1]
217 + non_penalized_token_id = sampler_output.logprob_token_ids[batch_idx][0]
218 + non_penalized_log_prod = sampler_output.logprobs[batch_idx][0]
219 + assert non_penalized_log_prod > penalized_log_prod
220 + if presence_penalty > 0:
221 + # If `presence_penalty` is set to a value greater than 0, it
222 + # indicates a preference for new tokens over those already
223 + # present in the output.
224 + # Verify that the penalized token ID exists in the output, while the
225 + # non-penalized token ID does not.
226 + assert penalized_token_id in output_token_ids[batch_idx]
227 + assert non_penalized_token_id not in output_token_ids[batch_idx]
228 + elif presence_penalty < 0:
229 + # If `presence_penalty` is set to a value less than 0, it indicates
230 + # a preference for existing tokens over new ones. Verify that the
231 + # non-penalized token ID exists in the output, while the penalized
232 + # token ID does not.
233 + assert non_penalized_token_id in output_token_ids[batch_idx]
234 + assert penalized_token_id not in output_token_ids[batch_idx]
235 +
236 +
237 +@pytest.mark.parametrize("device", CUDA_DEVICES)
238 +@pytest.mark.parametrize("batch_size", [1, 2, 32])
239 +@pytest.mark.parametrize("frequency_penalty", [-2.0, 2.0])
240 +def test_sampler_frequency_penalty(device: str, batch_size: int,
241 + frequency_penalty: float):
242 + """
243 + Test to verify that if frequency penalty is enabled then tokens are
244 + penalized as per their frequency of occurrence.
245 + """
246 + torch.set_default_device(device)
247 + # Create fake logits where each token is assigned the same
248 + # logit value.
249 + fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
250 + sampling_metadata = _create_default_sampling_metadata(
251 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
252 + sampling_metadata.frequency_penalties = _create_penalty_tensor(
253 + batch_size, frequency_penalty, torch.device(device))
254 + output_token_ids, sorted_token_ids_in_output = \
255 + _create_weighted_output_token_list(batch_size, VOCAB_SIZE)
256 + sampling_metadata.output_token_ids = output_token_ids
257 + sampling_metadata.no_penalties = False
258 + sampler = Sampler()
259 + sampler_output = sampler(fake_logits, sampling_metadata)
260 + for batch_idx in range(batch_size):
261 + logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
262 + non_penalized_token_id = logprobs_token_ids[0]
263 + penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
264 + distinct_sorted_token_ids_in_output = \
265 + sorted_token_ids_in_output[batch_idx]
266 + most_frequent_token_id = distinct_sorted_token_ids_in_output[
267 + len(distinct_sorted_token_ids_in_output) - 1]
268 + if frequency_penalty > 0:
269 + # If `frequency_penalty` is set to > 0, it indicates
270 + # a preference for new tokens over existing ones. Verify that the
271 + # non-penalized token ID is not present in the output, while the
272 + # most penalized token is the one that occurs most frequently in
273 + # the output.
274 + assert non_penalized_token_id \
275 + not in distinct_sorted_token_ids_in_output
276 + assert penalized_token_id == most_frequent_token_id
277 + elif frequency_penalty < 0:
278 + # If `frequency_penalty` is set to < 0, it indicates
279 + # a preference for existing tokens over new ones. Verify that the
280 + # non-penalized token ID is the one that occurs most frequently
281 + # in the output, while the penalized token ID is one that has not
282 + # yet appeared.
283 + assert non_penalized_token_id == most_frequent_token_id
284 + assert penalized_token_id \
285 + not in distinct_sorted_token_ids_in_output
286 +
287 +
288 +@pytest.mark.parametrize("device", CUDA_DEVICES)
289 +@pytest.mark.parametrize("batch_size", [1, 2, 32])
290 +@pytest.mark.parametrize("repetition_penalty", [0.1, 1.9])
291 +def test_sampler_repetition_penalty(device: str, batch_size: int,
292 + repetition_penalty: float):
293 + """
294 + Test to verify that when the repetition penalty is enabled, tokens
295 + are penalized based on their presence in the prompt or the existing
296 + output.
297 + """
298 + torch.set_default_device(device)
299 + # Create fake logits where each token is assigned the same
300 + # logit value.
301 + fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
302 + sampling_metadata = _create_default_sampling_metadata(
303 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
304 + sampling_metadata.repetition_penalties = _create_penalty_tensor(
305 + batch_size, repetition_penalty, torch.device(device))
306 + sampling_metadata.no_penalties = False
307 + sampler = Sampler()
308 + sampler_output = sampler(fake_logits, sampling_metadata)
309 + for batch_idx in range(batch_size):
310 + logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
311 + non_penalized_token_id = logprobs_token_ids[0]
312 + penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
313 + prompt_tokens = sampling_metadata.prompt_token_ids[
314 + batch_idx][:].tolist()
315 + output_tokens = sampling_metadata.output_token_ids[batch_idx]
316 + if repetition_penalty > 1.0:
317 + # If `repetition_penalty` > 1.0, verify that the non-penalized
318 + # token ID has not been seen before, while the penalized token ID
319 + # exists either in the prompt or the output.
320 + assert (non_penalized_token_id not in prompt_tokens and \
321 + non_penalized_token_id not in output_tokens)
322 + assert (penalized_token_id in prompt_tokens or \
323 + penalized_token_id in output_tokens)
324 + elif repetition_penalty < 1.0:
325 + # If `repetition_penalty` < 1.0, verify that the penalized
326 + # token ID has not been seen before, while the non-penalized
327 + # token ID exists either in the prompt or the output.
328 + assert (penalized_token_id not in prompt_tokens and \
329 + penalized_token_id not in output_tokens)
330 + assert (non_penalized_token_id in prompt_tokens or \
331 + non_penalized_token_id in output_tokens)
tests/v1/worker/__init__.py
tests/v1/worker/test_gpu_input_batch.py new
+224
@@ -0,0 +1,224 @@
1 +from typing import Dict, List, Set, Tuple
2 +
3 +import numpy as np
4 +import pytest
5 +import torch
6 +
7 +from vllm.sampling_params import SamplingParams
8 +from vllm.utils import is_pin_memory_available, make_tensor_with_pad
9 +from vllm.v1.sample.metadata import SamplingMetadata
10 +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
11 +
12 +VOCAB_SIZE = 1024
13 +NUM_OUTPUT_TOKENS = 20
14 +MAX_PROMPT_SIZE = 100
15 +CUDA_DEVICES = [
16 + f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
17 +]
18 +MAX_NUM_PROMPT_TOKENS = 64
19 +
20 +
21 +def _remove_requests(
22 + input_batch: InputBatch, batch_size: int,
23 + reqs: List[CachedRequestState]) -> Tuple[Set[str], List[int]]:
24 + """
25 + Remove some requests randomly from the batch and returns a Tuple
26 + of 1) set of request removed 2) indices of the requests removed
27 + ordered in descending order
28 + """
29 +
30 + num_reqs_to_remove = np.random.randint(0, batch_size)
31 + req_indices_to_remove: Set[int] = set()
32 + for _ in range(num_reqs_to_remove):
33 + req_index_to_remove = np.random.randint(0, batch_size)
34 + req_indices_to_remove.add(req_index_to_remove)
35 +
36 + req_indices_to_remove_list = list(req_indices_to_remove)
37 + req_indices_to_remove_list.sort(reverse=True)
38 + req_ids_to_remove: Set[str] = set()
39 + for index in req_indices_to_remove:
40 + input_batch.remove_request(reqs[index].req_id)
41 + req_ids_to_remove.add(reqs[index].req_id)
42 + return (req_ids_to_remove, req_indices_to_remove_list)
43 +
44 +
45 +def _construct_expected_sampling_metadata(
46 + reqs: List[CachedRequestState], req_ids_retained: Set[int],
47 + req_id_index_in_input_batch: Dict[str, int],
48 + device: torch.device) -> SamplingMetadata:
49 + """
50 + Constructs and returns the expected SamplingMetadata for this
51 + batch.
52 + """
53 + num_reqs = len(req_ids_retained)
54 + output_token_ids: List[List[int]] = [list() for _ in range(num_reqs)]
55 + prompt_token_ids: List[List[int]] = [list() for _ in range(num_reqs)]
56 + presence_penalties = [0.0 for _ in range(num_reqs)]
57 + frequency_penalties = [0.0 for _ in range(num_reqs)]
58 + repetition_penalties = [1.0 for _ in range(num_reqs)]
59 + top_k = [0 for _ in range(num_reqs)]
60 + top_p = [0.0 for _ in range(num_reqs)]
61 + temperature = [0.0 for _ in range(num_reqs)]
62 + stop_token_ids: List[Set[int]] = [set() for _ in range(num_reqs)]
63 + min_tokens = [0 for _ in range(num_reqs)]
64 + for req in reqs:
65 + if req.req_id not in req_ids_retained:
66 + continue
67 + index_in_input_batch = req_id_index_in_input_batch[req.req_id]
68 + output_token_ids[index_in_input_batch] = req.output_token_ids
69 + prompt_token_ids[index_in_input_batch] = req.prompt_token_ids
70 + presence_penalties[
71 + index_in_input_batch] = req.sampling_params.presence_penalty
72 + frequency_penalties[
73 + index_in_input_batch] = req.sampling_params.frequency_penalty
74 + repetition_penalties[
75 + index_in_input_batch] = req.sampling_params.repetition_penalty
76 + top_k[index_in_input_batch] = req.sampling_params.top_k
77 + top_p[index_in_input_batch] = req.sampling_params.top_p
78 + temperature[index_in_input_batch] = req.sampling_params.temperature
79 + stop_token_ids[
80 + index_in_input_batch] = req.sampling_params.all_stop_token_ids
81 + min_tokens[index_in_input_batch] = req.sampling_params.min_tokens
82 +
83 +
84 + return SamplingMetadata(
85 + temperature=torch.tensor(temperature, dtype=torch.float, device=device),
86 + all_greedy=False,
87 + all_random=True,
88 + top_p=torch.tensor(top_p, dtype=torch.float, device=device),
89 + top_k=torch.tensor(top_k, dtype=torch.int, device=device),
90 + no_top_p=all(x == 1.0 for x in top_p),
91 + no_top_k=all(x == 0 for x in top_k),
92 + generators={},
93 + max_num_logprobs=0,
94 + prompt_token_ids= make_tensor_with_pad(
95 + prompt_token_ids,
96 + pad=VOCAB_SIZE,
97 + device=torch.device(device),
98 + dtype=torch.int64,
99 + ),
100 + frequency_penalties=torch.tensor(
101 + frequency_penalties, dtype=torch.float,
102 + device=device),
103 + presence_penalties=torch.tensor(
104 + presence_penalties, dtype=torch.float,
105 + device=device),
106 + repetition_penalties=torch.tensor(
107 + repetition_penalties, dtype=torch.float,
108 + device=device),
109 + output_token_ids=output_token_ids,
110 + min_tokens=min_tokens,
111 + stop_token_ids=stop_token_ids,
112 + no_penalties=(all(x ==0 for x in presence_penalties) and \
113 + all(x ==0 for x in frequency_penalties) and \
114 + all(x ==1 for x in repetition_penalties))
115 + )
116 +
117 +
118 +def _create_sampling_params():
119 + return SamplingParams(top_k=np.random.randint(1, 10),
120 + top_p=np.random.uniform(0.0, 1.0),
121 + presence_penalty=np.random.uniform(-2.0, 2.0),
122 + repetition_penalty=np.random.uniform(0.0, 2.0),
123 + frequency_penalty=np.random.uniform(-2.0, 2.0),
124 + min_tokens=np.random.randint(1, 10),
125 + stop_token_ids=[
126 + np.random.randint(0, VOCAB_SIZE)
127 + for _ in range(np.random.randint(10))
128 + ])
129 +
130 +
131 +def _construct_cached_request_state(req_id_suffix: int):
132 + prompt_token_ids = [
133 + np.random.randint(0, VOCAB_SIZE)
134 + for _ in range(np.random.randint(0, MAX_PROMPT_SIZE))
135 + ]
136 + output_token_ids = [
137 + np.random.randint(0, VOCAB_SIZE)
138 + for _ in range(np.random.randint(0, NUM_OUTPUT_TOKENS))
139 + ]
140 + return CachedRequestState(req_id=f"req_id_{req_id_suffix}",
141 + prompt_token_ids=prompt_token_ids,
142 + prompt=None,
143 + sampling_params=_create_sampling_params(),
144 + mm_inputs=[],
145 + mm_positions=[],
146 + block_ids=[],
147 + generator=None,
148 + num_computed_tokens=len(output_token_ids),
149 + output_token_ids=output_token_ids)
150 +
151 +
152 +@pytest.mark.parametrize("device", CUDA_DEVICES)
153 +@pytest.mark.parametrize("batch_size", [1, 2, 32, 64])
154 +def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
155 + """
156 + Tests the logic for managing sampling metadata in the InputBatch.
157 +
158 + This test involves adding a set of requests to the InputBatch,
159 + followed by removing a subset of them. Afterward, the batch is compacted,
160 + and the `make_sampling_metadata` method is invoked on the batch. The
161 + output of `make_sampling_metadata` is then compared against the expected
162 + results to ensure correctness.
163 + """
164 + input_batch: InputBatch = InputBatch(max_num_reqs=batch_size,
165 + max_model_len=1024,
166 + max_num_blocks_per_req=10,
167 + device=torch.device(device),
168 + pin_memory=is_pin_memory_available(),
169 + vocab_size=1024)
170 + reqs: List[CachedRequestState] = []
171 + req_id_reqs = {}
172 + req_id_output_token_ids = {}
173 + # Add requests
174 + for req_index in range(batch_size):
175 + req: CachedRequestState = _construct_cached_request_state(req_index)
176 + input_batch.add_request(req, req_index)
177 + reqs.append(req)
178 + req_id_reqs[req.req_id] = req
179 + req_id_output_token_ids[req.req_id] = req.output_token_ids
180 +
181 + # Remove some requests
182 + req_ids_to_remove, req_indices_to_remove = _remove_requests(
183 + input_batch, batch_size, reqs)
184 + req_ids_retained = set(req_id_reqs.keys()) - req_ids_to_remove
185 +
186 + # Compact the input batch
187 + input_batch.condense(req_indices_to_remove)
188 +
189 + # Generate the sampling metadata
190 + sampling_metadata = input_batch.make_sampling_metadata(
191 + req_id_output_token_ids, skip_copy=False)
192 +
193 + # Create expected output.
194 + expected_sampling_metadata = _construct_expected_sampling_metadata(
195 + reqs,
196 + req_ids_retained,
197 + input_batch.req_id_to_index,
198 + device=torch.device(device))
199 +
200 + # Assert the actual and expected output.
201 + assert torch.allclose(expected_sampling_metadata.temperature,
202 + sampling_metadata.temperature)
203 + assert torch.allclose(expected_sampling_metadata.top_p,
204 + sampling_metadata.top_p)
205 + assert torch.allclose(expected_sampling_metadata.top_k,
206 + sampling_metadata.top_k)
207 + assert torch.allclose(expected_sampling_metadata.frequency_penalties,
208 + sampling_metadata.frequency_penalties)
209 + assert torch.allclose(expected_sampling_metadata.presence_penalties,
210 + sampling_metadata.presence_penalties)
211 + assert torch.allclose(expected_sampling_metadata.repetition_penalties,
212 + sampling_metadata.repetition_penalties)
213 + assert torch.allclose(expected_sampling_metadata.prompt_token_ids,
214 + sampling_metadata.prompt_token_ids)
215 + assert (expected_sampling_metadata.output_token_ids ==
216 + sampling_metadata.output_token_ids)
217 + assert (
218 + expected_sampling_metadata.min_tokens == sampling_metadata.min_tokens)
219 + assert (expected_sampling_metadata.stop_token_ids ==
220 + sampling_metadata.stop_token_ids)
221 + assert (expected_sampling_metadata.no_penalties ==
222 + sampling_metadata.no_penalties)
223 + assert (expected_sampling_metadata.no_top_p == sampling_metadata.no_top_p)
224 + assert (expected_sampling_metadata.no_top_k == sampling_metadata.no_top_k)
vllm/model_executor/layers/sampler.py
+6 -45
@@ -11,6 +11,7 @@ import torch
11 import torch.nn as nn
12
13 import vllm.envs as envs
14 +from vllm.model_executor.layers.utils import apply_penalties
15 from vllm.model_executor.sampling_metadata import (SamplingMetadata,
16 SamplingTensors,
17 SequenceGroupToSample)
@@ -258,11 +259,11 @@ class Sampler(nn.Module):
259
260 # Apply presence and frequency penalties.
261 if do_penalties:
261 - logits = _apply_penalties(logits, sampling_tensors.prompt_tokens,
262 - sampling_tensors.output_tokens,
263 - sampling_tensors.presence_penalties,
264 - sampling_tensors.frequency_penalties,
265 - sampling_tensors.repetition_penalties)
262 + logits = apply_penalties(logits, sampling_tensors.prompt_tokens,
263 + sampling_tensors.output_tokens,
264 + sampling_tensors.presence_penalties,
265 + sampling_tensors.frequency_penalties,
266 + sampling_tensors.repetition_penalties)
267
268 # Use float32 to apply temperature scaling.
269 # Use in-place division to avoid creating a new tensor.
@@ -336,23 +337,6 @@ class Sampler(nn.Module):
337 return self.should_modify_greedy_probs_inplace
338
339
339 -def _get_bin_counts_and_mask(
340 - tokens: torch.Tensor,
341 - vocab_size: int,
342 - num_seqs: int,
343 -) -> Tuple[torch.Tensor, torch.Tensor]:
344 - # Compute the bin counts for the tokens.
345 - # vocab_size + 1 for padding.
346 - bin_counts = torch.zeros((num_seqs, vocab_size + 1),
347 - dtype=torch.long,
348 - device=tokens.device)
349 - bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens))
350 - bin_counts = bin_counts[:, :vocab_size]
351 - mask = bin_counts > 0
352 -
353 - return bin_counts, mask
354 -
355 -
340 def _apply_min_tokens_penalty(
341 logits: torch.Tensor,
342 sampling_metadata: SamplingMetadata,
@@ -400,29 +384,6 @@ def _apply_min_tokens_penalty(
384 return logits
385
386
403 -def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
404 - output_tokens_tensor: torch.Tensor,
405 - presence_penalties: torch.Tensor,
406 - frequency_penalties: torch.Tensor,
407 - repetition_penalties: torch.Tensor) -> torch.Tensor:
408 - num_seqs, vocab_size = logits.shape
409 - _, prompt_mask = _get_bin_counts_and_mask(prompt_tokens_tensor, vocab_size,
410 - num_seqs)
411 - output_bin_counts, output_mask = _get_bin_counts_and_mask(
412 - output_tokens_tensor, vocab_size, num_seqs)
413 -
414 - repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
415 - repetition_penalties[~(prompt_mask | output_mask)] = 1.0
416 - logits = torch.where(logits > 0, logits / repetition_penalties,
417 - logits * repetition_penalties)
418 -
419 - # We follow the definition in OpenAI API.
420 - # Refer to https://platform.openai.com/docs/api-reference/parameter-details
421 - logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts
422 - logits -= presence_penalties.unsqueeze_(dim=1) * output_mask
423 - return logits
424 -
425 -
387 def _apply_top_k_top_p(
388 logits: torch.Tensor,
389 p: torch.Tensor,
vllm/model_executor/layers/utils.py new
+57
@@ -0,0 +1,57 @@
1 +"""Utility methods for model layers."""
2 +from typing import Tuple
3 +
4 +import torch
5 +
6 +
7 +def get_token_bin_counts_and_mask(
8 + tokens: torch.Tensor,
9 + vocab_size: int,
10 + num_seqs: int,
11 +) -> Tuple[torch.Tensor, torch.Tensor]:
12 + # Compute the bin counts for the tokens.
13 + # vocab_size + 1 for padding.
14 + bin_counts = torch.zeros((num_seqs, vocab_size + 1),
15 + dtype=torch.long,
16 + device=tokens.device)
17 + bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens))
18 + bin_counts = bin_counts[:, :vocab_size]
19 + mask = bin_counts > 0
20 +
21 + return bin_counts, mask
22 +
23 +
24 +def apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
25 + output_tokens_tensor: torch.Tensor,
26 + presence_penalties: torch.Tensor,
27 + frequency_penalties: torch.Tensor,
28 + repetition_penalties: torch.Tensor) -> torch.Tensor:
29 + """
30 + Applies penalties in place to the logits tensor
31 + logits : The input logits tensor of shape [num_seqs, vocab_size]
32 + prompt_tokens_tensor: A tensor containing the prompt tokens. The prompts
33 + are padded to the maximum prompt length within the batch using
34 + `vocab_size` as the padding value. The value `vocab_size` is used
35 + for padding because it does not correspond to any valid token ID
36 + in the vocabulary.
37 + output_tokens_tensor: The output tokens tensor.
38 + presence_penalties: The presence penalties of shape (num_seqs, )
39 + frequency_penalties: The frequency penalties of shape (num_seqs, )
40 + repetition_penalties: The repetition penalties of shape (num_seqs, )
41 + """
42 + num_seqs, vocab_size = logits.shape
43 + _, prompt_mask = get_token_bin_counts_and_mask(prompt_tokens_tensor,
44 + vocab_size, num_seqs)
45 + output_bin_counts, output_mask = get_token_bin_counts_and_mask(
46 + output_tokens_tensor, vocab_size, num_seqs)
47 + repetition_penalties = repetition_penalties.unsqueeze_(dim=1).repeat(
48 + 1, vocab_size)
49 + logits[logits > 0] /= torch.where(prompt_mask | output_mask,
50 + repetition_penalties, 1.0)[logits > 0]
51 + logits[logits <= 0] *= torch.where(prompt_mask | output_mask,
52 + repetition_penalties, 1.0)[logits <= 0]
53 + # We follow the definition in OpenAI API.
54 + # Refer to https://platform.openai.com/docs/api-reference/parameter-details
55 + logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts
56 + logits -= presence_penalties.unsqueeze_(dim=1) * output_mask
57 + return logits
vllm/v1/sample/metadata.py
+11 -1
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import Dict
2 +from typing import Dict, List, Optional, Set
3
4 import torch
5
@@ -19,3 +19,13 @@ class SamplingMetadata:
19 generators: Dict[int, torch.Generator]
20
21 max_num_logprobs: int
22 +
23 + no_penalties: bool
24 + prompt_token_ids: Optional[torch.Tensor]
25 + frequency_penalties: torch.Tensor
26 + presence_penalties: torch.Tensor
27 + repetition_penalties: torch.Tensor
28 +
29 + output_token_ids: List[List[int]]
30 + min_tokens: List[int]
31 + stop_token_ids: List[Set[int]]
vllm/v1/sample/sampler.py
+63 -2
@@ -1,9 +1,11 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 -from typing import Dict
2 +from typing import Dict, List, Set, Tuple
3
4 import torch
5 import torch.nn as nn
6
7 +from vllm.model_executor.layers.utils import apply_penalties
8 +from vllm.utils import is_pin_memory_available, make_tensor_with_pad
9 from vllm.v1.outputs import SamplerOutput
10 from vllm.v1.sample.metadata import SamplingMetadata
11
@@ -17,9 +19,18 @@ class Sampler(nn.Module):
19 logits: torch.Tensor,
20 sampling_metadata: SamplingMetadata,
21 ) -> SamplerOutput:
22 + _apply_min_token_penalties(logits, sampling_metadata.output_token_ids,
23 + sampling_metadata.stop_token_ids,
24 + sampling_metadata.min_tokens)
25 + if not sampling_metadata.no_penalties:
26 + assert sampling_metadata.prompt_token_ids is not None
27 + _apply_penalties(logits, sampling_metadata.prompt_token_ids,
28 + sampling_metadata.presence_penalties,
29 + sampling_metadata.frequency_penalties,
30 + sampling_metadata.repetition_penalties,
31 + sampling_metadata.output_token_ids)
32 logits = self.apply_temperature(logits, sampling_metadata.temperature)
33 logits = self.apply_top_k_top_p(logits, sampling_metadata)
22 -
34 probs = self.get_probs(logits)
35 sampled = self.sample(probs, sampling_metadata)
36 # Use int32 to reduce the tensor size.
@@ -157,3 +168,53 @@ def _apply_top_k_top_p(
168 # Re-sort the probabilities.
169 logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
170 return logits
171 +
172 +
173 +def _apply_min_token_penalties(logits: torch.Tensor,
174 + output_token_ids: List[List[int]],
175 + stop_token_ids: List[Set[int]],
176 + min_tokens: List[int]):
177 + """
178 + Applies minimum token penalty by setting the logits of the stop tokens
179 + to -inf.
180 + """
181 + min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
182 + for index, min_token in enumerate(min_tokens):
183 + if (len(output_token_ids[index]) < min_token):
184 + for stop_token_id in stop_token_ids[index]:
185 + min_tokens_logits_to_penalize.append((index, stop_token_id))
186 + if min_tokens_logits_to_penalize:
187 + logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
188 +
189 +
190 +def _apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
191 + presence_penalties: torch.Tensor,
192 + frequency_penalties: torch.Tensor,
193 + repetition_penalties: torch.Tensor,
194 + output_token_ids: List[List[int]]):
195 + """
196 + Applies presence, frequency and repetition penalties to the logits.
197 + """
198 + _, vocab_size = logits.shape
199 + output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
200 + logits.device)
201 + return apply_penalties(logits, prompt_token_ids, output_tokens_t,
202 + presence_penalties, frequency_penalties,
203 + repetition_penalties)
204 +
205 +
206 +def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
207 + device: torch.device) -> torch.Tensor:
208 + """
209 + Convert the different list data structures to tensors.
210 + """
211 + output_tokens_tensor = make_tensor_with_pad(
212 + output_token_ids,
213 + # Use the value of vocab_size as a pad since we don't have a
214 + # token_id of this value.
215 + pad=vocab_size,
216 + device="cpu",
217 + dtype=torch.int64,
218 + pin_memory=is_pin_memory_available(),
219 + )
220 + return output_tokens_tensor.to(device, non_blocking=True)
vllm/v1/worker/gpu_input_batch.py
+142
@@ -43,12 +43,14 @@ class InputBatch:
43 max_num_blocks_per_req: int,
44 device: torch.device,
45 pin_memory: bool,
46 + vocab_size: int,
47 ):
48 self.max_num_reqs = max_num_reqs
49 self.max_model_len = max_model_len
50 self.max_num_blocks_per_req = max_num_blocks_per_req
51 self.device = device
52 self.pin_memory = pin_memory
53 + self.vocab_size = vocab_size
54
55 self.req_ids: List[Optional[str]] = [None] * max_num_reqs
56 self.req_id_to_index: Dict[str, int] = {}
@@ -63,6 +65,7 @@ class InputBatch:
65 )
66 self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
67 self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
68 + self.num_prompt_tokens = np.zeros(max_num_reqs, dtype=np.int32)
69
70 # Attention-related.
71 self.block_table = torch.zeros(
@@ -110,6 +113,50 @@ class InputBatch:
113 self.top_k_cpu = self.top_k_cpu_tensor.numpy()
114 self.top_k_reqs: Set[str] = set()
115
116 + # Frequency penalty related data structures
117 + self.frequency_penalties = torch.empty((max_num_reqs, ),
118 + dtype=torch.float,
119 + device=device)
120 + self.frequency_penalties_cpu_tensor = torch.empty(
121 + (max_num_reqs, ),
122 + dtype=torch.float,
123 + device="cpu",
124 + pin_memory=pin_memory)
125 + self.frequency_penalties_cpu = \
126 + self.frequency_penalties_cpu_tensor.numpy()
127 + self.frequency_penalties_reqs: Set[str] = set()
128 +
129 + # Presence penalty related data structures
130 + self.presence_penalties = torch.empty((max_num_reqs, ),
131 + dtype=torch.float,
132 + device=device)
133 + self.presence_penalties_cpu_tensor = torch.empty((max_num_reqs, ),
134 + dtype=torch.float,
135 + device="cpu",
136 + pin_memory=pin_memory)
137 + self.presence_penalties_cpu = \
138 + self.presence_penalties_cpu_tensor.numpy()
139 + self.presence_penalties_reqs: Set[str] = set()
140 +
141 + # Repetition penalty related data structures
142 + self.repetition_penalties = torch.empty((max_num_reqs, ),
143 + dtype=torch.float,
144 + device=device)
145 + self.repetition_penalties_cpu_tensor = torch.empty(
146 + (max_num_reqs, ),
147 + dtype=torch.float,
148 + device="cpu",
149 + pin_memory=pin_memory)
150 + self.repetition_penalties_cpu = \
151 + self.repetition_penalties_cpu_tensor.numpy()
152 + self.repetition_penalties_reqs: Set[str] = set()
153 +
154 + self.min_tokens: List[int] = [0] * max_num_reqs
155 + self.stop_token_ids: List[Set[int]] = [
156 + set() for _ in range(max_num_reqs)
157 + ]
158 + self.prompt_token_ids: Optional[torch.Tensor] = None
159 +
160 # req_index -> generator
161 # NOTE(woosuk): The indices of the requests that do not have their own
162 # generator should not be included in the dictionary.
@@ -133,6 +180,7 @@ class InputBatch:
180
181 # Copy the prompt token ids and output token ids.
182 num_prompt_tokens = len(request.prompt_token_ids)
183 + self.num_prompt_tokens[req_index] = num_prompt_tokens
184 self.token_ids_cpu[
185 req_index, :num_prompt_tokens] = request.prompt_token_ids
186 start_idx = num_prompt_tokens
@@ -157,6 +205,20 @@ class InputBatch:
205 self.top_k_cpu[req_index] = sampling_params.top_k
206 if sampling_params.top_k > 0:
207 self.top_k_reqs.add(req_id)
208 + self.frequency_penalties_cpu[req_index] = \
209 + sampling_params.frequency_penalty
210 + if sampling_params.frequency_penalty != 0.0:
211 + self.frequency_penalties_reqs.add(req_id)
212 + self.presence_penalties_cpu[req_index] = \
213 + sampling_params.presence_penalty
214 + if sampling_params.presence_penalty != 0.0:
215 + self.presence_penalties_reqs.add(req_id)
216 + self.repetition_penalties_cpu[req_index] = \
217 + sampling_params.repetition_penalty
218 + if sampling_params.repetition_penalty != 1.0:
219 + self.repetition_penalties_reqs.add(req_id)
220 + self.min_tokens[req_index] = sampling_params.min_tokens
221 + self.stop_token_ids[req_index] = sampling_params.all_stop_token_ids
222
223 # NOTE(woosuk): self.generators should not include the requests that
224 # do not have their own generator.
@@ -179,6 +241,9 @@ class InputBatch:
241 self.random_reqs.discard(req_id)
242 self.top_p_reqs.discard(req_id)
243 self.top_k_reqs.discard(req_id)
244 + self.frequency_penalties_reqs.discard(req_id)
245 + self.presence_penalties_reqs.discard(req_id)
246 + self.repetition_penalties_reqs.discard(req_id)
247 self.generators.pop(req_index, None)
248 self.num_logprobs.pop(req_id, None)
249 self.prompt_logprob_reqs.discard(req_id)
@@ -191,6 +256,9 @@ class InputBatch:
256 self.random_reqs.clear()
257 self.top_p_reqs.clear()
258 self.top_k_reqs.clear()
259 + self.frequency_penalties_reqs.clear()
260 + self.presence_penalties_reqs.clear()
261 + self.repetition_penalties_reqs.clear()
262 self.generators.clear()
263 self.num_logprobs.clear()
264 self.prompt_logprob_reqs.clear()
@@ -224,6 +292,8 @@ class InputBatch:
292 # block_table_cpu.
293 self.token_ids_cpu[empty_index] = self.token_ids_cpu[
294 last_req_index]
295 + self.num_prompt_tokens[empty_index] = \
296 + self.num_prompt_tokens[last_req_index]
297 self.num_computed_tokens_cpu[
298 empty_index] = self.num_computed_tokens_cpu[last_req_index]
299 self.block_table_cpu[empty_index] = self.block_table_cpu[
@@ -232,6 +302,15 @@ class InputBatch:
302 last_req_index]
303 self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
304 self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
305 + self.frequency_penalties_cpu[empty_index] = \
306 + self.frequency_penalties_cpu[last_req_index]
307 + self.presence_penalties_cpu[empty_index] = \
308 + self.presence_penalties_cpu[last_req_index]
309 + self.repetition_penalties_cpu[empty_index] = \
310 + self.repetition_penalties_cpu[last_req_index]
311 + self.min_tokens[empty_index] = self.min_tokens[last_req_index]
312 + self.stop_token_ids[empty_index] = \
313 + self.stop_token_ids[last_req_index]
314 generator = self.generators.pop(last_req_index, None)
315 if generator is not None:
316 self.generators[empty_index] = generator
@@ -241,6 +320,7 @@ class InputBatch:
320
321 def make_sampling_metadata(
322 self,
323 + req_id_output_token_ids: Dict[str, List[int]],
324 skip_copy: bool = False,
325 ) -> SamplingMetadata:
326 if not skip_copy:
@@ -250,6 +330,37 @@ class InputBatch:
330 self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
331 self.top_k[:self.num_reqs].copy_(
332 self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
333 + if not self.no_penalties:
334 + # Since syncing these tensors is expensive only copy them
335 + # if necessary i.e. if there are requests which require
336 + # penalties to be applied during sampling.
337 + self.frequency_penalties[:self.num_reqs].copy_(
338 + self.frequency_penalties_cpu_tensor[:self.num_reqs],
339 + non_blocking=True)
340 + self.presence_penalties[:self.num_reqs].copy_(
341 + self.presence_penalties_cpu_tensor[:self.num_reqs],
342 + non_blocking=True)
343 + self.repetition_penalties[:self.num_reqs].copy_(
344 + self.repetition_penalties_cpu_tensor[:self.num_reqs],
345 + non_blocking=True)
346 + # The prompt tokens are used only for applying penalties during
347 + # the sampling process. Hence copy these tensors only when
348 + # there are requests which need penalties to be applied.
349 + self.prompt_token_ids = self._make_prompt_token_ids_tensor()
350 +
351 + output_token_ids: List[List[int]] = []
352 +
353 + for req_id in self.req_ids[:self.num_reqs]:
354 + assert req_id is not None
355 + # Currently we create a tensor for output_token_ids from scratch
356 + # at each step. However, for the penalties computation what we
357 + # need is stats about the token ids present in the output. This
358 + # stats can be maintained incrementally instead of computing it
359 + # from scratch at each step.
360 + # TODO - Replace this with incremental update to output token
361 + # statistics.
362 + output_token_ids.append(req_id_output_token_ids[req_id])
363 +
364 return SamplingMetadata(
365 temperature=self.temperature[:self.num_reqs],
366 all_greedy=self.all_greedy,
@@ -260,8 +371,33 @@ class InputBatch:
371 no_top_k=self.no_top_k,
372 generators=self.generators,
373 max_num_logprobs=self.max_num_logprobs,
374 + prompt_token_ids=self.prompt_token_ids,
375 + frequency_penalties=self.frequency_penalties[:self.num_reqs],
376 + presence_penalties=self.presence_penalties[:self.num_reqs],
377 + repetition_penalties=self.repetition_penalties[:self.num_reqs],
378 + output_token_ids=output_token_ids,
379 + min_tokens=self.min_tokens[:self.num_reqs],
380 + stop_token_ids=self.stop_token_ids[:self.num_reqs],
381 + no_penalties=self.no_penalties,
382 )
383
384 + def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
385 + max_prompt_len = self.num_prompt_tokens[:self.num_reqs].max()
386 + prompt_token_ids_cpu_tensor = torch.empty(
387 + (self.num_reqs, max_prompt_len),
388 + device="cpu",
389 + dtype=torch.int64,
390 + pin_memory=self.pin_memory)
391 + prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
392 + prompt_token_ids[:] = (
393 + self.token_ids_cpu[:self.num_reqs, :max_prompt_len])
394 + # Use the value of vocab_size as a pad since we don't have a
395 + # token_id of this value.
396 + for i in range(self.num_reqs):
397 + prompt_token_ids[i, self.num_prompt_tokens[i]:] = self.vocab_size
398 + return prompt_token_ids_cpu_tensor.to(device=self.device,
399 + non_blocking=True)
400 +
401 @property
402 def num_reqs(self) -> int:
403 return len(self.req_id_to_index)
@@ -282,6 +418,12 @@ class InputBatch:
418 def no_top_k(self) -> bool:
419 return len(self.top_k_reqs) == 0
420
421 + @property
422 + def no_penalties(self) -> bool:
423 + return (len(self.presence_penalties_reqs) == 0
424 + and len(self.frequency_penalties_reqs) == 0
425 + and len(self.repetition_penalties_reqs) == 0)
426 +
427 @property
428 def max_num_logprobs(self) -> int:
429 return max(self.num_logprobs.values()) if self.num_logprobs else 0
vllm/v1/worker/gpu_model_runner.py
+7 -1
@@ -105,6 +105,7 @@ class GPUModelRunner:
105 max_num_blocks_per_req=self.max_num_blocks_per_req,
106 device=self.device,
107 pin_memory=self.pin_memory,
108 + vocab_size=model_config.get_vocab_size(),
109 )
110
111 self.use_cuda_graph = (self.vllm_config.compilation_config.level
@@ -383,7 +384,12 @@ class GPUModelRunner:
384 or scheduler_output.scheduled_resumed_reqs):
385 skip_copy = False
386 # Create the sampling metadata.
386 - sampling_metadata = self.input_batch.make_sampling_metadata(skip_copy)
387 + req_id_output_token_ids: Dict[str, List[int]] = \
388 + {req_id: req.output_token_ids \
389 + for req_id, req in self.requests.items()}
390 +
391 + sampling_metadata = self.input_batch.make_sampling_metadata(
392 + req_id_output_token_ids, skip_copy)
393 return sampling_metadata
394
395 def _execute_encoder(self, scheduler_output: "SchedulerOutput"):