A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1] Adding min tokens/repetition/presence/frequence penalties to V1 sampler (#10681)
Signed-off-by: Sourashis Roy <sroy@roblox.com> Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
sroy745 committed
Dec 26, 2024 at 02:02 UTC
dcb1a944d4cf95b4a7b3522ddf970e6d3a28b8b5
11 files changed
+879
-49
tests/v1/engine/test_engine_core.py
+38
@@ -139,3 +139,41 @@ def test_engine_core(monkeypatch):
139
engine_core.abort_requests([req2.request_id, req0.request_id])
140
assert len(engine_core.scheduler.waiting) == 0
141
assert len(engine_core.scheduler.running) == 0
142
+
143
+
144
+def test_engine_core_advanced_sampling(monkeypatch):
145
+ """
146
+ A basic end-to-end test to verify that the engine functions correctly
147
+ when additional sampling parameters, such as min_tokens and
148
+ presence_penalty, are set.
149
+ """
150
+ with monkeypatch.context() as m:
151
+ m.setenv("VLLM_USE_V1", "1")
152
+ """Setup the EngineCore."""
153
+ engine_args = EngineArgs(model=MODEL_NAME)
154
+ vllm_config = engine_args.create_engine_config(
155
+ usage_context=UsageContext.UNKNOWN_CONTEXT)
156
+ executor_class = AsyncLLM._get_executor_cls(vllm_config)
157
+
158
+ engine_core = EngineCore(vllm_config=vllm_config,
159
+ executor_class=executor_class,
160
+ usage_context=UsageContext.UNKNOWN_CONTEXT)
161
+ """Test basic request lifecycle."""
162
+ # First request.
163
+ request: EngineCoreRequest = make_request()
164
+ request.sampling_params = SamplingParams(
165
+ min_tokens=4,
166
+ presence_penalty=1.0,
167
+ frequency_penalty=1.0,
168
+ repetition_penalty=0.1,
169
+ stop_token_ids=[1001, 1002],
170
+ )
171
+ engine_core.add_request(request)
172
+ assert len(engine_core.scheduler.waiting) == 1
173
+ assert len(engine_core.scheduler.running) == 0
174
+ # Loop through until they are all done.
175
+ while len(engine_core.step()) > 0:
176
+ pass
177
+
178
+ assert len(engine_core.scheduler.waiting) == 0
179
+ assert len(engine_core.scheduler.running) == 0
tests/v1/sample/__init__.py
tests/v1/sample/test_sampler.py
new
+331
@@ -0,0 +1,331 @@
1
+from typing import List, Set, Tuple
2
+
3
+import numpy as np
4
+import pytest
5
+import torch
6
+
7
+from vllm.utils import make_tensor_with_pad
8
+from vllm.v1.sample.metadata import SamplingMetadata
9
+from vllm.v1.sample.sampler import Sampler
10
+
11
+VOCAB_SIZE = 1024
12
+NUM_OUTPUT_TOKENS = 20
13
+CUDA_DEVICES = [
14
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
15
+]
16
+MAX_NUM_PROMPT_TOKENS = 64
17
+
18
+
19
+def _create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor:
20
+ fake_logits = torch.full((batch_size, vocab_size), 1e-2, dtype=torch.float)
21
+ return fake_logits
22
+
23
+
24
+def _create_penalty_tensor(batch_size: int, penalty_value: float,
25
+ device: torch.device) -> torch.Tensor:
26
+ return torch.full((batch_size, ),
27
+ fill_value=penalty_value,
28
+ dtype=torch.float,
29
+ device=device)
30
+
31
+
32
+def _create_prompt_tokens_tensor(
33
+ prompt_token_ids: List[List[int]],
34
+ vocab_size: int,
35
+ device: torch.device,
36
+) -> torch.Tensor:
37
+ return make_tensor_with_pad(
38
+ prompt_token_ids,
39
+ pad=vocab_size,
40
+ device=device,
41
+ dtype=torch.int64,
42
+ pin_memory=False,
43
+ )
44
+
45
+
46
+def _create_default_sampling_metadata(
47
+ num_output_tokens: int,
48
+ batch_size: int,
49
+ vocab_size: int,
50
+ device: torch.device,
51
+) -> SamplingMetadata:
52
+ output_token_ids: List[List[int]] = []
53
+ prompt_token_ids: List[List[int]] = []
54
+ for _ in range(batch_size):
55
+ output_token_ids.append(
56
+ np.random.randint(0, vocab_size, size=num_output_tokens).tolist())
57
+ prompt_token_ids.append(
58
+ np.random.randint(0,
59
+ vocab_size,
60
+ size=np.random.randint(
61
+ 1, MAX_NUM_PROMPT_TOKENS)).tolist())
62
+ fake_sampling_metadata = SamplingMetadata(
63
+ temperature=torch.full((batch_size, ), 0.0),
64
+ all_greedy=True,
65
+ all_random=False,
66
+ top_p=torch.empty(batch_size, ),
67
+ top_k=torch.empty(batch_size, ),
68
+ no_top_p=True,
69
+ no_top_k=True,
70
+ generators={},
71
+ max_num_logprobs=VOCAB_SIZE,
72
+ prompt_token_ids=_create_prompt_tokens_tensor(prompt_token_ids,
73
+ vocab_size, device),
74
+ output_token_ids=output_token_ids,
75
+ frequency_penalties=_create_penalty_tensor(batch_size, 0.0, device),
76
+ presence_penalties=_create_penalty_tensor(batch_size, 0.0, device),
77
+ repetition_penalties=_create_penalty_tensor(batch_size, 1.0, device),
78
+ no_penalties=True,
79
+ min_tokens=[],
80
+ stop_token_ids=[],
81
+ )
82
+ return fake_sampling_metadata
83
+
84
+
85
+def _generate_min_token_penalties_and_stop_tokens(
86
+ num_output_tokens: int, batch_size: int, vocab_size: int,
87
+ batch_indices_for_min_token_penalty: List[int]
88
+) -> Tuple[List[int], List[Set[int]]]:
89
+ """
90
+ Generates and returns a list of minimum token penalties (`min_tokens`)
91
+ and a corresponding list of stop token IDs (`stop_token_ids`) for each
92
+ batch.
93
+
94
+ If a batch index is included in `batch_indices_for_min_token_penalty`,
95
+ a higher `min_tokens` value is assigned (within a randomized range),
96
+ and a random set of stop token IDs is created. Otherwise, a lower
97
+ `min_tokens` value is assigned, and the stop token IDs set is empty.
98
+ """
99
+ stop_token_ids: List[Set[int]] = []
100
+ min_tokens: List[int] = []
101
+ for index in range(batch_size):
102
+ if index in batch_indices_for_min_token_penalty:
103
+ min_tokens.append(
104
+ np.random.randint(num_output_tokens + 1,
105
+ 2 * num_output_tokens))
106
+ stop_token_ids.append(
107
+ set(
108
+ np.random.randint(0, vocab_size - 1)
109
+ for _ in range(np.random.randint(0, vocab_size))))
110
+
111
+ else:
112
+ min_tokens.append(np.random.randint(0, num_output_tokens))
113
+ stop_token_ids.append(set())
114
+ return (min_tokens, stop_token_ids)
115
+
116
+
117
+def _create_weighted_output_token_list(
118
+ batch_size: int,
119
+ vocab_size: int) -> Tuple[List[List[int]], List[List[int]]]:
120
+ """
121
+ Creates an output token list where each token occurs a distinct
122
+ number of times.
123
+
124
+ For each batch, a random subset of token IDs is selected from the
125
+ vocabulary. The selected tokens are then added to the output token
126
+ list, each with a different frequency.
127
+
128
+ Returns:
129
+ Tuple[List[List[int]], List[List[int]]]:
130
+ - The first element is the output token list, where each sublist
131
+ corresponds to a batch and contains tokens with weighted
132
+ frequencies.
133
+ - The second element is a list of distinct token IDs for each
134
+ batch, ordered by their frequency in the corresponding output
135
+ list.
136
+ """
137
+ output_token_ids: List[List[int]] = []
138
+ sorted_token_ids_in_output: List[List[int]] = []
139
+ for _ in range(batch_size):
140
+ distinct_token_ids = np.random.choice(vocab_size,
141
+ size=np.random.randint(1, 10),
142
+ replace=False).tolist()
143
+ sorted_token_ids_in_output.append(distinct_token_ids)
144
+ output_token_ids_for_batch = []
145
+ for index, token_id in enumerate(distinct_token_ids):
146
+ output_token_ids_for_batch.extend(
147
+ [token_id for _ in range(index + 1)])
148
+ output_token_ids.append(output_token_ids_for_batch)
149
+ return (output_token_ids, sorted_token_ids_in_output)
150
+
151
+
152
+@pytest.mark.parametrize("device", CUDA_DEVICES)
153
+@pytest.mark.parametrize("batch_size", [1, 2, 32])
154
+def test_sampler_min_tokens_penalty(device: str, batch_size: int):
155
+ """
156
+ Tests that if the number of output tokens is less than
157
+ SamplingParams.min_tokens then we will set the logits for
158
+ the stop token ids to -inf.
159
+ """
160
+ torch.set_default_device(device)
161
+ fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
162
+ sampling_metadata = _create_default_sampling_metadata(
163
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
164
+ batch_indices_for_min_token_penalty = np.random.randint(
165
+ 0, batch_size - 1, size=np.random.randint(0, batch_size)).tolist()
166
+ min_tokens, stop_token_ids = _generate_min_token_penalties_and_stop_tokens(
167
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE,
168
+ batch_indices_for_min_token_penalty)
169
+ sampling_metadata.min_tokens = min_tokens
170
+ sampling_metadata.stop_token_ids = stop_token_ids
171
+ sampler = Sampler()
172
+ sampler_output = sampler(fake_logits, sampling_metadata)
173
+ for batch_idx in range(batch_size):
174
+ for vocab in range(VOCAB_SIZE):
175
+ # Verify that the logprobs for stop token ids is set
176
+ # to -inf.
177
+ logprob_index = torch.where(
178
+ sampler_output.logprob_token_ids[batch_idx] ==
179
+ vocab)[0].item()
180
+ if vocab in stop_token_ids[batch_idx]:
181
+ assert sampler_output.logprobs[batch_idx][
182
+ logprob_index] == -float("inf")
183
+ else:
184
+ assert sampler_output.logprobs[batch_idx][
185
+ logprob_index] != -float("inf")
186
+
187
+
188
+@pytest.mark.parametrize("device", CUDA_DEVICES)
189
+@pytest.mark.parametrize("batch_size", [1, 2, 32])
190
+@pytest.mark.parametrize("presence_penalty", [-2.0, 2.0])
191
+def test_sampler_presence_penalty(device: str, batch_size: int,
192
+ presence_penalty: float):
193
+ """
194
+ Test to verify that if presence penalty is enabled then tokens
195
+ are penalized as per their presence in the existing output.
196
+ """
197
+ torch.set_default_device(device)
198
+ # Create fake logits where each token is assigned the same
199
+ # logit value.
200
+ fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
201
+ sampling_metadata = _create_default_sampling_metadata(
202
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
203
+ output_token_ids = sampling_metadata.output_token_ids
204
+ sampling_metadata.presence_penalties = _create_penalty_tensor(
205
+ batch_size, presence_penalty, torch.device(device))
206
+ sampling_metadata.no_penalties = False
207
+ sampler = Sampler()
208
+ sampler_output = sampler(fake_logits, sampling_metadata)
209
+ for batch_idx in range(batch_size):
210
+ # The logprobs in the SamplerOutput are arranged in descending order.
211
+ # Since all tokens initially have the same logprobs, the non-penalized
212
+ # tokens will appear at the beginning, while the penalized tokens
213
+ # will appear at the end of the list.
214
+ penalized_token_id = sampler_output.logprob_token_ids[batch_idx][
215
+ VOCAB_SIZE - 1]
216
+ penalized_log_prod = sampler_output.logprobs[batch_idx][VOCAB_SIZE - 1]
217
+ non_penalized_token_id = sampler_output.logprob_token_ids[batch_idx][0]
218
+ non_penalized_log_prod = sampler_output.logprobs[batch_idx][0]
219
+ assert non_penalized_log_prod > penalized_log_prod
220
+ if presence_penalty > 0:
221
+ # If `presence_penalty` is set to a value greater than 0, it
222
+ # indicates a preference for new tokens over those already
223
+ # present in the output.
224
+ # Verify that the penalized token ID exists in the output, while the
225
+ # non-penalized token ID does not.
226
+ assert penalized_token_id in output_token_ids[batch_idx]
227
+ assert non_penalized_token_id not in output_token_ids[batch_idx]
228
+ elif presence_penalty < 0:
229
+ # If `presence_penalty` is set to a value less than 0, it indicates
230
+ # a preference for existing tokens over new ones. Verify that the
231
+ # non-penalized token ID exists in the output, while the penalized
232
+ # token ID does not.
233
+ assert non_penalized_token_id in output_token_ids[batch_idx]
234
+ assert penalized_token_id not in output_token_ids[batch_idx]
235
+
236
+
237
+@pytest.mark.parametrize("device", CUDA_DEVICES)
238
+@pytest.mark.parametrize("batch_size", [1, 2, 32])
239
+@pytest.mark.parametrize("frequency_penalty", [-2.0, 2.0])
240
+def test_sampler_frequency_penalty(device: str, batch_size: int,
241
+ frequency_penalty: float):
242
+ """
243
+ Test to verify that if frequency penalty is enabled then tokens are
244
+ penalized as per their frequency of occurrence.
245
+ """
246
+ torch.set_default_device(device)
247
+ # Create fake logits where each token is assigned the same
248
+ # logit value.
249
+ fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
250
+ sampling_metadata = _create_default_sampling_metadata(
251
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
252
+ sampling_metadata.frequency_penalties = _create_penalty_tensor(
253
+ batch_size, frequency_penalty, torch.device(device))
254
+ output_token_ids, sorted_token_ids_in_output = \
255
+ _create_weighted_output_token_list(batch_size, VOCAB_SIZE)
256
+ sampling_metadata.output_token_ids = output_token_ids
257
+ sampling_metadata.no_penalties = False
258
+ sampler = Sampler()
259
+ sampler_output = sampler(fake_logits, sampling_metadata)
260
+ for batch_idx in range(batch_size):
261
+ logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
262
+ non_penalized_token_id = logprobs_token_ids[0]
263
+ penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
264
+ distinct_sorted_token_ids_in_output = \
265
+ sorted_token_ids_in_output[batch_idx]
266
+ most_frequent_token_id = distinct_sorted_token_ids_in_output[
267
+ len(distinct_sorted_token_ids_in_output) - 1]
268
+ if frequency_penalty > 0:
269
+ # If `frequency_penalty` is set to > 0, it indicates
270
+ # a preference for new tokens over existing ones. Verify that the
271
+ # non-penalized token ID is not present in the output, while the
272
+ # most penalized token is the one that occurs most frequently in
273
+ # the output.
274
+ assert non_penalized_token_id \
275
+ not in distinct_sorted_token_ids_in_output
276
+ assert penalized_token_id == most_frequent_token_id
277
+ elif frequency_penalty < 0:
278
+ # If `frequency_penalty` is set to < 0, it indicates
279
+ # a preference for existing tokens over new ones. Verify that the
280
+ # non-penalized token ID is the one that occurs most frequently
281
+ # in the output, while the penalized token ID is one that has not
282
+ # yet appeared.
283
+ assert non_penalized_token_id == most_frequent_token_id
284
+ assert penalized_token_id \
285
+ not in distinct_sorted_token_ids_in_output
286
+
287
+
288
+@pytest.mark.parametrize("device", CUDA_DEVICES)
289
+@pytest.mark.parametrize("batch_size", [1, 2, 32])
290
+@pytest.mark.parametrize("repetition_penalty", [0.1, 1.9])
291
+def test_sampler_repetition_penalty(device: str, batch_size: int,
292
+ repetition_penalty: float):
293
+ """
294
+ Test to verify that when the repetition penalty is enabled, tokens
295
+ are penalized based on their presence in the prompt or the existing
296
+ output.
297
+ """
298
+ torch.set_default_device(device)
299
+ # Create fake logits where each token is assigned the same
300
+ # logit value.
301
+ fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
302
+ sampling_metadata = _create_default_sampling_metadata(
303
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
304
+ sampling_metadata.repetition_penalties = _create_penalty_tensor(
305
+ batch_size, repetition_penalty, torch.device(device))
306
+ sampling_metadata.no_penalties = False
307
+ sampler = Sampler()
308
+ sampler_output = sampler(fake_logits, sampling_metadata)
309
+ for batch_idx in range(batch_size):
310
+ logprobs_token_ids = sampler_output.logprob_token_ids[batch_idx]
311
+ non_penalized_token_id = logprobs_token_ids[0]
312
+ penalized_token_id = logprobs_token_ids[VOCAB_SIZE - 1]
313
+ prompt_tokens = sampling_metadata.prompt_token_ids[
314
+ batch_idx][:].tolist()
315
+ output_tokens = sampling_metadata.output_token_ids[batch_idx]
316
+ if repetition_penalty > 1.0:
317
+ # If `repetition_penalty` > 1.0, verify that the non-penalized
318
+ # token ID has not been seen before, while the penalized token ID
319
+ # exists either in the prompt or the output.
320
+ assert (non_penalized_token_id not in prompt_tokens and \
321
+ non_penalized_token_id not in output_tokens)
322
+ assert (penalized_token_id in prompt_tokens or \
323
+ penalized_token_id in output_tokens)
324
+ elif repetition_penalty < 1.0:
325
+ # If `repetition_penalty` < 1.0, verify that the penalized
326
+ # token ID has not been seen before, while the non-penalized
327
+ # token ID exists either in the prompt or the output.
328
+ assert (penalized_token_id not in prompt_tokens and \
329
+ penalized_token_id not in output_tokens)
330
+ assert (non_penalized_token_id in prompt_tokens or \
331
+ non_penalized_token_id in output_tokens)
tests/v1/worker/__init__.py
tests/v1/worker/test_gpu_input_batch.py
new
+224
@@ -0,0 +1,224 @@
1
+from typing import Dict, List, Set, Tuple
2
+
3
+import numpy as np
4
+import pytest
5
+import torch
6
+
7
+from vllm.sampling_params import SamplingParams
8
+from vllm.utils import is_pin_memory_available, make_tensor_with_pad
9
+from vllm.v1.sample.metadata import SamplingMetadata
10
+from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
11
+
12
+VOCAB_SIZE = 1024
13
+NUM_OUTPUT_TOKENS = 20
14
+MAX_PROMPT_SIZE = 100
15
+CUDA_DEVICES = [
16
+ f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
17
+]
18
+MAX_NUM_PROMPT_TOKENS = 64
19
+
20
+
21
+def _remove_requests(
22
+ input_batch: InputBatch, batch_size: int,
23
+ reqs: List[CachedRequestState]) -> Tuple[Set[str], List[int]]:
24
+ """
25
+ Remove some requests randomly from the batch and returns a Tuple
26
+ of 1) set of request removed 2) indices of the requests removed
27
+ ordered in descending order
28
+ """
29
+
30
+ num_reqs_to_remove = np.random.randint(0, batch_size)
31
+ req_indices_to_remove: Set[int] = set()
32
+ for _ in range(num_reqs_to_remove):
33
+ req_index_to_remove = np.random.randint(0, batch_size)
34
+ req_indices_to_remove.add(req_index_to_remove)
35
+
36
+ req_indices_to_remove_list = list(req_indices_to_remove)
37
+ req_indices_to_remove_list.sort(reverse=True)
38
+ req_ids_to_remove: Set[str] = set()
39
+ for index in req_indices_to_remove:
40
+ input_batch.remove_request(reqs[index].req_id)
41
+ req_ids_to_remove.add(reqs[index].req_id)
42
+ return (req_ids_to_remove, req_indices_to_remove_list)
43
+
44
+
45
+def _construct_expected_sampling_metadata(
46
+ reqs: List[CachedRequestState], req_ids_retained: Set[int],
47
+ req_id_index_in_input_batch: Dict[str, int],
48
+ device: torch.device) -> SamplingMetadata:
49
+ """
50
+ Constructs and returns the expected SamplingMetadata for this
51
+ batch.
52
+ """
53
+ num_reqs = len(req_ids_retained)
54
+ output_token_ids: List[List[int]] = [list() for _ in range(num_reqs)]
55
+ prompt_token_ids: List[List[int]] = [list() for _ in range(num_reqs)]
56
+ presence_penalties = [0.0 for _ in range(num_reqs)]
57
+ frequency_penalties = [0.0 for _ in range(num_reqs)]
58
+ repetition_penalties = [1.0 for _ in range(num_reqs)]
59
+ top_k = [0 for _ in range(num_reqs)]
60
+ top_p = [0.0 for _ in range(num_reqs)]
61
+ temperature = [0.0 for _ in range(num_reqs)]
62
+ stop_token_ids: List[Set[int]] = [set() for _ in range(num_reqs)]
63
+ min_tokens = [0 for _ in range(num_reqs)]
64
+ for req in reqs:
65
+ if req.req_id not in req_ids_retained:
66
+ continue
67
+ index_in_input_batch = req_id_index_in_input_batch[req.req_id]
68
+ output_token_ids[index_in_input_batch] = req.output_token_ids
69
+ prompt_token_ids[index_in_input_batch] = req.prompt_token_ids
70
+ presence_penalties[
71
+ index_in_input_batch] = req.sampling_params.presence_penalty
72
+ frequency_penalties[
73
+ index_in_input_batch] = req.sampling_params.frequency_penalty
74
+ repetition_penalties[
75
+ index_in_input_batch] = req.sampling_params.repetition_penalty
76
+ top_k[index_in_input_batch] = req.sampling_params.top_k
77
+ top_p[index_in_input_batch] = req.sampling_params.top_p
78
+ temperature[index_in_input_batch] = req.sampling_params.temperature
79
+ stop_token_ids[
80
+ index_in_input_batch] = req.sampling_params.all_stop_token_ids
81
+ min_tokens[index_in_input_batch] = req.sampling_params.min_tokens
82
+
83
+
84
+ return SamplingMetadata(
85
+ temperature=torch.tensor(temperature, dtype=torch.float, device=device),
86
+ all_greedy=False,
87
+ all_random=True,
88
+ top_p=torch.tensor(top_p, dtype=torch.float, device=device),
89
+ top_k=torch.tensor(top_k, dtype=torch.int, device=device),
90
+ no_top_p=all(x == 1.0 for x in top_p),
91
+ no_top_k=all(x == 0 for x in top_k),
92
+ generators={},
93
+ max_num_logprobs=0,
94
+ prompt_token_ids= make_tensor_with_pad(
95
+ prompt_token_ids,
96
+ pad=VOCAB_SIZE,
97
+ device=torch.device(device),
98
+ dtype=torch.int64,
99
+ ),
100
+ frequency_penalties=torch.tensor(
101
+ frequency_penalties, dtype=torch.float,
102
+ device=device),
103
+ presence_penalties=torch.tensor(
104
+ presence_penalties, dtype=torch.float,
105
+ device=device),
106
+ repetition_penalties=torch.tensor(
107
+ repetition_penalties, dtype=torch.float,
108
+ device=device),
109
+ output_token_ids=output_token_ids,
110
+ min_tokens=min_tokens,
111
+ stop_token_ids=stop_token_ids,
112
+ no_penalties=(all(x ==0 for x in presence_penalties) and \
113
+ all(x ==0 for x in frequency_penalties) and \
114
+ all(x ==1 for x in repetition_penalties))
115
+ )
116
+
117
+
118
+def _create_sampling_params():
119
+ return SamplingParams(top_k=np.random.randint(1, 10),
120
+ top_p=np.random.uniform(0.0, 1.0),
121
+ presence_penalty=np.random.uniform(-2.0, 2.0),
122
+ repetition_penalty=np.random.uniform(0.0, 2.0),
123
+ frequency_penalty=np.random.uniform(-2.0, 2.0),
124
+ min_tokens=np.random.randint(1, 10),
125
+ stop_token_ids=[
126
+ np.random.randint(0, VOCAB_SIZE)
127
+ for _ in range(np.random.randint(10))
128
+ ])
129
+
130
+
131
+def _construct_cached_request_state(req_id_suffix: int):
132
+ prompt_token_ids = [
133
+ np.random.randint(0, VOCAB_SIZE)
134
+ for _ in range(np.random.randint(0, MAX_PROMPT_SIZE))
135
+ ]
136
+ output_token_ids = [
137
+ np.random.randint(0, VOCAB_SIZE)
138
+ for _ in range(np.random.randint(0, NUM_OUTPUT_TOKENS))
139
+ ]
140
+ return CachedRequestState(req_id=f"req_id_{req_id_suffix}",
141
+ prompt_token_ids=prompt_token_ids,
142
+ prompt=None,
143
+ sampling_params=_create_sampling_params(),
144
+ mm_inputs=[],
145
+ mm_positions=[],
146
+ block_ids=[],
147
+ generator=None,
148
+ num_computed_tokens=len(output_token_ids),
149
+ output_token_ids=output_token_ids)
150
+
151
+
152
+@pytest.mark.parametrize("device", CUDA_DEVICES)
153
+@pytest.mark.parametrize("batch_size", [1, 2, 32, 64])
154
+def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
155
+ """
156
+ Tests the logic for managing sampling metadata in the InputBatch.
157
+
158
+ This test involves adding a set of requests to the InputBatch,
159
+ followed by removing a subset of them. Afterward, the batch is compacted,
160
+ and the `make_sampling_metadata` method is invoked on the batch. The
161
+ output of `make_sampling_metadata` is then compared against the expected
162
+ results to ensure correctness.
163
+ """
164
+ input_batch: InputBatch = InputBatch(max_num_reqs=batch_size,
165
+ max_model_len=1024,
166
+ max_num_blocks_per_req=10,
167
+ device=torch.device(device),
168
+ pin_memory=is_pin_memory_available(),
169
+ vocab_size=1024)
170
+ reqs: List[CachedRequestState] = []
171
+ req_id_reqs = {}
172
+ req_id_output_token_ids = {}
173
+ # Add requests
174
+ for req_index in range(batch_size):
175
+ req: CachedRequestState = _construct_cached_request_state(req_index)
176
+ input_batch.add_request(req, req_index)
177
+ reqs.append(req)
178
+ req_id_reqs[req.req_id] = req
179
+ req_id_output_token_ids[req.req_id] = req.output_token_ids
180
+
181
+ # Remove some requests
182
+ req_ids_to_remove, req_indices_to_remove = _remove_requests(
183
+ input_batch, batch_size, reqs)
184
+ req_ids_retained = set(req_id_reqs.keys()) - req_ids_to_remove
185
+
186
+ # Compact the input batch
187
+ input_batch.condense(req_indices_to_remove)
188
+
189
+ # Generate the sampling metadata
190
+ sampling_metadata = input_batch.make_sampling_metadata(
191
+ req_id_output_token_ids, skip_copy=False)
192
+
193
+ # Create expected output.
194
+ expected_sampling_metadata = _construct_expected_sampling_metadata(
195
+ reqs,
196
+ req_ids_retained,
197
+ input_batch.req_id_to_index,
198
+ device=torch.device(device))
199
+
200
+ # Assert the actual and expected output.
201
+ assert torch.allclose(expected_sampling_metadata.temperature,
202
+ sampling_metadata.temperature)
203
+ assert torch.allclose(expected_sampling_metadata.top_p,
204
+ sampling_metadata.top_p)
205
+ assert torch.allclose(expected_sampling_metadata.top_k,
206
+ sampling_metadata.top_k)
207
+ assert torch.allclose(expected_sampling_metadata.frequency_penalties,
208
+ sampling_metadata.frequency_penalties)
209
+ assert torch.allclose(expected_sampling_metadata.presence_penalties,
210
+ sampling_metadata.presence_penalties)
211
+ assert torch.allclose(expected_sampling_metadata.repetition_penalties,
212
+ sampling_metadata.repetition_penalties)
213
+ assert torch.allclose(expected_sampling_metadata.prompt_token_ids,
214
+ sampling_metadata.prompt_token_ids)
215
+ assert (expected_sampling_metadata.output_token_ids ==
216
+ sampling_metadata.output_token_ids)
217
+ assert (
218
+ expected_sampling_metadata.min_tokens == sampling_metadata.min_tokens)
219
+ assert (expected_sampling_metadata.stop_token_ids ==
220
+ sampling_metadata.stop_token_ids)
221
+ assert (expected_sampling_metadata.no_penalties ==
222
+ sampling_metadata.no_penalties)
223
+ assert (expected_sampling_metadata.no_top_p == sampling_metadata.no_top_p)
224
+ assert (expected_sampling_metadata.no_top_k == sampling_metadata.no_top_k)
vllm/model_executor/layers/sampler.py
+6
-45
@@ -11,6 +11,7 @@ import torch
11
import torch.nn as nn
12
13
import vllm.envs as envs
14
+from vllm.model_executor.layers.utils import apply_penalties
15
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
16
SamplingTensors,
17
SequenceGroupToSample)
@@ -258,11 +259,11 @@ class Sampler(nn.Module):
259
260
# Apply presence and frequency penalties.
261
if do_penalties:
261
- logits = _apply_penalties(logits, sampling_tensors.prompt_tokens,
262
- sampling_tensors.output_tokens,
263
- sampling_tensors.presence_penalties,
264
- sampling_tensors.frequency_penalties,
265
- sampling_tensors.repetition_penalties)
262
+ logits = apply_penalties(logits, sampling_tensors.prompt_tokens,
263
+ sampling_tensors.output_tokens,
264
+ sampling_tensors.presence_penalties,
265
+ sampling_tensors.frequency_penalties,
266
+ sampling_tensors.repetition_penalties)
267
268
# Use float32 to apply temperature scaling.
269
# Use in-place division to avoid creating a new tensor.
@@ -336,23 +337,6 @@ class Sampler(nn.Module):
337
return self.should_modify_greedy_probs_inplace
338
339
339
-def _get_bin_counts_and_mask(
340
- tokens: torch.Tensor,
341
- vocab_size: int,
342
- num_seqs: int,
343
-) -> Tuple[torch.Tensor, torch.Tensor]:
344
- # Compute the bin counts for the tokens.
345
- # vocab_size + 1 for padding.
346
- bin_counts = torch.zeros((num_seqs, vocab_size + 1),
347
- dtype=torch.long,
348
- device=tokens.device)
349
- bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens))
350
- bin_counts = bin_counts[:, :vocab_size]
351
- mask = bin_counts > 0
352
-
353
- return bin_counts, mask
354
-
355
-
340
def _apply_min_tokens_penalty(
341
logits: torch.Tensor,
342
sampling_metadata: SamplingMetadata,
@@ -400,29 +384,6 @@ def _apply_min_tokens_penalty(
384
return logits
385
386
403
-def _apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
404
- output_tokens_tensor: torch.Tensor,
405
- presence_penalties: torch.Tensor,
406
- frequency_penalties: torch.Tensor,
407
- repetition_penalties: torch.Tensor) -> torch.Tensor:
408
- num_seqs, vocab_size = logits.shape
409
- _, prompt_mask = _get_bin_counts_and_mask(prompt_tokens_tensor, vocab_size,
410
- num_seqs)
411
- output_bin_counts, output_mask = _get_bin_counts_and_mask(
412
- output_tokens_tensor, vocab_size, num_seqs)
413
-
414
- repetition_penalties = repetition_penalties[:, None].repeat(1, vocab_size)
415
- repetition_penalties[~(prompt_mask | output_mask)] = 1.0
416
- logits = torch.where(logits > 0, logits / repetition_penalties,
417
- logits * repetition_penalties)
418
-
419
- # We follow the definition in OpenAI API.
420
- # Refer to https://platform.openai.com/docs/api-reference/parameter-details
421
- logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts
422
- logits -= presence_penalties.unsqueeze_(dim=1) * output_mask
423
- return logits
424
-
425
-
387
def _apply_top_k_top_p(
388
logits: torch.Tensor,
389
p: torch.Tensor,
vllm/model_executor/layers/utils.py
new
+57
@@ -0,0 +1,57 @@
1
+"""Utility methods for model layers."""
2
+from typing import Tuple
3
+
4
+import torch
5
+
6
+
7
+def get_token_bin_counts_and_mask(
8
+ tokens: torch.Tensor,
9
+ vocab_size: int,
10
+ num_seqs: int,
11
+) -> Tuple[torch.Tensor, torch.Tensor]:
12
+ # Compute the bin counts for the tokens.
13
+ # vocab_size + 1 for padding.
14
+ bin_counts = torch.zeros((num_seqs, vocab_size + 1),
15
+ dtype=torch.long,
16
+ device=tokens.device)
17
+ bin_counts.scatter_add_(1, tokens, torch.ones_like(tokens))
18
+ bin_counts = bin_counts[:, :vocab_size]
19
+ mask = bin_counts > 0
20
+
21
+ return bin_counts, mask
22
+
23
+
24
+def apply_penalties(logits: torch.Tensor, prompt_tokens_tensor: torch.Tensor,
25
+ output_tokens_tensor: torch.Tensor,
26
+ presence_penalties: torch.Tensor,
27
+ frequency_penalties: torch.Tensor,
28
+ repetition_penalties: torch.Tensor) -> torch.Tensor:
29
+ """
30
+ Applies penalties in place to the logits tensor
31
+ logits : The input logits tensor of shape [num_seqs, vocab_size]
32
+ prompt_tokens_tensor: A tensor containing the prompt tokens. The prompts
33
+ are padded to the maximum prompt length within the batch using
34
+ `vocab_size` as the padding value. The value `vocab_size` is used
35
+ for padding because it does not correspond to any valid token ID
36
+ in the vocabulary.
37
+ output_tokens_tensor: The output tokens tensor.
38
+ presence_penalties: The presence penalties of shape (num_seqs, )
39
+ frequency_penalties: The frequency penalties of shape (num_seqs, )
40
+ repetition_penalties: The repetition penalties of shape (num_seqs, )
41
+ """
42
+ num_seqs, vocab_size = logits.shape
43
+ _, prompt_mask = get_token_bin_counts_and_mask(prompt_tokens_tensor,
44
+ vocab_size, num_seqs)
45
+ output_bin_counts, output_mask = get_token_bin_counts_and_mask(
46
+ output_tokens_tensor, vocab_size, num_seqs)
47
+ repetition_penalties = repetition_penalties.unsqueeze_(dim=1).repeat(
48
+ 1, vocab_size)
49
+ logits[logits > 0] /= torch.where(prompt_mask | output_mask,
50
+ repetition_penalties, 1.0)[logits > 0]
51
+ logits[logits <= 0] *= torch.where(prompt_mask | output_mask,
52
+ repetition_penalties, 1.0)[logits <= 0]
53
+ # We follow the definition in OpenAI API.
54
+ # Refer to https://platform.openai.com/docs/api-reference/parameter-details
55
+ logits -= frequency_penalties.unsqueeze_(dim=1) * output_bin_counts
56
+ logits -= presence_penalties.unsqueeze_(dim=1) * output_mask
57
+ return logits
vllm/v1/sample/metadata.py
+11
-1
@@ -1,5 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import Dict
2
+from typing import Dict, List, Optional, Set
3
4
import torch
5
@@ -19,3 +19,13 @@ class SamplingMetadata:
19
generators: Dict[int, torch.Generator]
20
21
max_num_logprobs: int
22
+
23
+ no_penalties: bool
24
+ prompt_token_ids: Optional[torch.Tensor]
25
+ frequency_penalties: torch.Tensor
26
+ presence_penalties: torch.Tensor
27
+ repetition_penalties: torch.Tensor
28
+
29
+ output_token_ids: List[List[int]]
30
+ min_tokens: List[int]
31
+ stop_token_ids: List[Set[int]]
vllm/v1/sample/sampler.py
+63
-2
@@ -1,9 +1,11 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
-from typing import Dict
2
+from typing import Dict, List, Set, Tuple
3
4
import torch
5
import torch.nn as nn
6
7
+from vllm.model_executor.layers.utils import apply_penalties
8
+from vllm.utils import is_pin_memory_available, make_tensor_with_pad
9
from vllm.v1.outputs import SamplerOutput
10
from vllm.v1.sample.metadata import SamplingMetadata
11
@@ -17,9 +19,18 @@ class Sampler(nn.Module):
19
logits: torch.Tensor,
20
sampling_metadata: SamplingMetadata,
21
) -> SamplerOutput:
22
+ _apply_min_token_penalties(logits, sampling_metadata.output_token_ids,
23
+ sampling_metadata.stop_token_ids,
24
+ sampling_metadata.min_tokens)
25
+ if not sampling_metadata.no_penalties:
26
+ assert sampling_metadata.prompt_token_ids is not None
27
+ _apply_penalties(logits, sampling_metadata.prompt_token_ids,
28
+ sampling_metadata.presence_penalties,
29
+ sampling_metadata.frequency_penalties,
30
+ sampling_metadata.repetition_penalties,
31
+ sampling_metadata.output_token_ids)
32
logits = self.apply_temperature(logits, sampling_metadata.temperature)
33
logits = self.apply_top_k_top_p(logits, sampling_metadata)
22
-
34
probs = self.get_probs(logits)
35
sampled = self.sample(probs, sampling_metadata)
36
# Use int32 to reduce the tensor size.
@@ -157,3 +168,53 @@ def _apply_top_k_top_p(
168
# Re-sort the probabilities.
169
logits = logits_sort.scatter(dim=-1, index=logits_idx, src=logits_sort)
170
return logits
171
+
172
+
173
+def _apply_min_token_penalties(logits: torch.Tensor,
174
+ output_token_ids: List[List[int]],
175
+ stop_token_ids: List[Set[int]],
176
+ min_tokens: List[int]):
177
+ """
178
+ Applies minimum token penalty by setting the logits of the stop tokens
179
+ to -inf.
180
+ """
181
+ min_tokens_logits_to_penalize: List[Tuple[int, int]] = []
182
+ for index, min_token in enumerate(min_tokens):
183
+ if (len(output_token_ids[index]) < min_token):
184
+ for stop_token_id in stop_token_ids[index]:
185
+ min_tokens_logits_to_penalize.append((index, stop_token_id))
186
+ if min_tokens_logits_to_penalize:
187
+ logits[tuple(zip(*min_tokens_logits_to_penalize))] = -float("inf")
188
+
189
+
190
+def _apply_penalties(logits: torch.Tensor, prompt_token_ids: torch.Tensor,
191
+ presence_penalties: torch.Tensor,
192
+ frequency_penalties: torch.Tensor,
193
+ repetition_penalties: torch.Tensor,
194
+ output_token_ids: List[List[int]]):
195
+ """
196
+ Applies presence, frequency and repetition penalties to the logits.
197
+ """
198
+ _, vocab_size = logits.shape
199
+ output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size,
200
+ logits.device)
201
+ return apply_penalties(logits, prompt_token_ids, output_tokens_t,
202
+ presence_penalties, frequency_penalties,
203
+ repetition_penalties)
204
+
205
+
206
+def _convert_to_tensors(output_token_ids: List[List[int]], vocab_size: int,
207
+ device: torch.device) -> torch.Tensor:
208
+ """
209
+ Convert the different list data structures to tensors.
210
+ """
211
+ output_tokens_tensor = make_tensor_with_pad(
212
+ output_token_ids,
213
+ # Use the value of vocab_size as a pad since we don't have a
214
+ # token_id of this value.
215
+ pad=vocab_size,
216
+ device="cpu",
217
+ dtype=torch.int64,
218
+ pin_memory=is_pin_memory_available(),
219
+ )
220
+ return output_tokens_tensor.to(device, non_blocking=True)
vllm/v1/worker/gpu_input_batch.py
+142
@@ -43,12 +43,14 @@ class InputBatch:
43
max_num_blocks_per_req: int,
44
device: torch.device,
45
pin_memory: bool,
46
+ vocab_size: int,
47
):
48
self.max_num_reqs = max_num_reqs
49
self.max_model_len = max_model_len
50
self.max_num_blocks_per_req = max_num_blocks_per_req
51
self.device = device
52
self.pin_memory = pin_memory
53
+ self.vocab_size = vocab_size
54
55
self.req_ids: List[Optional[str]] = [None] * max_num_reqs
56
self.req_id_to_index: Dict[str, int] = {}
@@ -63,6 +65,7 @@ class InputBatch:
65
)
66
self.token_ids_cpu = self.token_ids_cpu_tensor.numpy()
67
self.num_computed_tokens_cpu = np.empty(max_num_reqs, dtype=np.int32)
68
+ self.num_prompt_tokens = np.zeros(max_num_reqs, dtype=np.int32)
69
70
# Attention-related.
71
self.block_table = torch.zeros(
@@ -110,6 +113,50 @@ class InputBatch:
113
self.top_k_cpu = self.top_k_cpu_tensor.numpy()
114
self.top_k_reqs: Set[str] = set()
115
116
+ # Frequency penalty related data structures
117
+ self.frequency_penalties = torch.empty((max_num_reqs, ),
118
+ dtype=torch.float,
119
+ device=device)
120
+ self.frequency_penalties_cpu_tensor = torch.empty(
121
+ (max_num_reqs, ),
122
+ dtype=torch.float,
123
+ device="cpu",
124
+ pin_memory=pin_memory)
125
+ self.frequency_penalties_cpu = \
126
+ self.frequency_penalties_cpu_tensor.numpy()
127
+ self.frequency_penalties_reqs: Set[str] = set()
128
+
129
+ # Presence penalty related data structures
130
+ self.presence_penalties = torch.empty((max_num_reqs, ),
131
+ dtype=torch.float,
132
+ device=device)
133
+ self.presence_penalties_cpu_tensor = torch.empty((max_num_reqs, ),
134
+ dtype=torch.float,
135
+ device="cpu",
136
+ pin_memory=pin_memory)
137
+ self.presence_penalties_cpu = \
138
+ self.presence_penalties_cpu_tensor.numpy()
139
+ self.presence_penalties_reqs: Set[str] = set()
140
+
141
+ # Repetition penalty related data structures
142
+ self.repetition_penalties = torch.empty((max_num_reqs, ),
143
+ dtype=torch.float,
144
+ device=device)
145
+ self.repetition_penalties_cpu_tensor = torch.empty(
146
+ (max_num_reqs, ),
147
+ dtype=torch.float,
148
+ device="cpu",
149
+ pin_memory=pin_memory)
150
+ self.repetition_penalties_cpu = \
151
+ self.repetition_penalties_cpu_tensor.numpy()
152
+ self.repetition_penalties_reqs: Set[str] = set()
153
+
154
+ self.min_tokens: List[int] = [0] * max_num_reqs
155
+ self.stop_token_ids: List[Set[int]] = [
156
+ set() for _ in range(max_num_reqs)
157
+ ]
158
+ self.prompt_token_ids: Optional[torch.Tensor] = None
159
+
160
# req_index -> generator
161
# NOTE(woosuk): The indices of the requests that do not have their own
162
# generator should not be included in the dictionary.
@@ -133,6 +180,7 @@ class InputBatch:
180
181
# Copy the prompt token ids and output token ids.
182
num_prompt_tokens = len(request.prompt_token_ids)
183
+ self.num_prompt_tokens[req_index] = num_prompt_tokens
184
self.token_ids_cpu[
185
req_index, :num_prompt_tokens] = request.prompt_token_ids
186
start_idx = num_prompt_tokens
@@ -157,6 +205,20 @@ class InputBatch:
205
self.top_k_cpu[req_index] = sampling_params.top_k
206
if sampling_params.top_k > 0:
207
self.top_k_reqs.add(req_id)
208
+ self.frequency_penalties_cpu[req_index] = \
209
+ sampling_params.frequency_penalty
210
+ if sampling_params.frequency_penalty != 0.0:
211
+ self.frequency_penalties_reqs.add(req_id)
212
+ self.presence_penalties_cpu[req_index] = \
213
+ sampling_params.presence_penalty
214
+ if sampling_params.presence_penalty != 0.0:
215
+ self.presence_penalties_reqs.add(req_id)
216
+ self.repetition_penalties_cpu[req_index] = \
217
+ sampling_params.repetition_penalty
218
+ if sampling_params.repetition_penalty != 1.0:
219
+ self.repetition_penalties_reqs.add(req_id)
220
+ self.min_tokens[req_index] = sampling_params.min_tokens
221
+ self.stop_token_ids[req_index] = sampling_params.all_stop_token_ids
222
223
# NOTE(woosuk): self.generators should not include the requests that
224
# do not have their own generator.
@@ -179,6 +241,9 @@ class InputBatch:
241
self.random_reqs.discard(req_id)
242
self.top_p_reqs.discard(req_id)
243
self.top_k_reqs.discard(req_id)
244
+ self.frequency_penalties_reqs.discard(req_id)
245
+ self.presence_penalties_reqs.discard(req_id)
246
+ self.repetition_penalties_reqs.discard(req_id)
247
self.generators.pop(req_index, None)
248
self.num_logprobs.pop(req_id, None)
249
self.prompt_logprob_reqs.discard(req_id)
@@ -191,6 +256,9 @@ class InputBatch:
256
self.random_reqs.clear()
257
self.top_p_reqs.clear()
258
self.top_k_reqs.clear()
259
+ self.frequency_penalties_reqs.clear()
260
+ self.presence_penalties_reqs.clear()
261
+ self.repetition_penalties_reqs.clear()
262
self.generators.clear()
263
self.num_logprobs.clear()
264
self.prompt_logprob_reqs.clear()
@@ -224,6 +292,8 @@ class InputBatch:
292
# block_table_cpu.
293
self.token_ids_cpu[empty_index] = self.token_ids_cpu[
294
last_req_index]
295
+ self.num_prompt_tokens[empty_index] = \
296
+ self.num_prompt_tokens[last_req_index]
297
self.num_computed_tokens_cpu[
298
empty_index] = self.num_computed_tokens_cpu[last_req_index]
299
self.block_table_cpu[empty_index] = self.block_table_cpu[
@@ -232,6 +302,15 @@ class InputBatch:
302
last_req_index]
303
self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
304
self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
305
+ self.frequency_penalties_cpu[empty_index] = \
306
+ self.frequency_penalties_cpu[last_req_index]
307
+ self.presence_penalties_cpu[empty_index] = \
308
+ self.presence_penalties_cpu[last_req_index]
309
+ self.repetition_penalties_cpu[empty_index] = \
310
+ self.repetition_penalties_cpu[last_req_index]
311
+ self.min_tokens[empty_index] = self.min_tokens[last_req_index]
312
+ self.stop_token_ids[empty_index] = \
313
+ self.stop_token_ids[last_req_index]
314
generator = self.generators.pop(last_req_index, None)
315
if generator is not None:
316
self.generators[empty_index] = generator
@@ -241,6 +320,7 @@ class InputBatch:
320
321
def make_sampling_metadata(
322
self,
323
+ req_id_output_token_ids: Dict[str, List[int]],
324
skip_copy: bool = False,
325
) -> SamplingMetadata:
326
if not skip_copy:
@@ -250,6 +330,37 @@ class InputBatch:
330
self.top_p_cpu_tensor[:self.num_reqs], non_blocking=True)
331
self.top_k[:self.num_reqs].copy_(
332
self.top_k_cpu_tensor[:self.num_reqs], non_blocking=True)
333
+ if not self.no_penalties:
334
+ # Since syncing these tensors is expensive only copy them
335
+ # if necessary i.e. if there are requests which require
336
+ # penalties to be applied during sampling.
337
+ self.frequency_penalties[:self.num_reqs].copy_(
338
+ self.frequency_penalties_cpu_tensor[:self.num_reqs],
339
+ non_blocking=True)
340
+ self.presence_penalties[:self.num_reqs].copy_(
341
+ self.presence_penalties_cpu_tensor[:self.num_reqs],
342
+ non_blocking=True)
343
+ self.repetition_penalties[:self.num_reqs].copy_(
344
+ self.repetition_penalties_cpu_tensor[:self.num_reqs],
345
+ non_blocking=True)
346
+ # The prompt tokens are used only for applying penalties during
347
+ # the sampling process. Hence copy these tensors only when
348
+ # there are requests which need penalties to be applied.
349
+ self.prompt_token_ids = self._make_prompt_token_ids_tensor()
350
+
351
+ output_token_ids: List[List[int]] = []
352
+
353
+ for req_id in self.req_ids[:self.num_reqs]:
354
+ assert req_id is not None
355
+ # Currently we create a tensor for output_token_ids from scratch
356
+ # at each step. However, for the penalties computation what we
357
+ # need is stats about the token ids present in the output. This
358
+ # stats can be maintained incrementally instead of computing it
359
+ # from scratch at each step.
360
+ # TODO - Replace this with incremental update to output token
361
+ # statistics.
362
+ output_token_ids.append(req_id_output_token_ids[req_id])
363
+
364
return SamplingMetadata(
365
temperature=self.temperature[:self.num_reqs],
366
all_greedy=self.all_greedy,
@@ -260,8 +371,33 @@ class InputBatch:
371
no_top_k=self.no_top_k,
372
generators=self.generators,
373
max_num_logprobs=self.max_num_logprobs,
374
+ prompt_token_ids=self.prompt_token_ids,
375
+ frequency_penalties=self.frequency_penalties[:self.num_reqs],
376
+ presence_penalties=self.presence_penalties[:self.num_reqs],
377
+ repetition_penalties=self.repetition_penalties[:self.num_reqs],
378
+ output_token_ids=output_token_ids,
379
+ min_tokens=self.min_tokens[:self.num_reqs],
380
+ stop_token_ids=self.stop_token_ids[:self.num_reqs],
381
+ no_penalties=self.no_penalties,
382
)
383
384
+ def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
385
+ max_prompt_len = self.num_prompt_tokens[:self.num_reqs].max()
386
+ prompt_token_ids_cpu_tensor = torch.empty(
387
+ (self.num_reqs, max_prompt_len),
388
+ device="cpu",
389
+ dtype=torch.int64,
390
+ pin_memory=self.pin_memory)
391
+ prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
392
+ prompt_token_ids[:] = (
393
+ self.token_ids_cpu[:self.num_reqs, :max_prompt_len])
394
+ # Use the value of vocab_size as a pad since we don't have a
395
+ # token_id of this value.
396
+ for i in range(self.num_reqs):
397
+ prompt_token_ids[i, self.num_prompt_tokens[i]:] = self.vocab_size
398
+ return prompt_token_ids_cpu_tensor.to(device=self.device,
399
+ non_blocking=True)
400
+
401
@property
402
def num_reqs(self) -> int:
403
return len(self.req_id_to_index)
@@ -282,6 +418,12 @@ class InputBatch:
418
def no_top_k(self) -> bool:
419
return len(self.top_k_reqs) == 0
420
421
+ @property
422
+ def no_penalties(self) -> bool:
423
+ return (len(self.presence_penalties_reqs) == 0
424
+ and len(self.frequency_penalties_reqs) == 0
425
+ and len(self.repetition_penalties_reqs) == 0)
426
+
427
@property
428
def max_num_logprobs(self) -> int:
429
return max(self.num_logprobs.values()) if self.num_logprobs else 0
vllm/v1/worker/gpu_model_runner.py
+7
-1
@@ -105,6 +105,7 @@ class GPUModelRunner:
105
max_num_blocks_per_req=self.max_num_blocks_per_req,
106
device=self.device,
107
pin_memory=self.pin_memory,
108
+ vocab_size=model_config.get_vocab_size(),
109
)
110
111
self.use_cuda_graph = (self.vllm_config.compilation_config.level
@@ -383,7 +384,12 @@ class GPUModelRunner:
384
or scheduler_output.scheduled_resumed_reqs):
385
skip_copy = False
386
# Create the sampling metadata.
386
- sampling_metadata = self.input_batch.make_sampling_metadata(skip_copy)
387
+ req_id_output_token_ids: Dict[str, List[int]] = \
388
+ {req_id: req.output_token_ids \
389
+ for req_id, req in self.requests.items()}
390
+
391
+ sampling_metadata = self.input_batch.make_sampling_metadata(
392
+ req_id_output_token_ids, skip_copy)
393
return sampling_metadata
394
395
def _execute_encoder(self, scheduler_output: "SchedulerOutput"):