A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Support logit_bias in v1 Sampler (#13079)
Lu Fang committed
Feb 14, 2025 at 04:34 UTC
6224a9f620c15c7c5d41c9878026decce8198840
6 files changed
+200
-101
tests/v1/sample/test_sampler.py
+59
-12
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import List, Set, Tuple
3
+from typing import Dict, List, Optional, Set, Tuple
4
5
import numpy as np
6
import pytest
@@ -45,6 +45,18 @@ def _create_prompt_tokens_tensor(
45
)
46
47
48
+def _create_logit_bias(
49
+ batch_size: int,
50
+ vocab_size: int,
51
+ bias_value: float,
52
+) -> List[Optional[Dict[int, float]]]:
53
+ res: List[Optional[Dict[int, float]]] = []
54
+ for i in range(batch_size):
55
+ logit_bias = {min(i, vocab_size - 1): bias_value}
56
+ res.append(logit_bias)
57
+ return res
58
+
59
+
60
def _create_default_sampling_metadata(
61
num_output_tokens: int,
62
batch_size: int,
@@ -80,6 +92,7 @@ def _create_default_sampling_metadata(
92
no_penalties=True,
93
min_tokens=[],
94
stop_token_ids=[],
95
+ logit_bias=[None] * batch_size,
96
)
97
return fake_sampling_metadata
98
@@ -89,14 +102,14 @@ def _generate_min_token_penalties_and_stop_tokens(
102
batch_indices_for_min_token_penalty: List[int]
103
) -> Tuple[List[int], List[Set[int]]]:
104
"""
92
- Generates and returns a list of minimum token penalties (`min_tokens`)
93
- and a corresponding list of stop token IDs (`stop_token_ids`) for each
105
+ Generates and returns a list of minimum token penalties (`min_tokens`)
106
+ and a corresponding list of stop token IDs (`stop_token_ids`) for each
107
batch.
108
96
- If a batch index is included in `batch_indices_for_min_token_penalty`,
97
- a higher `min_tokens` value is assigned (within a randomized range),
98
- and a random set of stop token IDs is created. Otherwise, a lower
99
- `min_tokens` value is assigned, and the stop token IDs set is empty.
109
+ If a batch index is included in `batch_indices_for_min_token_penalty`,
110
+ a higher `min_tokens` value is assigned (within a randomized range),
111
+ and a random set of stop token IDs is created. Otherwise, a lower
112
+ `min_tokens` value is assigned, and the stop token IDs set is empty.
113
"""
114
stop_token_ids: List[Set[int]] = []
115
min_tokens: List[int] = []
@@ -120,7 +133,7 @@ def _create_weighted_output_token_list(
133
batch_size: int,
134
vocab_size: int) -> Tuple[List[List[int]], List[List[int]]]:
135
"""
123
- Creates an output token list where each token occurs a distinct
136
+ Creates an output token list where each token occurs a distinct
137
number of times.
138
139
For each batch, a random subset of token IDs is selected from the
@@ -129,8 +142,8 @@ def _create_weighted_output_token_list(
142
143
Returns:
144
Tuple[List[List[int]], List[List[int]]]:
132
- - The first element is the output token list, where each sublist
133
- corresponds to a batch and contains tokens with weighted
145
+ - The first element is the output token list, where each sublist
146
+ corresponds to a batch and contains tokens with weighted
147
frequencies.
148
- The second element is a list of distinct token IDs for each
149
batch, ordered by their frequency in the corresponding output
@@ -155,7 +168,7 @@ def _create_weighted_output_token_list(
168
@pytest.mark.parametrize("batch_size", [1, 2, 32])
169
def test_sampler_min_tokens_penalty(device: str, batch_size: int):
170
"""
158
- Tests that if the number of output tokens is less than
171
+ Tests that if the number of output tokens is less than
172
SamplingParams.min_tokens then we will set the logits for
173
the stop token ids to -inf.
174
"""
@@ -283,7 +296,7 @@ def test_sampler_frequency_penalty(device: str, batch_size: int,
296
def test_sampler_repetition_penalty(device: str, batch_size: int,
297
repetition_penalty: float):
298
"""
286
- Test to verify that when the repetition penalty is enabled, tokens
299
+ Test to verify that when the repetition penalty is enabled, tokens
300
are penalized based on their presence in the prompt or the existing
301
output.
302
"""
@@ -321,3 +334,37 @@ def test_sampler_repetition_penalty(device: str, batch_size: int,
334
penalized_token_id not in output_tokens)
335
assert (non_penalized_token_id in prompt_tokens or \
336
non_penalized_token_id in output_tokens)
337
+
338
+
339
+@pytest.mark.parametrize("device", CUDA_DEVICES)
340
+@pytest.mark.parametrize("batch_size", [1, 2, 32])
341
+@pytest.mark.parametrize("bias_value", [-0.1, 1.2])
342
+def test_sampler_logit_bias(device: str, batch_size: int, bias_value: float):
343
+ """
344
+ Test to verify that when the repetition penalty is enabled, tokens
345
+ are penalized based on their presence in the prompt or the existing
346
+ output.
347
+ """
348
+ torch.set_default_device(device)
349
+ # Create fake logits where each token is assigned the same
350
+ # logit value.
351
+ fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
352
+ sampling_metadata = _create_default_sampling_metadata(
353
+ NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
354
+ sampling_metadata.logit_bias = _create_logit_bias(
355
+ batch_size=batch_size,
356
+ vocab_size=VOCAB_SIZE,
357
+ bias_value=bias_value,
358
+ )
359
+ sampler = Sampler()
360
+ logits = sampler.apply_logits_bias(fake_logits, sampling_metadata)
361
+ logits = logits.cpu()
362
+ for batch_idx in range(batch_size):
363
+ logits_for_req = logits[batch_idx]
364
+ biased_index = min(batch_idx, VOCAB_SIZE - 1)
365
+ for token_id in range(VOCAB_SIZE):
366
+ if biased_index == token_id:
367
+ assert logits_for_req[token_id] == pytest.approx(bias_value +
368
+ 1e-2)
369
+ else:
370
+ assert logits_for_req[token_id] == pytest.approx(1e-2)
tests/v1/worker/test_gpu_input_batch.py
+80
-62
@@ -45,9 +45,11 @@ def _remove_requests(
45
46
47
def _construct_expected_sampling_metadata(
48
- reqs: List[CachedRequestState], req_ids_retained: Set[int],
49
- req_id_index_in_input_batch: Dict[str, int],
50
- device: torch.device) -> SamplingMetadata:
48
+ reqs: List[CachedRequestState],
49
+ req_ids_retained: Set[int],
50
+ req_id_index_in_input_batch: Dict[str, int],
51
+ device: torch.device,
52
+) -> SamplingMetadata:
53
"""
54
Constructs and returns the expected SamplingMetadata for this
55
batch.
@@ -63,6 +65,7 @@ def _construct_expected_sampling_metadata(
65
temperature = [0.0 for _ in range(num_reqs)]
66
stop_token_ids: List[Set[int]] = [set() for _ in range(num_reqs)]
67
min_tokens = [0 for _ in range(num_reqs)]
68
+ logit_bias = [None] * num_reqs
69
for req in reqs:
70
if req.req_id not in req_ids_retained:
71
continue
@@ -71,20 +74,21 @@ def _construct_expected_sampling_metadata(
74
prompt_token_ids[index_in_input_batch] = req.prompt_token_ids
75
presence_penalties[
76
index_in_input_batch] = req.sampling_params.presence_penalty
74
- frequency_penalties[
75
- index_in_input_batch] = req.sampling_params.frequency_penalty
76
- repetition_penalties[
77
- index_in_input_batch] = req.sampling_params.repetition_penalty
77
+ frequency_penalties[index_in_input_batch] = (
78
+ req.sampling_params.frequency_penalty)
79
+ repetition_penalties[index_in_input_batch] = (
80
+ req.sampling_params.repetition_penalty)
81
top_k[index_in_input_batch] = req.sampling_params.top_k
82
top_p[index_in_input_batch] = req.sampling_params.top_p
83
temperature[index_in_input_batch] = req.sampling_params.temperature
84
stop_token_ids[
85
index_in_input_batch] = req.sampling_params.all_stop_token_ids
86
min_tokens[index_in_input_batch] = req.sampling_params.min_tokens
84
-
87
+ logit_bias[index_in_input_batch] = req.sampling_params.logit_bias
88
89
return SamplingMetadata(
87
- temperature=torch.tensor(temperature, dtype=torch.float, device=device),
90
+ temperature=torch.tensor(temperature, dtype=torch.float,
91
+ device=device),
92
all_greedy=False,
93
all_random=True,
94
top_p=torch.tensor(top_p, dtype=torch.float, device=device),
@@ -93,41 +97,45 @@ def _construct_expected_sampling_metadata(
97
no_top_k=all(x == 0 for x in top_k),
98
generators={},
99
max_num_logprobs=0,
96
- prompt_token_ids= make_tensor_with_pad(
100
+ prompt_token_ids=make_tensor_with_pad(
101
prompt_token_ids,
102
pad=VOCAB_SIZE,
103
device=torch.device(device),
104
dtype=torch.int64,
105
),
102
- frequency_penalties=torch.tensor(
103
- frequency_penalties, dtype=torch.float,
104
- device=device),
105
- presence_penalties=torch.tensor(
106
- presence_penalties, dtype=torch.float,
107
- device=device),
108
- repetition_penalties=torch.tensor(
109
- repetition_penalties, dtype=torch.float,
110
- device=device),
106
+ frequency_penalties=torch.tensor(frequency_penalties,
107
+ dtype=torch.float,
108
+ device=device),
109
+ presence_penalties=torch.tensor(presence_penalties,
110
+ dtype=torch.float,
111
+ device=device),
112
+ repetition_penalties=torch.tensor(repetition_penalties,
113
+ dtype=torch.float,
114
+ device=device),
115
output_token_ids=output_token_ids,
116
min_tokens=min_tokens,
117
stop_token_ids=stop_token_ids,
114
- no_penalties=(all(x ==0 for x in presence_penalties) and \
115
- all(x ==0 for x in frequency_penalties) and \
116
- all(x ==1 for x in repetition_penalties))
118
+ no_penalties=(all(x == 0 for x in presence_penalties)
119
+ and all(x == 0 for x in frequency_penalties)
120
+ and all(x == 1 for x in repetition_penalties)),
121
+ logit_bias=logit_bias,
122
)
123
124
125
def _create_sampling_params():
121
- return SamplingParams(top_k=np.random.randint(1, 10),
122
- top_p=np.random.uniform(0.0, 1.0),
123
- presence_penalty=np.random.uniform(-2.0, 2.0),
124
- repetition_penalty=np.random.uniform(0.0, 2.0),
125
- frequency_penalty=np.random.uniform(-2.0, 2.0),
126
- min_tokens=np.random.randint(1, 10),
127
- stop_token_ids=[
128
- np.random.randint(0, VOCAB_SIZE)
129
- for _ in range(np.random.randint(10))
130
- ])
126
+ return SamplingParams(
127
+ top_k=np.random.randint(1, 10),
128
+ top_p=np.random.uniform(0.0, 1.0),
129
+ presence_penalty=np.random.uniform(-2.0, 2.0),
130
+ repetition_penalty=np.random.uniform(0.0, 2.0),
131
+ frequency_penalty=np.random.uniform(-2.0, 2.0),
132
+ min_tokens=np.random.randint(1, 10),
133
+ stop_token_ids=[
134
+ np.random.randint(0, VOCAB_SIZE)
135
+ for _ in range(np.random.randint(10))
136
+ ],
137
+ logit_bias={0: np.random.uniform(-3.0, 3.0)},
138
+ )
139
140
141
def _construct_cached_request_state(req_id_suffix: int):
@@ -139,16 +147,18 @@ def _construct_cached_request_state(req_id_suffix: int):
147
np.random.randint(0, VOCAB_SIZE)
148
for _ in range(np.random.randint(0, NUM_OUTPUT_TOKENS))
149
]
142
- return CachedRequestState(req_id=f"req_id_{req_id_suffix}",
143
- prompt_token_ids=prompt_token_ids,
144
- prompt=None,
145
- sampling_params=_create_sampling_params(),
146
- mm_inputs=[],
147
- mm_positions=[],
148
- block_ids=[],
149
- generator=None,
150
- num_computed_tokens=len(output_token_ids),
151
- output_token_ids=output_token_ids)
150
+ return CachedRequestState(
151
+ req_id=f"req_id_{req_id_suffix}",
152
+ prompt_token_ids=prompt_token_ids,
153
+ prompt=None,
154
+ sampling_params=_create_sampling_params(),
155
+ mm_inputs=[],
156
+ mm_positions=[],
157
+ block_ids=[],
158
+ generator=None,
159
+ num_computed_tokens=len(output_token_ids),
160
+ output_token_ids=output_token_ids,
161
+ )
162
163
164
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -163,12 +173,14 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
173
output of `make_sampling_metadata` is then compared against the expected
174
results to ensure correctness.
175
"""
166
- input_batch: InputBatch = InputBatch(max_num_reqs=batch_size,
167
- max_model_len=1024,
168
- max_num_blocks_per_req=10,
169
- device=torch.device(device),
170
- pin_memory=is_pin_memory_available(),
171
- vocab_size=1024)
176
+ input_batch: InputBatch = InputBatch(
177
+ max_num_reqs=batch_size,
178
+ max_model_len=1024,
179
+ max_num_blocks_per_req=10,
180
+ device=torch.device(device),
181
+ pin_memory=is_pin_memory_available(),
182
+ vocab_size=1024,
183
+ )
184
reqs: List[CachedRequestState] = []
185
req_id_reqs = {}
186
req_id_output_token_ids = {}
@@ -206,21 +218,27 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
218
sampling_metadata.top_p)
219
assert torch.allclose(expected_sampling_metadata.top_k,
220
sampling_metadata.top_k)
209
- assert torch.allclose(expected_sampling_metadata.frequency_penalties,
210
- sampling_metadata.frequency_penalties)
211
- assert torch.allclose(expected_sampling_metadata.presence_penalties,
212
- sampling_metadata.presence_penalties)
213
- assert torch.allclose(expected_sampling_metadata.repetition_penalties,
214
- sampling_metadata.repetition_penalties)
221
+ assert torch.allclose(
222
+ expected_sampling_metadata.frequency_penalties,
223
+ sampling_metadata.frequency_penalties,
224
+ )
225
+ assert torch.allclose(
226
+ expected_sampling_metadata.presence_penalties,
227
+ sampling_metadata.presence_penalties,
228
+ )
229
+ assert torch.allclose(
230
+ expected_sampling_metadata.repetition_penalties,
231
+ sampling_metadata.repetition_penalties,
232
+ )
233
assert torch.allclose(expected_sampling_metadata.prompt_token_ids,
234
sampling_metadata.prompt_token_ids)
235
assert (expected_sampling_metadata.output_token_ids ==
236
sampling_metadata.output_token_ids)
219
- assert (
220
- expected_sampling_metadata.min_tokens == sampling_metadata.min_tokens)
221
- assert (expected_sampling_metadata.stop_token_ids ==
222
- sampling_metadata.stop_token_ids)
223
- assert (expected_sampling_metadata.no_penalties ==
224
- sampling_metadata.no_penalties)
225
- assert (expected_sampling_metadata.no_top_p == sampling_metadata.no_top_p)
226
- assert (expected_sampling_metadata.no_top_k == sampling_metadata.no_top_k)
237
+ assert expected_sampling_metadata.min_tokens == sampling_metadata.min_tokens
238
+ assert expected_sampling_metadata.stop_token_ids == \
239
+ sampling_metadata.stop_token_ids
240
+ assert expected_sampling_metadata.no_penalties == \
241
+ sampling_metadata.no_penalties
242
+ assert expected_sampling_metadata.no_top_p == sampling_metadata.no_top_p
243
+ assert expected_sampling_metadata.no_top_k == sampling_metadata.no_top_k
244
+ assert expected_sampling_metadata.logit_bias == sampling_metadata.logit_bias
vllm/sampling_params.py
+3
-1
@@ -243,8 +243,10 @@ class SamplingParams(
243
allowed_token_ids: Optional[List[int]] = None,
244
) -> "SamplingParams":
245
if logit_bias is not None:
246
+ # Convert token_id to integer
247
+ # Clamp the bias between -100 and 100 per OpenAI API spec
248
logit_bias = {
247
- int(token): bias
249
+ int(token): min(100.0, max(-100.0, bias))
250
for token, bias in logit_bias.items()
251
}
252
vllm/v1/sample/metadata.py
+2
@@ -32,3 +32,5 @@ class SamplingMetadata:
32
output_token_ids: List[List[int]]
33
min_tokens: List[int]
34
stop_token_ids: List[Set[int]]
35
+
36
+ logit_bias: List[Optional[Dict[int, float]]]
vllm/v1/sample/sampler.py
+16
@@ -37,6 +37,8 @@ class Sampler(nn.Module):
37
38
# Use float32 for the logits.
39
logits = logits.to(torch.float32)
40
+ # Apply logits bias.
41
+ logits = self.apply_logits_bias(logits, sampling_metadata)
42
# Apply penalties (e.g., min_tokens, freq_penalties).
43
logits = self.apply_penalties(logits, sampling_metadata)
44
# Apply temperature.
@@ -166,3 +168,17 @@ class Sampler(nn.Module):
168
sampling_metadata.repetition_penalties,
169
sampling_metadata.output_token_ids)
170
return logits
171
+
172
+ def apply_logits_bias(
173
+ self,
174
+ logits: torch.Tensor,
175
+ sampling_metadata: SamplingMetadata,
176
+ ) -> torch.Tensor:
177
+ # TODO(houseroad): this implementation is extremely inefficient.
178
+ # One idea is implement this as a PyTorch C++ op, and we may
179
+ # even optimize the logit_bias layout.
180
+ for i, logit_bias in enumerate(sampling_metadata.logit_bias):
181
+ if logit_bias:
182
+ for token_id, bias in logit_bias.items():
183
+ logits[i, token_id] += bias
184
+ return logits
vllm/v1/worker/gpu_input_batch.py
+40
-26
@@ -130,7 +130,7 @@ class InputBatch:
130
device="cpu",
131
pin_memory=pin_memory)
132
self.frequency_penalties_cpu = \
133
- self.frequency_penalties_cpu_tensor.numpy()
133
+ self.frequency_penalties_cpu_tensor.numpy()
134
self.frequency_penalties_reqs: Set[str] = set()
135
136
# Presence penalty related data structures
@@ -141,8 +141,8 @@ class InputBatch:
141
dtype=torch.float,
142
device="cpu",
143
pin_memory=pin_memory)
144
- self.presence_penalties_cpu = \
145
- self.presence_penalties_cpu_tensor.numpy()
144
+ self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy(
145
+ )
146
self.presence_penalties_reqs: Set[str] = set()
147
148
# Repetition penalty related data structures
@@ -155,7 +155,7 @@ class InputBatch:
155
device="cpu",
156
pin_memory=pin_memory)
157
self.repetition_penalties_cpu = \
158
- self.repetition_penalties_cpu_tensor.numpy()
158
+ self.repetition_penalties_cpu_tensor.numpy()
159
self.repetition_penalties_reqs: Set[str] = set()
160
161
self.min_tokens: List[int] = [0] * max_num_reqs
@@ -180,6 +180,9 @@ class InputBatch:
180
# that are currently in the prefill phase.
181
self.num_prompt_logprobs: Dict[str, int] = {}
182
183
+ self.logit_bias: List[Optional[Dict[int,
184
+ float]]] = [None] * max_num_reqs
185
+
186
def add_request(
187
self,
188
request: "CachedRequestState",
@@ -220,16 +223,16 @@ class InputBatch:
223
self.top_k_cpu[req_index] = sampling_params.top_k
224
if sampling_params.top_k > 0:
225
self.top_k_reqs.add(req_id)
223
- self.frequency_penalties_cpu[req_index] = \
224
- sampling_params.frequency_penalty
226
+ self.frequency_penalties_cpu[
227
+ req_index] = sampling_params.frequency_penalty
228
if sampling_params.frequency_penalty != 0.0:
229
self.frequency_penalties_reqs.add(req_id)
227
- self.presence_penalties_cpu[req_index] = \
228
- sampling_params.presence_penalty
230
+ self.presence_penalties_cpu[
231
+ req_index] = sampling_params.presence_penalty
232
if sampling_params.presence_penalty != 0.0:
233
self.presence_penalties_reqs.add(req_id)
231
- self.repetition_penalties_cpu[req_index] = \
232
- sampling_params.repetition_penalty
234
+ self.repetition_penalties_cpu[
235
+ req_index] = sampling_params.repetition_penalty
236
if sampling_params.repetition_penalty != 1.0:
237
self.repetition_penalties_reqs.add(req_id)
238
self.min_tokens[req_index] = sampling_params.min_tokens
@@ -244,6 +247,8 @@ class InputBatch:
247
self.num_logprobs[req_id] = sampling_params.logprobs
248
if sampling_params.prompt_logprobs is not None:
249
self.num_prompt_logprobs[req_id] = sampling_params.prompt_logprobs
250
+ if sampling_params.logit_bias is not None:
251
+ self.logit_bias[req_index] = sampling_params.logit_bias
252
253
# Add request lora ID
254
if request.lora_request:
@@ -284,6 +289,7 @@ class InputBatch:
289
self.lora_id_to_lora_request.pop(lora_id)
290
self.request_lora_mapping[req_index] = 0
291
292
+ self.logit_bias[req_index] = None
293
return req_index
294
295
def clear(self) -> None:
@@ -302,6 +308,7 @@ class InputBatch:
308
self.request_lora_mapping.fill(0)
309
self.lora_id_to_lora_request.clear()
310
self.lora_id_to_request_ids.clear()
311
+ self.logit_bias = [None] * self.max_num_reqs
312
313
def condense(self, empty_req_indices: List[int]) -> None:
314
if self.num_reqs == 0:
@@ -332,8 +339,8 @@ class InputBatch:
339
self.token_ids_cpu[empty_index, :num_tokens] = self.token_ids_cpu[
340
last_req_index, :num_tokens]
341
self.num_tokens[empty_index] = num_tokens
335
- self.num_prompt_tokens[empty_index] = \
336
- self.num_prompt_tokens[last_req_index]
342
+ self.num_prompt_tokens[empty_index] = self.num_prompt_tokens[
343
+ last_req_index]
344
self.num_computed_tokens_cpu[
345
empty_index] = self.num_computed_tokens_cpu[last_req_index]
346
self.block_table.move_row(last_req_index, empty_index)
@@ -341,15 +348,15 @@ class InputBatch:
348
last_req_index]
349
self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
350
self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
344
- self.frequency_penalties_cpu[empty_index] = \
345
- self.frequency_penalties_cpu[last_req_index]
346
- self.presence_penalties_cpu[empty_index] = \
347
- self.presence_penalties_cpu[last_req_index]
348
- self.repetition_penalties_cpu[empty_index] = \
349
- self.repetition_penalties_cpu[last_req_index]
351
+ self.frequency_penalties_cpu[
352
+ empty_index] = self.frequency_penalties_cpu[last_req_index]
353
+ self.presence_penalties_cpu[
354
+ empty_index] = self.presence_penalties_cpu[last_req_index]
355
+ self.repetition_penalties_cpu[
356
+ empty_index] = self.repetition_penalties_cpu[last_req_index]
357
self.min_tokens[empty_index] = self.min_tokens[last_req_index]
351
- self.stop_token_ids[empty_index] = \
352
- self.stop_token_ids[last_req_index]
358
+ self.stop_token_ids[empty_index] = self.stop_token_ids[
359
+ last_req_index]
360
generator = self.generators.pop(last_req_index, None)
361
if generator is not None:
362
self.generators[empty_index] = generator
@@ -357,6 +364,8 @@ class InputBatch:
364
self.request_lora_mapping[empty_index] = self.request_lora_mapping[
365
last_req_index]
366
367
+ self.logit_bias[empty_index] = self.logit_bias[last_req_index]
368
+
369
# Decrement last_req_index since it is now empty.
370
last_req_index -= 1
371
@@ -378,13 +387,16 @@ class InputBatch:
387
# penalties to be applied during sampling.
388
self.frequency_penalties[:self.num_reqs].copy_(
389
self.frequency_penalties_cpu_tensor[:self.num_reqs],
381
- non_blocking=True)
390
+ non_blocking=True,
391
+ )
392
self.presence_penalties[:self.num_reqs].copy_(
393
self.presence_penalties_cpu_tensor[:self.num_reqs],
384
- non_blocking=True)
394
+ non_blocking=True,
395
+ )
396
self.repetition_penalties[:self.num_reqs].copy_(
397
self.repetition_penalties_cpu_tensor[:self.num_reqs],
387
- non_blocking=True)
398
+ non_blocking=True,
399
+ )
400
# The prompt tokens are used only for applying penalties during
401
# the sampling process. Hence copy these tensors only when
402
# there are requests which need penalties to be applied.
@@ -421,6 +433,7 @@ class InputBatch:
433
min_tokens=self.min_tokens[:self.num_reqs],
434
stop_token_ids=self.stop_token_ids[:self.num_reqs],
435
no_penalties=self.no_penalties,
436
+ logit_bias=self.logit_bias[:self.num_reqs],
437
)
438
439
def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
@@ -429,10 +442,11 @@ class InputBatch:
442
(self.num_reqs, max_prompt_len),
443
device="cpu",
444
dtype=torch.int64,
432
- pin_memory=self.pin_memory)
445
+ pin_memory=self.pin_memory,
446
+ )
447
prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
434
- prompt_token_ids[:] = (
435
- self.token_ids_cpu[:self.num_reqs, :max_prompt_len])
448
+ prompt_token_ids[:] = self.token_ids_cpu[:self.
449
+ num_reqs, :max_prompt_len]
450
# Use the value of vocab_size as a pad since we don't have a
451
# token_id of this value.
452
for i in range(self.num_reqs):