@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6224a9f620
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Support logit_bias in v1 Sampler (#13079)

Lu Fang committed Feb 14, 2025 at 04:34 UTC 6224a9f620c15c7c5d41c9878026decce8198840
6 files changed +200 -101
tests/v1/sample/test_sampler.py
+59 -12
@@ -1,6 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from typing import List, Set, Tuple
3 +from typing import Dict, List, Optional, Set, Tuple
4
5 import numpy as np
6 import pytest
@@ -45,6 +45,18 @@ def _create_prompt_tokens_tensor(
45 )
46
47
48 +def _create_logit_bias(
49 + batch_size: int,
50 + vocab_size: int,
51 + bias_value: float,
52 +) -> List[Optional[Dict[int, float]]]:
53 + res: List[Optional[Dict[int, float]]] = []
54 + for i in range(batch_size):
55 + logit_bias = {min(i, vocab_size - 1): bias_value}
56 + res.append(logit_bias)
57 + return res
58 +
59 +
60 def _create_default_sampling_metadata(
61 num_output_tokens: int,
62 batch_size: int,
@@ -80,6 +92,7 @@ def _create_default_sampling_metadata(
92 no_penalties=True,
93 min_tokens=[],
94 stop_token_ids=[],
95 + logit_bias=[None] * batch_size,
96 )
97 return fake_sampling_metadata
98
@@ -89,14 +102,14 @@ def _generate_min_token_penalties_and_stop_tokens(
102 batch_indices_for_min_token_penalty: List[int]
103 ) -> Tuple[List[int], List[Set[int]]]:
104 """
92 - Generates and returns a list of minimum token penalties (`min_tokens`)
93 - and a corresponding list of stop token IDs (`stop_token_ids`) for each
105 + Generates and returns a list of minimum token penalties (`min_tokens`)
106 + and a corresponding list of stop token IDs (`stop_token_ids`) for each
107 batch.
108
96 - If a batch index is included in `batch_indices_for_min_token_penalty`,
97 - a higher `min_tokens` value is assigned (within a randomized range),
98 - and a random set of stop token IDs is created. Otherwise, a lower
99 - `min_tokens` value is assigned, and the stop token IDs set is empty.
109 + If a batch index is included in `batch_indices_for_min_token_penalty`,
110 + a higher `min_tokens` value is assigned (within a randomized range),
111 + and a random set of stop token IDs is created. Otherwise, a lower
112 + `min_tokens` value is assigned, and the stop token IDs set is empty.
113 """
114 stop_token_ids: List[Set[int]] = []
115 min_tokens: List[int] = []
@@ -120,7 +133,7 @@ def _create_weighted_output_token_list(
133 batch_size: int,
134 vocab_size: int) -> Tuple[List[List[int]], List[List[int]]]:
135 """
123 - Creates an output token list where each token occurs a distinct
136 + Creates an output token list where each token occurs a distinct
137 number of times.
138
139 For each batch, a random subset of token IDs is selected from the
@@ -129,8 +142,8 @@ def _create_weighted_output_token_list(
142
143 Returns:
144 Tuple[List[List[int]], List[List[int]]]:
132 - - The first element is the output token list, where each sublist
133 - corresponds to a batch and contains tokens with weighted
145 + - The first element is the output token list, where each sublist
146 + corresponds to a batch and contains tokens with weighted
147 frequencies.
148 - The second element is a list of distinct token IDs for each
149 batch, ordered by their frequency in the corresponding output
@@ -155,7 +168,7 @@ def _create_weighted_output_token_list(
168 @pytest.mark.parametrize("batch_size", [1, 2, 32])
169 def test_sampler_min_tokens_penalty(device: str, batch_size: int):
170 """
158 - Tests that if the number of output tokens is less than
171 + Tests that if the number of output tokens is less than
172 SamplingParams.min_tokens then we will set the logits for
173 the stop token ids to -inf.
174 """
@@ -283,7 +296,7 @@ def test_sampler_frequency_penalty(device: str, batch_size: int,
296 def test_sampler_repetition_penalty(device: str, batch_size: int,
297 repetition_penalty: float):
298 """
286 - Test to verify that when the repetition penalty is enabled, tokens
299 + Test to verify that when the repetition penalty is enabled, tokens
300 are penalized based on their presence in the prompt or the existing
301 output.
302 """
@@ -321,3 +334,37 @@ def test_sampler_repetition_penalty(device: str, batch_size: int,
334 penalized_token_id not in output_tokens)
335 assert (non_penalized_token_id in prompt_tokens or \
336 non_penalized_token_id in output_tokens)
337 +
338 +
339 +@pytest.mark.parametrize("device", CUDA_DEVICES)
340 +@pytest.mark.parametrize("batch_size", [1, 2, 32])
341 +@pytest.mark.parametrize("bias_value", [-0.1, 1.2])
342 +def test_sampler_logit_bias(device: str, batch_size: int, bias_value: float):
343 + """
344 + Test to verify that when the repetition penalty is enabled, tokens
345 + are penalized based on their presence in the prompt or the existing
346 + output.
347 + """
348 + torch.set_default_device(device)
349 + # Create fake logits where each token is assigned the same
350 + # logit value.
351 + fake_logits = _create_fake_logits(batch_size, VOCAB_SIZE)
352 + sampling_metadata = _create_default_sampling_metadata(
353 + NUM_OUTPUT_TOKENS, batch_size, VOCAB_SIZE, torch.device(device))
354 + sampling_metadata.logit_bias = _create_logit_bias(
355 + batch_size=batch_size,
356 + vocab_size=VOCAB_SIZE,
357 + bias_value=bias_value,
358 + )
359 + sampler = Sampler()
360 + logits = sampler.apply_logits_bias(fake_logits, sampling_metadata)
361 + logits = logits.cpu()
362 + for batch_idx in range(batch_size):
363 + logits_for_req = logits[batch_idx]
364 + biased_index = min(batch_idx, VOCAB_SIZE - 1)
365 + for token_id in range(VOCAB_SIZE):
366 + if biased_index == token_id:
367 + assert logits_for_req[token_id] == pytest.approx(bias_value +
368 + 1e-2)
369 + else:
370 + assert logits_for_req[token_id] == pytest.approx(1e-2)
tests/v1/worker/test_gpu_input_batch.py
+80 -62
@@ -45,9 +45,11 @@ def _remove_requests(
45
46
47 def _construct_expected_sampling_metadata(
48 - reqs: List[CachedRequestState], req_ids_retained: Set[int],
49 - req_id_index_in_input_batch: Dict[str, int],
50 - device: torch.device) -> SamplingMetadata:
48 + reqs: List[CachedRequestState],
49 + req_ids_retained: Set[int],
50 + req_id_index_in_input_batch: Dict[str, int],
51 + device: torch.device,
52 +) -> SamplingMetadata:
53 """
54 Constructs and returns the expected SamplingMetadata for this
55 batch.
@@ -63,6 +65,7 @@ def _construct_expected_sampling_metadata(
65 temperature = [0.0 for _ in range(num_reqs)]
66 stop_token_ids: List[Set[int]] = [set() for _ in range(num_reqs)]
67 min_tokens = [0 for _ in range(num_reqs)]
68 + logit_bias = [None] * num_reqs
69 for req in reqs:
70 if req.req_id not in req_ids_retained:
71 continue
@@ -71,20 +74,21 @@ def _construct_expected_sampling_metadata(
74 prompt_token_ids[index_in_input_batch] = req.prompt_token_ids
75 presence_penalties[
76 index_in_input_batch] = req.sampling_params.presence_penalty
74 - frequency_penalties[
75 - index_in_input_batch] = req.sampling_params.frequency_penalty
76 - repetition_penalties[
77 - index_in_input_batch] = req.sampling_params.repetition_penalty
77 + frequency_penalties[index_in_input_batch] = (
78 + req.sampling_params.frequency_penalty)
79 + repetition_penalties[index_in_input_batch] = (
80 + req.sampling_params.repetition_penalty)
81 top_k[index_in_input_batch] = req.sampling_params.top_k
82 top_p[index_in_input_batch] = req.sampling_params.top_p
83 temperature[index_in_input_batch] = req.sampling_params.temperature
84 stop_token_ids[
85 index_in_input_batch] = req.sampling_params.all_stop_token_ids
86 min_tokens[index_in_input_batch] = req.sampling_params.min_tokens
84 -
87 + logit_bias[index_in_input_batch] = req.sampling_params.logit_bias
88
89 return SamplingMetadata(
87 - temperature=torch.tensor(temperature, dtype=torch.float, device=device),
90 + temperature=torch.tensor(temperature, dtype=torch.float,
91 + device=device),
92 all_greedy=False,
93 all_random=True,
94 top_p=torch.tensor(top_p, dtype=torch.float, device=device),
@@ -93,41 +97,45 @@ def _construct_expected_sampling_metadata(
97 no_top_k=all(x == 0 for x in top_k),
98 generators={},
99 max_num_logprobs=0,
96 - prompt_token_ids= make_tensor_with_pad(
100 + prompt_token_ids=make_tensor_with_pad(
101 prompt_token_ids,
102 pad=VOCAB_SIZE,
103 device=torch.device(device),
104 dtype=torch.int64,
105 ),
102 - frequency_penalties=torch.tensor(
103 - frequency_penalties, dtype=torch.float,
104 - device=device),
105 - presence_penalties=torch.tensor(
106 - presence_penalties, dtype=torch.float,
107 - device=device),
108 - repetition_penalties=torch.tensor(
109 - repetition_penalties, dtype=torch.float,
110 - device=device),
106 + frequency_penalties=torch.tensor(frequency_penalties,
107 + dtype=torch.float,
108 + device=device),
109 + presence_penalties=torch.tensor(presence_penalties,
110 + dtype=torch.float,
111 + device=device),
112 + repetition_penalties=torch.tensor(repetition_penalties,
113 + dtype=torch.float,
114 + device=device),
115 output_token_ids=output_token_ids,
116 min_tokens=min_tokens,
117 stop_token_ids=stop_token_ids,
114 - no_penalties=(all(x ==0 for x in presence_penalties) and \
115 - all(x ==0 for x in frequency_penalties) and \
116 - all(x ==1 for x in repetition_penalties))
118 + no_penalties=(all(x == 0 for x in presence_penalties)
119 + and all(x == 0 for x in frequency_penalties)
120 + and all(x == 1 for x in repetition_penalties)),
121 + logit_bias=logit_bias,
122 )
123
124
125 def _create_sampling_params():
121 - return SamplingParams(top_k=np.random.randint(1, 10),
122 - top_p=np.random.uniform(0.0, 1.0),
123 - presence_penalty=np.random.uniform(-2.0, 2.0),
124 - repetition_penalty=np.random.uniform(0.0, 2.0),
125 - frequency_penalty=np.random.uniform(-2.0, 2.0),
126 - min_tokens=np.random.randint(1, 10),
127 - stop_token_ids=[
128 - np.random.randint(0, VOCAB_SIZE)
129 - for _ in range(np.random.randint(10))
130 - ])
126 + return SamplingParams(
127 + top_k=np.random.randint(1, 10),
128 + top_p=np.random.uniform(0.0, 1.0),
129 + presence_penalty=np.random.uniform(-2.0, 2.0),
130 + repetition_penalty=np.random.uniform(0.0, 2.0),
131 + frequency_penalty=np.random.uniform(-2.0, 2.0),
132 + min_tokens=np.random.randint(1, 10),
133 + stop_token_ids=[
134 + np.random.randint(0, VOCAB_SIZE)
135 + for _ in range(np.random.randint(10))
136 + ],
137 + logit_bias={0: np.random.uniform(-3.0, 3.0)},
138 + )
139
140
141 def _construct_cached_request_state(req_id_suffix: int):
@@ -139,16 +147,18 @@ def _construct_cached_request_state(req_id_suffix: int):
147 np.random.randint(0, VOCAB_SIZE)
148 for _ in range(np.random.randint(0, NUM_OUTPUT_TOKENS))
149 ]
142 - return CachedRequestState(req_id=f"req_id_{req_id_suffix}",
143 - prompt_token_ids=prompt_token_ids,
144 - prompt=None,
145 - sampling_params=_create_sampling_params(),
146 - mm_inputs=[],
147 - mm_positions=[],
148 - block_ids=[],
149 - generator=None,
150 - num_computed_tokens=len(output_token_ids),
151 - output_token_ids=output_token_ids)
150 + return CachedRequestState(
151 + req_id=f"req_id_{req_id_suffix}",
152 + prompt_token_ids=prompt_token_ids,
153 + prompt=None,
154 + sampling_params=_create_sampling_params(),
155 + mm_inputs=[],
156 + mm_positions=[],
157 + block_ids=[],
158 + generator=None,
159 + num_computed_tokens=len(output_token_ids),
160 + output_token_ids=output_token_ids,
161 + )
162
163
164 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -163,12 +173,14 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
173 output of `make_sampling_metadata` is then compared against the expected
174 results to ensure correctness.
175 """
166 - input_batch: InputBatch = InputBatch(max_num_reqs=batch_size,
167 - max_model_len=1024,
168 - max_num_blocks_per_req=10,
169 - device=torch.device(device),
170 - pin_memory=is_pin_memory_available(),
171 - vocab_size=1024)
176 + input_batch: InputBatch = InputBatch(
177 + max_num_reqs=batch_size,
178 + max_model_len=1024,
179 + max_num_blocks_per_req=10,
180 + device=torch.device(device),
181 + pin_memory=is_pin_memory_available(),
182 + vocab_size=1024,
183 + )
184 reqs: List[CachedRequestState] = []
185 req_id_reqs = {}
186 req_id_output_token_ids = {}
@@ -206,21 +218,27 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
218 sampling_metadata.top_p)
219 assert torch.allclose(expected_sampling_metadata.top_k,
220 sampling_metadata.top_k)
209 - assert torch.allclose(expected_sampling_metadata.frequency_penalties,
210 - sampling_metadata.frequency_penalties)
211 - assert torch.allclose(expected_sampling_metadata.presence_penalties,
212 - sampling_metadata.presence_penalties)
213 - assert torch.allclose(expected_sampling_metadata.repetition_penalties,
214 - sampling_metadata.repetition_penalties)
221 + assert torch.allclose(
222 + expected_sampling_metadata.frequency_penalties,
223 + sampling_metadata.frequency_penalties,
224 + )
225 + assert torch.allclose(
226 + expected_sampling_metadata.presence_penalties,
227 + sampling_metadata.presence_penalties,
228 + )
229 + assert torch.allclose(
230 + expected_sampling_metadata.repetition_penalties,
231 + sampling_metadata.repetition_penalties,
232 + )
233 assert torch.allclose(expected_sampling_metadata.prompt_token_ids,
234 sampling_metadata.prompt_token_ids)
235 assert (expected_sampling_metadata.output_token_ids ==
236 sampling_metadata.output_token_ids)
219 - assert (
220 - expected_sampling_metadata.min_tokens == sampling_metadata.min_tokens)
221 - assert (expected_sampling_metadata.stop_token_ids ==
222 - sampling_metadata.stop_token_ids)
223 - assert (expected_sampling_metadata.no_penalties ==
224 - sampling_metadata.no_penalties)
225 - assert (expected_sampling_metadata.no_top_p == sampling_metadata.no_top_p)
226 - assert (expected_sampling_metadata.no_top_k == sampling_metadata.no_top_k)
237 + assert expected_sampling_metadata.min_tokens == sampling_metadata.min_tokens
238 + assert expected_sampling_metadata.stop_token_ids == \
239 + sampling_metadata.stop_token_ids
240 + assert expected_sampling_metadata.no_penalties == \
241 + sampling_metadata.no_penalties
242 + assert expected_sampling_metadata.no_top_p == sampling_metadata.no_top_p
243 + assert expected_sampling_metadata.no_top_k == sampling_metadata.no_top_k
244 + assert expected_sampling_metadata.logit_bias == sampling_metadata.logit_bias
vllm/sampling_params.py
+3 -1
@@ -243,8 +243,10 @@ class SamplingParams(
243 allowed_token_ids: Optional[List[int]] = None,
244 ) -> "SamplingParams":
245 if logit_bias is not None:
246 + # Convert token_id to integer
247 + # Clamp the bias between -100 and 100 per OpenAI API spec
248 logit_bias = {
247 - int(token): bias
249 + int(token): min(100.0, max(-100.0, bias))
250 for token, bias in logit_bias.items()
251 }
252
vllm/v1/sample/metadata.py
+2
@@ -32,3 +32,5 @@ class SamplingMetadata:
32 output_token_ids: List[List[int]]
33 min_tokens: List[int]
34 stop_token_ids: List[Set[int]]
35 +
36 + logit_bias: List[Optional[Dict[int, float]]]
vllm/v1/sample/sampler.py
+16
@@ -37,6 +37,8 @@ class Sampler(nn.Module):
37
38 # Use float32 for the logits.
39 logits = logits.to(torch.float32)
40 + # Apply logits bias.
41 + logits = self.apply_logits_bias(logits, sampling_metadata)
42 # Apply penalties (e.g., min_tokens, freq_penalties).
43 logits = self.apply_penalties(logits, sampling_metadata)
44 # Apply temperature.
@@ -166,3 +168,17 @@ class Sampler(nn.Module):
168 sampling_metadata.repetition_penalties,
169 sampling_metadata.output_token_ids)
170 return logits
171 +
172 + def apply_logits_bias(
173 + self,
174 + logits: torch.Tensor,
175 + sampling_metadata: SamplingMetadata,
176 + ) -> torch.Tensor:
177 + # TODO(houseroad): this implementation is extremely inefficient.
178 + # One idea is implement this as a PyTorch C++ op, and we may
179 + # even optimize the logit_bias layout.
180 + for i, logit_bias in enumerate(sampling_metadata.logit_bias):
181 + if logit_bias:
182 + for token_id, bias in logit_bias.items():
183 + logits[i, token_id] += bias
184 + return logits
vllm/v1/worker/gpu_input_batch.py
+40 -26
@@ -130,7 +130,7 @@ class InputBatch:
130 device="cpu",
131 pin_memory=pin_memory)
132 self.frequency_penalties_cpu = \
133 - self.frequency_penalties_cpu_tensor.numpy()
133 + self.frequency_penalties_cpu_tensor.numpy()
134 self.frequency_penalties_reqs: Set[str] = set()
135
136 # Presence penalty related data structures
@@ -141,8 +141,8 @@ class InputBatch:
141 dtype=torch.float,
142 device="cpu",
143 pin_memory=pin_memory)
144 - self.presence_penalties_cpu = \
145 - self.presence_penalties_cpu_tensor.numpy()
144 + self.presence_penalties_cpu = self.presence_penalties_cpu_tensor.numpy(
145 + )
146 self.presence_penalties_reqs: Set[str] = set()
147
148 # Repetition penalty related data structures
@@ -155,7 +155,7 @@ class InputBatch:
155 device="cpu",
156 pin_memory=pin_memory)
157 self.repetition_penalties_cpu = \
158 - self.repetition_penalties_cpu_tensor.numpy()
158 + self.repetition_penalties_cpu_tensor.numpy()
159 self.repetition_penalties_reqs: Set[str] = set()
160
161 self.min_tokens: List[int] = [0] * max_num_reqs
@@ -180,6 +180,9 @@ class InputBatch:
180 # that are currently in the prefill phase.
181 self.num_prompt_logprobs: Dict[str, int] = {}
182
183 + self.logit_bias: List[Optional[Dict[int,
184 + float]]] = [None] * max_num_reqs
185 +
186 def add_request(
187 self,
188 request: "CachedRequestState",
@@ -220,16 +223,16 @@ class InputBatch:
223 self.top_k_cpu[req_index] = sampling_params.top_k
224 if sampling_params.top_k > 0:
225 self.top_k_reqs.add(req_id)
223 - self.frequency_penalties_cpu[req_index] = \
224 - sampling_params.frequency_penalty
226 + self.frequency_penalties_cpu[
227 + req_index] = sampling_params.frequency_penalty
228 if sampling_params.frequency_penalty != 0.0:
229 self.frequency_penalties_reqs.add(req_id)
227 - self.presence_penalties_cpu[req_index] = \
228 - sampling_params.presence_penalty
230 + self.presence_penalties_cpu[
231 + req_index] = sampling_params.presence_penalty
232 if sampling_params.presence_penalty != 0.0:
233 self.presence_penalties_reqs.add(req_id)
231 - self.repetition_penalties_cpu[req_index] = \
232 - sampling_params.repetition_penalty
234 + self.repetition_penalties_cpu[
235 + req_index] = sampling_params.repetition_penalty
236 if sampling_params.repetition_penalty != 1.0:
237 self.repetition_penalties_reqs.add(req_id)
238 self.min_tokens[req_index] = sampling_params.min_tokens
@@ -244,6 +247,8 @@ class InputBatch:
247 self.num_logprobs[req_id] = sampling_params.logprobs
248 if sampling_params.prompt_logprobs is not None:
249 self.num_prompt_logprobs[req_id] = sampling_params.prompt_logprobs
250 + if sampling_params.logit_bias is not None:
251 + self.logit_bias[req_index] = sampling_params.logit_bias
252
253 # Add request lora ID
254 if request.lora_request:
@@ -284,6 +289,7 @@ class InputBatch:
289 self.lora_id_to_lora_request.pop(lora_id)
290 self.request_lora_mapping[req_index] = 0
291
292 + self.logit_bias[req_index] = None
293 return req_index
294
295 def clear(self) -> None:
@@ -302,6 +308,7 @@ class InputBatch:
308 self.request_lora_mapping.fill(0)
309 self.lora_id_to_lora_request.clear()
310 self.lora_id_to_request_ids.clear()
311 + self.logit_bias = [None] * self.max_num_reqs
312
313 def condense(self, empty_req_indices: List[int]) -> None:
314 if self.num_reqs == 0:
@@ -332,8 +339,8 @@ class InputBatch:
339 self.token_ids_cpu[empty_index, :num_tokens] = self.token_ids_cpu[
340 last_req_index, :num_tokens]
341 self.num_tokens[empty_index] = num_tokens
335 - self.num_prompt_tokens[empty_index] = \
336 - self.num_prompt_tokens[last_req_index]
342 + self.num_prompt_tokens[empty_index] = self.num_prompt_tokens[
343 + last_req_index]
344 self.num_computed_tokens_cpu[
345 empty_index] = self.num_computed_tokens_cpu[last_req_index]
346 self.block_table.move_row(last_req_index, empty_index)
@@ -341,15 +348,15 @@ class InputBatch:
348 last_req_index]
349 self.top_p_cpu[empty_index] = self.top_p_cpu[last_req_index]
350 self.top_k_cpu[empty_index] = self.top_k_cpu[last_req_index]
344 - self.frequency_penalties_cpu[empty_index] = \
345 - self.frequency_penalties_cpu[last_req_index]
346 - self.presence_penalties_cpu[empty_index] = \
347 - self.presence_penalties_cpu[last_req_index]
348 - self.repetition_penalties_cpu[empty_index] = \
349 - self.repetition_penalties_cpu[last_req_index]
351 + self.frequency_penalties_cpu[
352 + empty_index] = self.frequency_penalties_cpu[last_req_index]
353 + self.presence_penalties_cpu[
354 + empty_index] = self.presence_penalties_cpu[last_req_index]
355 + self.repetition_penalties_cpu[
356 + empty_index] = self.repetition_penalties_cpu[last_req_index]
357 self.min_tokens[empty_index] = self.min_tokens[last_req_index]
351 - self.stop_token_ids[empty_index] = \
352 - self.stop_token_ids[last_req_index]
358 + self.stop_token_ids[empty_index] = self.stop_token_ids[
359 + last_req_index]
360 generator = self.generators.pop(last_req_index, None)
361 if generator is not None:
362 self.generators[empty_index] = generator
@@ -357,6 +364,8 @@ class InputBatch:
364 self.request_lora_mapping[empty_index] = self.request_lora_mapping[
365 last_req_index]
366
367 + self.logit_bias[empty_index] = self.logit_bias[last_req_index]
368 +
369 # Decrement last_req_index since it is now empty.
370 last_req_index -= 1
371
@@ -378,13 +387,16 @@ class InputBatch:
387 # penalties to be applied during sampling.
388 self.frequency_penalties[:self.num_reqs].copy_(
389 self.frequency_penalties_cpu_tensor[:self.num_reqs],
381 - non_blocking=True)
390 + non_blocking=True,
391 + )
392 self.presence_penalties[:self.num_reqs].copy_(
393 self.presence_penalties_cpu_tensor[:self.num_reqs],
384 - non_blocking=True)
394 + non_blocking=True,
395 + )
396 self.repetition_penalties[:self.num_reqs].copy_(
397 self.repetition_penalties_cpu_tensor[:self.num_reqs],
387 - non_blocking=True)
398 + non_blocking=True,
399 + )
400 # The prompt tokens are used only for applying penalties during
401 # the sampling process. Hence copy these tensors only when
402 # there are requests which need penalties to be applied.
@@ -421,6 +433,7 @@ class InputBatch:
433 min_tokens=self.min_tokens[:self.num_reqs],
434 stop_token_ids=self.stop_token_ids[:self.num_reqs],
435 no_penalties=self.no_penalties,
436 + logit_bias=self.logit_bias[:self.num_reqs],
437 )
438
439 def _make_prompt_token_ids_tensor(self) -> torch.Tensor:
@@ -429,10 +442,11 @@ class InputBatch:
442 (self.num_reqs, max_prompt_len),
443 device="cpu",
444 dtype=torch.int64,
432 - pin_memory=self.pin_memory)
445 + pin_memory=self.pin_memory,
446 + )
447 prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()
434 - prompt_token_ids[:] = (
435 - self.token_ids_cpu[:self.num_reqs, :max_prompt_len])
448 + prompt_token_ids[:] = self.token_ids_cpu[:self.
449 + num_reqs, :max_prompt_len]
450 # Use the value of vocab_size as a pad since we don't have a
451 # token_id of this value.
452 for i in range(self.num_reqs):