A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[BugFix][V1] Fix overhead related to bad_words sampling when not in use (#14894)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Mar 16, 2025 at 14:53 UTC
fc1f67715d95f24885288b75c736cc1fc1be0103
3 files changed
+10
-7
tests/v1/worker/test_gpu_input_batch.py
+3
-2
@@ -124,8 +124,9 @@ def _construct_expected_sampling_metadata(
124
if req.sampling_params.allowed_token_ids:
125
allowed_token_ids_mask[index_in_input_batch][
126
req.sampling_params.allowed_token_ids] = True
127
- bad_words_token_ids[
128
- index_in_input_batch] = req.sampling_params.bad_words_token_ids
127
+ if req.sampling_params.bad_words_token_ids:
128
+ bad_words_token_ids[
129
+ index_in_input_batch] = req.sampling_params.bad_words_token_ids
130
131
return SamplingMetadata(
132
temperature=torch.tensor(temperature, dtype=torch.float,
vllm/sampling_params.py
+4
-3
@@ -235,7 +235,7 @@ class SamplingParams(
235
236
# Fields used for bad words
237
bad_words: Optional[list[str]] = None
238
- _bad_words_token_ids: list[list[int]] = msgspec.field(default_factory=list)
238
+ _bad_words_token_ids: Optional[list[list[int]]] = None
239
240
@staticmethod
241
def from_optional(
@@ -464,8 +464,9 @@ class SamplingParams(
464
self.stop_token_ids = list(eos_ids)
465
466
def update_from_tokenizer(self, tokenizer: AnyTokenizer) -> None:
467
- if self.bad_words is None:
467
+ if not self.bad_words:
468
return
469
+ self._bad_words_token_ids = []
470
for bad_word in self.bad_words:
471
# To prohibit words both at the beginning
472
# and in the middle of text
@@ -516,7 +517,7 @@ class SamplingParams(
517
return self._all_stop_token_ids
518
519
@property
519
- def bad_words_token_ids(self) -> list[list[int]]:
520
+ def bad_words_token_ids(self) -> Optional[list[list[int]]]:
521
# For internal use only. Backward compatibility not guaranteed
522
return self._bad_words_token_ids
523
vllm/v1/worker/gpu_input_batch.py
+3
-2
@@ -324,8 +324,9 @@ class InputBatch:
324
self.allowed_token_ids_mask_cpu_tensor[req_index][
325
sampling_params.allowed_token_ids] = False
326
327
- self.bad_words_token_ids[
328
- req_index] = sampling_params.bad_words_token_ids
327
+ if sampling_params.bad_words_token_ids:
328
+ self.bad_words_token_ids[
329
+ req_index] = sampling_params.bad_words_token_ids
330
331
# Add request lora ID
332
if request.lora_request: