A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Core] Optimize update checks in LogitsProcessor (#21245)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com>
Jialin Ouyang committed
Jul 22, 2025 at 05:27 UTC
a32237665df876fcb51196dc209e8aff9fd89d29
1 file changed
+13
-5
vllm/v1/sample/logits_processor.py
+13
-5
@@ -335,14 +335,19 @@ class LogitBiasLogitsProcessor(LogitsProcessor):
335
if not batch_update:
336
return
337
338
+ needs_update: bool = False
339
# Process added requests.
339
- needs_update = bool(batch_update.added)
340
for index, params, _ in batch_update.added:
341
if isinstance(params, SamplingParams) and (lb :=
342
params.logit_bias):
343
self.biases[index] = lb
344
+ needs_update = True
345
else:
345
- self.biases.pop(index, None)
346
+ # Drop biases metadata at batch index
347
+ if self.biases.pop(index, None) is not None:
348
+ # If a new request replaces an old request which
349
+ # specified biases, we should update processor tensors
350
+ needs_update = True
351
352
if self.biases:
353
# Process removed requests.
@@ -419,7 +424,6 @@ class MinTokensLogitsProcessor(LogitsProcessor):
424
425
if batch_update:
426
# Process added requests.
422
- needs_update |= bool(batch_update.added)
427
for index, params, output_tok_ids in batch_update.added:
428
if (isinstance(params, SamplingParams)
429
and (min_tokens := params.min_tokens)
@@ -427,9 +431,13 @@ class MinTokensLogitsProcessor(LogitsProcessor):
431
# Replace request metadata at batch index
432
self.min_toks[index] = (min_tokens, output_tok_ids,
433
params.all_stop_token_ids)
434
+ needs_update = True
435
else:
431
- # Drop request metadata at batch index
432
- self.min_toks.pop(index, None)
436
+ # Drop min_toks metadata at batch index
437
+ if self.min_toks.pop(index, None) is not None:
438
+ # If a new request replaces an old request which
439
+ # specified min_toks, we should update processor tensors
440
+ needs_update = True
441
442
if self.min_toks:
443
# Process removed requests.