A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix Engine Failing After Invalid Request - AsyncEngineDeadError (#5963)
Co-authored-by: Robert Shaw <rshaw@neuralmagic>
Robert Shaw committed
Jun 28, 2024 at 17:46 UTC
6a62cb82ccace1209f7b8bbec95025e047a95ded
1 file changed
+24
-10
vllm/entrypoints/openai/protocol.py
+24
-10
@@ -234,15 +234,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
234
235
logits_processors = None
236
if self.logit_bias:
237
+ logit_bias: Dict[int, float] = {}
238
+ try:
239
+ for token_id, bias in self.logit_bias.items():
240
+ # Convert token_id to integer before we add to LLMEngine
241
+ # Clamp the bias between -100 and 100 per OpenAI API spec
242
+ logit_bias[int(token_id)] = min(100, max(-100, bias))
243
+ except ValueError as exc:
244
+ raise ValueError(f"Found token_id `{token_id}` in logit_bias "
245
+ f"but token_id must be an integer or string "
246
+ f"representing an integer") from exc
247
248
def logit_bias_logits_processor(
249
token_ids: List[int],
250
logits: torch.Tensor) -> torch.Tensor:
241
- assert self.logit_bias is not None
242
- for token_id, bias in self.logit_bias.items():
243
- # Clamp the bias between -100 and 100 per OpenAI API spec
244
- bias = min(100, max(-100, bias))
245
- logits[int(token_id)] += bias
251
+ for token_id, bias in logit_bias.items():
252
+ logits[token_id] += bias
253
return logits
254
255
logits_processors = [logit_bias_logits_processor]
@@ -419,15 +426,22 @@ class CompletionRequest(OpenAIBaseModel):
426
427
logits_processors = None
428
if self.logit_bias:
429
+ logit_bias: Dict[int, float] = {}
430
+ try:
431
+ for token_id, bias in self.logit_bias.items():
432
+ # Convert token_id to integer
433
+ # Clamp the bias between -100 and 100 per OpenAI API spec
434
+ logit_bias[int(token_id)] = min(100, max(-100, bias))
435
+ except ValueError as exc:
436
+ raise ValueError(f"Found token_id `{token_id}` in logit_bias "
437
+ f"but token_id must be an integer or string "
438
+ f"representing an integer") from exc
439
440
def logit_bias_logits_processor(
441
token_ids: List[int],
442
logits: torch.Tensor) -> torch.Tensor:
426
- assert self.logit_bias is not None
427
- for token_id, bias in self.logit_bias.items():
428
- # Clamp the bias between -100 and 100 per OpenAI API spec
429
- bias = min(100, max(-100, bias))
430
- logits[int(token_id)] += bias
443
+ for token_id, bias in logit_bias.items():
444
+ logits[token_id] += bias
445
return logits
446
447
logits_processors = [logit_bias_logits_processor]