@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6a62cb82cc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix Engine Failing After Invalid Request - AsyncEngineDeadError (#5963)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jun 28, 2024 at 17:46 UTC 6a62cb82ccace1209f7b8bbec95025e047a95ded
1 file changed +24 -10
vllm/entrypoints/openai/protocol.py
+24 -10
@@ -234,15 +234,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
234
235 logits_processors = None
236 if self.logit_bias:
237 + logit_bias: Dict[int, float] = {}
238 + try:
239 + for token_id, bias in self.logit_bias.items():
240 + # Convert token_id to integer before we add to LLMEngine
241 + # Clamp the bias between -100 and 100 per OpenAI API spec
242 + logit_bias[int(token_id)] = min(100, max(-100, bias))
243 + except ValueError as exc:
244 + raise ValueError(f"Found token_id `{token_id}` in logit_bias "
245 + f"but token_id must be an integer or string "
246 + f"representing an integer") from exc
247
248 def logit_bias_logits_processor(
249 token_ids: List[int],
250 logits: torch.Tensor) -> torch.Tensor:
241 - assert self.logit_bias is not None
242 - for token_id, bias in self.logit_bias.items():
243 - # Clamp the bias between -100 and 100 per OpenAI API spec
244 - bias = min(100, max(-100, bias))
245 - logits[int(token_id)] += bias
251 + for token_id, bias in logit_bias.items():
252 + logits[token_id] += bias
253 return logits
254
255 logits_processors = [logit_bias_logits_processor]
@@ -419,15 +426,22 @@ class CompletionRequest(OpenAIBaseModel):
426
427 logits_processors = None
428 if self.logit_bias:
429 + logit_bias: Dict[int, float] = {}
430 + try:
431 + for token_id, bias in self.logit_bias.items():
432 + # Convert token_id to integer
433 + # Clamp the bias between -100 and 100 per OpenAI API spec
434 + logit_bias[int(token_id)] = min(100, max(-100, bias))
435 + except ValueError as exc:
436 + raise ValueError(f"Found token_id `{token_id}` in logit_bias "
437 + f"but token_id must be an integer or string "
438 + f"representing an integer") from exc
439
440 def logit_bias_logits_processor(
441 token_ids: List[int],
442 logits: torch.Tensor) -> torch.Tensor:
426 - assert self.logit_bias is not None
427 - for token_id, bias in self.logit_bias.items():
428 - # Clamp the bias between -100 and 100 per OpenAI API spec
429 - bias = min(100, max(-100, bias))
430 - logits[int(token_id)] += bias
443 + for token_id, bias in logit_bias.items():
444 + logits[token_id] += bias
445 return logits
446
447 logits_processors = [logit_bias_logits_processor]