@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 57a314d155

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 6, 17:00 UTC

[CI][Bugfix] Fix 500 errors from priority overflow and TemplateError subclasses in schema fuzz tests (#37127)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

Andreas Karatzas committed Mar 16, 2026 at 00:27 UTC 57a314d1556cdcb17d26e55e324e21b02bdd9399

8 files changed +32 -18

requirements/rocm-test.txt

@@ -45,6 +45,8 @@ pystemmer==3.0.0

45 # via mteb

47 # Multi-modal processing

48 +av==16.1.0

49 + # required for audio_in_video tests

50 blobfile==3.0.0

51 # Multi-Modal Models Test

52 decord==0.6.0

vllm/entrypoints/openai/chat_completion/protocol.py

+6 -6

@@ -7,7 +7,6 @@ import json

7 import time

8 from typing import Annotated, Any, ClassVar, Literal

10 -import torch

10 from openai.types.chat.chat_completion_audio import (

11 ChatCompletionAudio as OpenAIChatCompletionAudio,

12 )

@@ -48,7 +47,8 @@ from vllm.utils import random_uuid

47 logger = init_logger(__name__)

51 -_LONG_INFO = torch.iinfo(torch.long)

50 +_INT64_MIN = -(2**63)

51 +_INT64_MAX = 2**63 - 1

54 class ChatMessage(OpenAIBaseModel):

@@ -165,7 +165,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

165 n: int | None = 1

166 presence_penalty: float | None = 0.0

167 response_format: AnyResponseFormat | None = None

168 - seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

168 + seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)

169 stop: str | list[str] | None = []

170 stream: bool | None = False

171 stream_options: StreamOptions | None = None

@@ -198,9 +198,7 @@ class ChatCompletionRequest(OpenAIBaseModel):

198 min_tokens: int = 0

199 skip_special_tokens: bool = True

200 spaces_between_special_tokens: bool = True

201 - truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (

202 - None

203 - )

201 + truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_INT64_MAX)] | None = None

202 prompt_logprobs: int | None = None

203 allowed_token_ids: list[int] | None = None

204 bad_words: list[str] = Field(default_factory=list)

@@ -285,6 +283,8 @@ class ChatCompletionRequest(OpenAIBaseModel):

283 )

284 priority: int = Field(

285 default=0,

286 + ge=_INT64_MIN,

287 + le=_INT64_MAX,

288 description=(

289 "The priority of the request (lower means earlier handling; "

290 "default: 0). Any priority other than 0 will raise an error "

vllm/entrypoints/openai/chat_completion/serving.py

+7 -1

@@ -6,6 +6,7 @@ import json

6 import time

7 from collections.abc import AsyncGenerator, AsyncIterator

8 from collections.abc import Sequence as GenericSequence

9 +from http import HTTPStatus

10 from typing import TYPE_CHECKING, Any, Final

12 import partial_json_parser

@@ -1289,7 +1290,12 @@ class OpenAIServingChat(OpenAIServing):

1290 except asyncio.CancelledError:

1291 return self.create_error_response("Client disconnected")

1292

1292 - assert final_res is not None

1293 + if final_res is None:

1294 + return self.create_error_response(

1295 + "No output received from the engine.",

1296 + err_type="InternalServerError",

1297 + status_code=HTTPStatus.INTERNAL_SERVER_ERROR,

1298 + )

1299

1300 choices: list[ChatCompletionResponseChoice] = []

1301 if self.tool_call_id_type == "kimi_k2":

vllm/entrypoints/openai/completion/protocol.py

+6 -6

@@ -7,7 +7,6 @@ import json

7 import time

8 from typing import Annotated, Any, Literal

10 -import torch

10 from pydantic import Field, model_validator

12 from vllm.config import ModelConfig

@@ -36,7 +35,8 @@ from vllm.utils import random_uuid

35 logger = init_logger(__name__)

39 -_LONG_INFO = torch.iinfo(torch.long)

38 +_INT64_MIN = -(2**63)

39 +_INT64_MAX = 2**63 - 1

42 class CompletionRequest(OpenAIBaseModel):

@@ -57,7 +57,7 @@ class CompletionRequest(OpenAIBaseModel):

57 max_tokens: int | None = 16

58 n: int = 1

59 presence_penalty: float | None = 0.0

60 - seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

60 + seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)

61 stop: str | list[str] | None = []

62 stream: bool | None = False

63 stream_options: StreamOptions | None = None

@@ -78,9 +78,7 @@ class CompletionRequest(OpenAIBaseModel):

78 min_tokens: int = 0

79 skip_special_tokens: bool = True

80 spaces_between_special_tokens: bool = True

81 - truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (

82 - None

83 - )

81 + truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_INT64_MAX)] | None = None

82 allowed_token_ids: list[int] | None = None

83 prompt_logprobs: int | None = None

84 # --8<-- [end:completion-sampling-params]

@@ -108,6 +106,8 @@ class CompletionRequest(OpenAIBaseModel):

106 )

107 priority: int = Field(

108 default=0,

109 + ge=_INT64_MIN,

110 + le=_INT64_MAX,

111 description=(

112 "The priority of the request (lower means earlier handling; "

113 "default: 0). Any priority other than 0 will raise an error "

vllm/entrypoints/openai/responses/protocol.py

+5 -3

@@ -6,7 +6,6 @@

6 import time

7 from typing import Any, Literal, TypeAlias

9 -import torch

9 from openai.types.responses import (

10 ResponseCodeInterpreterCallCodeDeltaEvent,

11 ResponseCodeInterpreterCallCodeDoneEvent,

@@ -78,7 +77,8 @@ from vllm.utils import random_uuid

78 logger = init_logger(__name__)

81 -_LONG_INFO = torch.iinfo(torch.long)

80 +_INT64_MIN = -(2**63)

81 +_INT64_MAX = 2**63 - 1

84 class InputTokensDetails(OpenAIBaseModel):

@@ -210,6 +210,8 @@ class ResponsesRequest(OpenAIBaseModel):

210 )

211 priority: int = Field(

212 default=0,

213 + ge=_INT64_MIN,

214 + le=_INT64_MAX,

215 description=(

216 "The priority of the request (lower means earlier handling; "

217 "default: 0). Any priority other than 0 will raise an error "

@@ -246,7 +248,7 @@ class ResponsesRequest(OpenAIBaseModel):

248 )

249

250 repetition_penalty: float | None = None

249 - seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)

251 + seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)

252 stop: str | list[str] | None = []

253 ignore_eos: bool = False

vllm/entrypoints/pooling/base/protocol.py

@@ -34,6 +34,8 @@ class PoolingBasicRequestMixin(OpenAIBaseModel):

34 )

35 priority: int = Field(

36 default=0,

37 + ge=-(2**63),

38 + le=2**63 - 1,

39 description=(

40 "The priority of the request (lower means earlier handling; "

41 "default: 0). Any priority other than 0 will raise an error "

vllm/entrypoints/serve/disagg/protocol.py

@@ -93,6 +93,8 @@ class GenerateRequest(BaseModel):

93 )

94 priority: int = Field(

95 default=0,

96 + ge=-(2**63),

97 + le=2**63 - 1,

98 description=(

99 "The priority of the request (lower means earlier handling; "

100 "default: 0). Any priority other than 0 will raise an error "

vllm/entrypoints/utils.py

+2 -2

@@ -331,8 +331,8 @@ def create_error_response(

331 err_type = "InternalServerError"

332 status_code = exc.status_code

333 param = None

334 - elif exc.__class__.__name__ == "TemplateError":

335 - # jinja2.TemplateError (avoid importing jinja2)

334 + elif any(cls.__name__ == "TemplateError" for cls in type(exc).__mro__):

335 + # jinja2.TemplateError and its subclasses (avoid importing jinja2)

336 err_type = "BadRequestError"

337 status_code = HTTPStatus.BAD_REQUEST

338 param = None