A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[Refactor] [6/N] to simplify the vLLM openai chat_completion serving architecture (#32240)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jan 13, 2026 at 21:01 UTC
fefce49807a52fc94678e336fc5d1f3b7ebe2daf
128 files changed
+1310
-1097
tests/entrypoints/openai/responses/test_errors.py
+2
-2
@@ -6,8 +6,8 @@ from unittest.mock import MagicMock
6
7
import pytest
8
9
-from vllm.entrypoints.openai.protocol import ErrorResponse
10
-from vllm.entrypoints.openai.serving_engine import GenerationError, OpenAIServing
9
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
+from vllm.entrypoints.openai.engine.serving import GenerationError, OpenAIServing
11
12
13
@pytest.mark.asyncio
tests/entrypoints/openai/responses/test_function_call_parsing.py
+2
-2
@@ -7,7 +7,7 @@ import json
7
import pytest
8
from openai.types.responses import ResponseFunctionToolCall
9
10
-from vllm.entrypoints.openai.protocol import ResponsesRequest
10
+from vllm.entrypoints.openai.engine.protocol import ResponsesRequest
11
12
13
def test_function_call_dict_converted_to_object():
@@ -253,7 +253,7 @@ def test_function_call_validation_failure_logs_debug(caplog):
253
}
254
255
# Mock the logger to verify debug was called
256
- with patch("vllm.entrypoints.openai.protocol.logger") as mock_logger:
256
+ with patch("vllm.entrypoints.openai.engine.protocol.logger") as mock_logger:
257
with pytest.raises(ValueError):
258
ResponsesRequest(**request_data)
259
tests/entrypoints/openai/test_chat_error.py
+3
-2
@@ -9,8 +9,9 @@ from unittest.mock import AsyncMock, MagicMock
9
import pytest
10
11
from vllm.config.multimodal import MultiModalConfig
12
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, ErrorResponse
13
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
12
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
13
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
14
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
16
from vllm.outputs import CompletionOutput, RequestOutput
17
from vllm.tokenizers import get_tokenizer
tests/entrypoints/openai/test_chat_template.py
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
6
from vllm.config import ModelConfig
7
from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9
from vllm.tokenizers import get_tokenizer
10
11
from ...models.registry import HF_EXAMPLE_MODELS
tests/entrypoints/openai/test_completion_error.py
+1
-1
@@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
9
import pytest
10
11
from vllm.config.multimodal import MultiModalConfig
12
-from vllm.entrypoints.openai.protocol import CompletionRequest, ErrorResponse
12
+from vllm.entrypoints.openai.engine.protocol import CompletionRequest, ErrorResponse
13
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
14
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
15
from vllm.outputs import CompletionOutput, RequestOutput
tests/entrypoints/openai/test_gptoss_structural_tags_integration.py
+1
-1
@@ -8,7 +8,7 @@ from unittest.mock import Mock
8
9
import pytest
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.engine.protocol import (
12
StructuredOutputsParams,
13
)
14
from vllm.entrypoints.tool_server import ToolServer
tests/entrypoints/openai/test_lora_resolvers.py
+1
-1
@@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
9
import pytest
10
11
from vllm.config.multimodal import MultiModalConfig
12
-from vllm.entrypoints.openai.protocol import CompletionRequest, ErrorResponse
12
+from vllm.entrypoints.openai.engine.protocol import CompletionRequest, ErrorResponse
13
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
14
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
15
from vllm.lora.request import LoRARequest
tests/entrypoints/openai/test_protocol.py
+4
-1
@@ -4,7 +4,10 @@ from openai_harmony import (
4
Message,
5
)
6
7
-from vllm.entrypoints.openai.protocol import serialize_message, serialize_messages
7
+from vllm.entrypoints.openai.engine.protocol import (
8
+ serialize_message,
9
+ serialize_messages,
10
+)
11
12
13
def test_serialize_message() -> None:
tests/entrypoints/openai/test_serving_chat.py
+13
-11
@@ -11,14 +11,16 @@ import pytest_asyncio
11
from openai import OpenAI
12
13
from vllm.config.multimodal import MultiModalConfig
14
-from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
15
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
ChatCompletionRequest,
16
ChatCompletionResponse,
17
+)
18
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
19
+from vllm.entrypoints.openai.engine.protocol import (
20
ErrorResponse,
21
RequestResponseMetadata,
22
)
21
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
23
+from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
24
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
25
from vllm.outputs import CompletionOutput, RequestOutput
26
from vllm.tokenizers import get_tokenizer
@@ -1517,12 +1519,12 @@ class TestCreateRemainingArgsDelta:
1519
1520
def test_preserves_id_type_name(self):
1521
"""Test that id, type, and name are preserved from original delta."""
1520
- from vllm.entrypoints.openai.protocol import (
1522
+ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
1523
+ from vllm.entrypoints.openai.engine.protocol import (
1524
DeltaFunctionCall,
1525
DeltaMessage,
1526
DeltaToolCall,
1527
)
1525
- from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1528
1529
original_delta = DeltaMessage(
1530
tool_calls=[
@@ -1552,12 +1554,12 @@ class TestCreateRemainingArgsDelta:
1554
1555
def test_matches_by_index(self):
1556
"""Test that the correct tool call is matched by index."""
1555
- from vllm.entrypoints.openai.protocol import (
1557
+ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
1558
+ from vllm.entrypoints.openai.engine.protocol import (
1559
DeltaFunctionCall,
1560
DeltaMessage,
1561
DeltaToolCall,
1562
)
1560
- from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1563
1564
original_delta = DeltaMessage(
1565
tool_calls=[
@@ -1588,12 +1590,12 @@ class TestCreateRemainingArgsDelta:
1590
1591
def test_no_matching_tool_call(self):
1592
"""Test graceful handling when no matching tool call is found."""
1591
- from vllm.entrypoints.openai.protocol import (
1593
+ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
1594
+ from vllm.entrypoints.openai.engine.protocol import (
1595
DeltaFunctionCall,
1596
DeltaMessage,
1597
DeltaToolCall,
1598
)
1596
- from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1599
1600
original_delta = DeltaMessage(
1601
tool_calls=[
@@ -1620,8 +1622,8 @@ class TestCreateRemainingArgsDelta:
1622
1623
def test_function_is_none(self):
1624
"""Test handling when original tool call has no function."""
1623
- from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall
1624
- from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
1625
+ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
1626
+ from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
1627
1628
original_delta = DeltaMessage(
1629
tool_calls=[
tests/entrypoints/openai/test_serving_chat_stream_harmony.py
+2
-2
@@ -9,7 +9,7 @@ from unittest.mock import patch
9
10
import pytest
11
12
-from vllm.entrypoints.openai.serving_chat_stream_harmony import (
12
+from vllm.entrypoints.openai.chat_completion.stream_harmony import (
13
extract_harmony_streaming_delta,
14
)
15
@@ -82,7 +82,7 @@ class TestExtractHarmonyStreamingDelta:
82
assert tools_streamed is False
83
84
@pytest.mark.parametrize("channel", ["commentary", "analysis"])
85
- @patch("vllm.entrypoints.openai.serving_chat_stream_harmony.make_tool_call_id")
85
+ @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
86
def test_new_tool_call(self, mock_make_tool_call_id, channel):
87
"""Test new tool call creation when recipient changes."""
88
mock_make_tool_call_id.return_value = "call_test123"
tests/entrypoints/openai/test_serving_engine.py
+1
-1
@@ -8,7 +8,7 @@ from unittest.mock import Mock
8
import pytest
9
10
from vllm.config import ModelConfig
11
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
11
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
12
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
13
from vllm.tokenizers.mistral import MistralTokenizer
14
tests/entrypoints/openai/test_serving_models.py
+1
-1
@@ -8,7 +8,7 @@ import pytest
8
9
from vllm.config import ModelConfig
10
from vllm.engine.protocol import EngineClient
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.engine.protocol import (
12
ErrorResponse,
13
LoadLoRAAdapterRequest,
14
UnloadLoRAAdapterRequest,
tests/entrypoints/openai/test_serving_responses.py
+1
-1
@@ -14,7 +14,7 @@ from openai.types.responses.tool import (
14
)
15
16
from vllm.entrypoints.context import ConversationContext
17
-from vllm.entrypoints.openai.protocol import ErrorResponse, ResponsesRequest
17
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse, ResponsesRequest
18
from vllm.entrypoints.openai.serving_responses import (
19
OpenAIServingResponses,
20
_extract_allowed_tools_from_mcp_requests,
tests/entrypoints/openai/tool_parsers/test_gigachat3_tool_parser.py
+1
-1
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
9
run_tool_extraction,
10
run_tool_extraction_streaming,
11
)
12
-from vllm.entrypoints.openai.protocol import FunctionCall
12
+from vllm.entrypoints.openai.engine.protocol import FunctionCall
13
from vllm.tokenizers import TokenizerLike
14
from vllm.tool_parsers import ToolParser, ToolParserManager
15
tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py
+1
-1
@@ -5,7 +5,7 @@ import json
5
6
import pytest
7
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9
from vllm.tokenizers import TokenizerLike
10
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
11
tests/entrypoints/openai/tool_parsers/test_hunyuan_a13b_tool_parser.py
+1
-1
@@ -11,7 +11,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
11
run_tool_extraction,
12
run_tool_extraction_streaming,
13
)
14
-from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
14
+from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
15
from vllm.tool_parsers import ToolParser, ToolParserManager
16
17
tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py
+1
-1
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch
5
6
import pytest
7
8
-from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation
8
+from vllm.entrypoints.openai.engine.protocol import ExtractedToolCallInformation
9
from vllm.tokenizers import TokenizerLike
10
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
11
tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py
+1
-1
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
9
run_tool_extraction,
10
run_tool_extraction_streaming,
11
)
12
-from vllm.entrypoints.openai.protocol import FunctionCall
12
+from vllm.entrypoints.openai.engine.protocol import FunctionCall
13
from vllm.tokenizers import TokenizerLike
14
from vllm.tool_parsers import ToolParser, ToolParserManager
15
tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py
+1
-1
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
9
run_tool_extraction,
10
run_tool_extraction_streaming,
11
)
12
-from vllm.entrypoints.openai.protocol import FunctionCall
12
+from vllm.entrypoints.openai.engine.protocol import FunctionCall
13
from vllm.tokenizers import TokenizerLike
14
from vllm.tool_parsers import ToolParser, ToolParserManager
15
tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py
+1
-1
@@ -9,7 +9,7 @@ from tests.entrypoints.openai.tool_parsers.utils import (
9
run_tool_extraction,
10
run_tool_extraction_streaming,
11
)
12
-from vllm.entrypoints.openai.protocol import FunctionCall
12
+from vllm.entrypoints.openai.engine.protocol import FunctionCall
13
from vllm.tokenizers import TokenizerLike
14
from vllm.tool_parsers import ToolParser, ToolParserManager
15
tests/entrypoints/openai/tool_parsers/utils.py
+2
-2
@@ -3,8 +3,8 @@
3
4
from collections.abc import Iterable
5
6
-from vllm.entrypoints.openai.protocol import (
7
- ChatCompletionRequest,
6
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
7
+from vllm.entrypoints.openai.engine.protocol import (
8
DeltaMessage,
9
ExtractedToolCallInformation,
10
FunctionCall,
tests/entrypoints/openai/utils.py
+3
-1
@@ -4,11 +4,13 @@ import json
4
from collections.abc import AsyncGenerator
5
from typing import Any
6
7
-from vllm.entrypoints.openai.protocol import (
7
+from vllm.entrypoints.openai.chat_completion.protocol import (
8
ChatCompletionResponse,
9
ChatCompletionResponseChoice,
10
ChatCompletionStreamResponse,
11
ChatMessage,
12
+)
13
+from vllm.entrypoints.openai.engine.protocol import (
14
UsageInfo,
15
)
16
tests/reasoning/test_base_thinking_reasoning_parser.py
+1
-1
@@ -5,7 +5,7 @@ import pytest
5
from transformers import AutoTokenizer
6
7
from tests.reasoning.utils import run_reasoning_extraction
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
10
11
tests/reasoning/test_deepseekv3_reasoning_parser.py
+2
-1
@@ -4,7 +4,8 @@
4
import pytest
5
from transformers import AutoTokenizer
6
7
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
7
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
8
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
9
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
10
from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
11
from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
tests/reasoning/utils.py
+2
-2
@@ -1,8 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-
5
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
4
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
5
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
6
from vllm.reasoning import ReasoningParser
7
from vllm.tokenizers.mistral import MistralTokenizer
8
tests/tool_parsers/test_ernie45_moe_tool_parser.py
+2
-2
@@ -7,8 +7,8 @@ from collections.abc import Generator
7
8
import pytest
9
10
-from vllm.entrypoints.openai.protocol import (
11
- ChatCompletionRequest,
10
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
11
+from vllm.entrypoints.openai.engine.protocol import (
12
DeltaMessage,
13
FunctionCall,
14
ToolCall,
tests/tool_parsers/test_functiongemma_tool_parser.py
+1
-1
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock
5
6
import pytest
7
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
8
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
9
from vllm.tool_parsers.functiongemma_tool_parser import FunctionGemmaToolParser
10
11
tests/tool_parsers/test_glm4_moe_tool_parser.py
+1
-1
@@ -6,7 +6,7 @@ import json
6
7
import pytest
8
9
-from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
9
+from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
10
from vllm.tokenizers import get_tokenizer
11
from vllm.tool_parsers.glm4_moe_tool_parser import (
12
Glm4MoeModelToolParser,
tests/tool_parsers/test_jamba_tool_parser.py
+1
-1
@@ -8,7 +8,7 @@ import partial_json_parser
8
import pytest
9
from partial_json_parser.core.options import Allow
10
11
-from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall
11
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage, FunctionCall, ToolCall
12
from vllm.tokenizers import TokenizerLike, get_tokenizer
13
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
14
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
tests/tool_parsers/test_kimi_k2_tool_parser.py
+1
-1
@@ -6,7 +6,7 @@ import json
6
7
import pytest
8
9
-from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
9
+from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
10
from vllm.tokenizers import get_tokenizer
11
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
12
tests/tool_parsers/test_minimax_tool_parser.py
+3
-1
@@ -7,8 +7,10 @@ from typing import Any
7
8
import pytest
9
10
-from vllm.entrypoints.openai.protocol import (
10
+from vllm.entrypoints.openai.chat_completion.protocol import (
11
ChatCompletionToolsParam,
12
+)
13
+from vllm.entrypoints.openai.engine.protocol import (
14
FunctionCall,
15
ToolCall,
16
)
tests/tool_parsers/test_mistral_tool_parser.py
+1
-1
@@ -11,7 +11,7 @@ from mistral_common.protocol.instruct.request import InstructRequest
11
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
12
from partial_json_parser.core.options import Allow
13
14
-from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall
14
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage, DeltaToolCall
15
from vllm.tokenizers import TokenizerLike, get_tokenizer
16
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
17
from vllm.tokenizers.mistral import MistralTokenizer
tests/tool_parsers/test_openai_tool_parser.py
+1
-1
@@ -14,7 +14,7 @@ from openai_harmony import (
14
load_harmony_encoding,
15
)
16
17
-from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
17
+from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall
18
from vllm.tokenizers import get_tokenizer
19
from vllm.tool_parsers.openai_tool_parser import OpenAIToolParser
20
tests/tool_parsers/test_qwen3coder_tool_parser.py
+3
-1
@@ -6,9 +6,11 @@ from collections.abc import Generator
6
7
import pytest
8
9
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
ChatCompletionRequest,
11
ChatCompletionToolsParam,
12
+)
13
+from vllm.entrypoints.openai.engine.protocol import (
14
DeltaMessage,
15
FunctionCall,
16
ToolCall,
tests/tool_parsers/test_seed_oss_tool_parser.py
+3
-1
@@ -7,9 +7,11 @@ from collections.abc import Generator
7
8
import pytest
9
10
-from vllm.entrypoints.openai.protocol import (
10
+from vllm.entrypoints.openai.chat_completion.protocol import (
11
ChatCompletionRequest,
12
ChatCompletionToolsParam,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaMessage,
16
FunctionCall,
17
ToolCall,
tests/tool_parsers/test_xlam_tool_parser.py
+2
-2
@@ -6,8 +6,8 @@ from collections.abc import Generator
6
7
import pytest
8
9
-from vllm.entrypoints.openai.protocol import (
10
- ChatCompletionRequest,
9
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
10
+from vllm.entrypoints.openai.engine.protocol import (
11
DeltaMessage,
12
FunctionCall,
13
ToolCall,
tests/tool_use/test_chat_completion_request_validations.py
+1
-1
@@ -3,7 +3,7 @@
3
4
import pytest
5
6
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
6
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
7
8
9
def test_chat_completion_request_with_no_tools():
tests/tool_use/test_tool_choice_required.py
+2
-2
@@ -8,10 +8,10 @@ import pytest
8
import regex as re
9
from pydantic import TypeAdapter
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionToolsParam,
13
)
14
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
14
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
15
from vllm.tool_parsers.utils import get_json_schema_from_tools
16
17
pytestmark = pytest.mark.cpu_test
tests/v1/engine/test_async_llm.py
+4
-2
@@ -11,12 +11,14 @@ from vllm import SamplingParams
11
from vllm.assets.image import ImageAsset
12
from vllm.config import VllmConfig
13
from vllm.engine.arg_utils import AsyncEngineArgs
14
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
ChatCompletionRequest,
16
ChatCompletionResponse,
17
+)
18
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
19
+from vllm.entrypoints.openai.engine.protocol import (
20
ErrorResponse,
21
)
19
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
22
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
23
from vllm.inputs import PromptType
24
from vllm.outputs import RequestOutput
vllm/entrypoints/anthropic/serving_messages.py
+4
-2
@@ -25,16 +25,18 @@ from vllm.entrypoints.anthropic.protocol import (
25
)
26
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
27
from vllm.entrypoints.logger import RequestLogger
28
-from vllm.entrypoints.openai.protocol import (
28
+from vllm.entrypoints.openai.chat_completion.protocol import (
29
ChatCompletionNamedToolChoiceParam,
30
ChatCompletionRequest,
31
ChatCompletionResponse,
32
ChatCompletionStreamResponse,
33
ChatCompletionToolsParam,
34
+)
35
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
36
+from vllm.entrypoints.openai.engine.protocol import (
37
ErrorResponse,
38
StreamOptions,
39
)
37
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
40
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
41
42
logger = logging.getLogger(__name__)
vllm/entrypoints/context.py
+6
-6
@@ -22,6 +22,12 @@ from vllm.entrypoints.chat_utils import (
22
ChatTemplateContentFormatOption,
23
)
24
from vllm.entrypoints.constants import MCP_PREFIX
25
+from vllm.entrypoints.openai.engine.protocol import (
26
+ FunctionCall,
27
+ ResponseInputOutputItem,
28
+ ResponseRawMessageAndToken,
29
+ ResponsesRequest,
30
+)
31
from vllm.entrypoints.openai.parser.harmony_utils import (
32
get_encoding,
33
get_streamable_parser_for_assistant,
@@ -30,12 +36,6 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
36
from vllm.entrypoints.openai.parser.responses_parser import (
37
get_responses_parser_for_simple_context,
38
)
33
-from vllm.entrypoints.openai.protocol import (
34
- FunctionCall,
35
- ResponseInputOutputItem,
36
- ResponseRawMessageAndToken,
37
- ResponsesRequest,
38
-)
39
from vllm.entrypoints.responses_utils import construct_tool_dicts
40
from vllm.entrypoints.tool import Tool
41
from vllm.entrypoints.tool_server import ToolServer
vllm/entrypoints/openai/api_server.py
+11
-48
@@ -42,11 +42,9 @@ from vllm.entrypoints.anthropic.protocol import (
42
from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
43
from vllm.entrypoints.launcher import serve_http
44
from vllm.entrypoints.logger import RequestLogger
45
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
46
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
46
-from vllm.entrypoints.openai.orca_metrics import metrics_header
47
-from vllm.entrypoints.openai.protocol import (
48
- ChatCompletionRequest,
49
- ChatCompletionResponse,
47
+from vllm.entrypoints.openai.engine.protocol import (
48
CompletionRequest,
49
CompletionResponse,
50
ErrorInfo,
@@ -59,9 +57,9 @@ from vllm.entrypoints.openai.protocol import (
57
TranslationRequest,
58
TranslationResponseVariant,
59
)
62
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
60
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
61
+from vllm.entrypoints.openai.orca_metrics import metrics_header
62
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
64
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
63
from vllm.entrypoints.openai.serving_models import (
64
BaseModelPath,
65
OpenAIServingModels,
@@ -475,47 +473,6 @@ async def create_messages(request: AnthropicMessagesRequest, raw_request: Reques
473
return StreamingResponse(content=generator, media_type="text/event-stream")
474
475
478
-@router.post(
479
- "/v1/chat/completions",
480
- dependencies=[Depends(validate_json_request)],
481
- responses={
482
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
483
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
484
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
485
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
486
- },
487
-)
488
-@with_cancellation
489
-@load_aware_call
490
-async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request):
491
- metrics_header_format = raw_request.headers.get(
492
- ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
493
- )
494
- handler = chat(raw_request)
495
- if handler is None:
496
- return base(raw_request).create_error_response(
497
- message="The model does not support Chat Completions API"
498
- )
499
- try:
500
- generator = await handler.create_chat_completion(request, raw_request)
501
- except Exception as e:
502
- raise HTTPException(
503
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
504
- ) from e
505
- if isinstance(generator, ErrorResponse):
506
- return JSONResponse(
507
- content=generator.model_dump(), status_code=generator.error.code
508
- )
509
-
510
- elif isinstance(generator, ChatCompletionResponse):
511
- return JSONResponse(
512
- content=generator.model_dump(),
513
- headers=metrics_header(metrics_header_format),
514
- )
515
-
516
- return StreamingResponse(content=generator, media_type="text/event-stream")
517
-
518
-
476
@router.post(
477
"/v1/completions",
478
dependencies=[Depends(validate_json_request)],
@@ -735,8 +692,10 @@ class XRequestIdMiddleware:
692
def _extract_content_from_chunk(chunk_data: dict) -> str:
693
"""Extract content from a streaming response chunk."""
694
try:
738
- from vllm.entrypoints.openai.protocol import (
695
+ from vllm.entrypoints.openai.chat_completion.protocol import (
696
ChatCompletionStreamResponse,
697
+ )
698
+ from vllm.entrypoints.openai.engine.protocol import (
699
CompletionStreamResponse,
700
)
701
@@ -880,7 +839,11 @@ def build_app(args: Namespace) -> FastAPI:
839
from vllm.entrypoints.serve import register_vllm_serve_api_routers
840
841
register_vllm_serve_api_routers(app)
842
+ from vllm.entrypoints.openai.chat_completion.api_router import (
843
+ attach_router as register_chat_api_router,
844
+ )
845
846
+ register_chat_api_router(app)
847
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
848
849
register_sagemaker_routes(router)
vllm/entrypoints/openai/chat_completion/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/openai/chat_completion/api_router.py
new
+77
@@ -0,0 +1,77 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+
7
+from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
8
+from fastapi.responses import JSONResponse, StreamingResponse
9
+
10
+from vllm.entrypoints.openai.chat_completion.protocol import (
11
+ ChatCompletionRequest,
12
+ ChatCompletionResponse,
13
+)
14
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
15
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
16
+from vllm.entrypoints.openai.orca_metrics import metrics_header
17
+from vllm.entrypoints.openai.utils import validate_json_request
18
+from vllm.entrypoints.utils import (
19
+ load_aware_call,
20
+ with_cancellation,
21
+)
22
+from vllm.logger import init_logger
23
+
24
+logger = init_logger(__name__)
25
+
26
+router = APIRouter()
27
+ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
28
+
29
+
30
+def chat(request: Request) -> OpenAIServingChat | None:
31
+ return request.app.state.openai_serving_chat
32
+
33
+
34
+@router.post(
35
+ "/v1/chat/completions",
36
+ dependencies=[Depends(validate_json_request)],
37
+ responses={
38
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
39
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
40
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
41
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
42
+ },
43
+)
44
+@with_cancellation
45
+@load_aware_call
46
+async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request):
47
+ metrics_header_format = raw_request.headers.get(
48
+ ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
49
+ )
50
+ handler = chat(raw_request)
51
+ if handler is None:
52
+ base_server = raw_request.app.state.openai_serving_tokenization
53
+ return base_server.create_error_response(
54
+ message="The model does not support Chat Completions API"
55
+ )
56
+ try:
57
+ generator = await handler.create_chat_completion(request, raw_request)
58
+ except Exception as e:
59
+ raise HTTPException(
60
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
61
+ ) from e
62
+ if isinstance(generator, ErrorResponse):
63
+ return JSONResponse(
64
+ content=generator.model_dump(), status_code=generator.error.code
65
+ )
66
+
67
+ elif isinstance(generator, ChatCompletionResponse):
68
+ return JSONResponse(
69
+ content=generator.model_dump(),
70
+ headers=metrics_header(metrics_header_format),
71
+ )
72
+
73
+ return StreamingResponse(content=generator, media_type="text/event-stream")
74
+
75
+
76
+def attach_router(app: FastAPI):
77
+ app.include_router(router)
vllm/entrypoints/openai/chat_completion/protocol.py
new
+654
@@ -0,0 +1,654 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
+import json
7
+import time
8
+from typing import Annotated, Any, ClassVar, Literal
9
+
10
+import torch
11
+from openai.types.chat.chat_completion_audio import (
12
+ ChatCompletionAudio as OpenAIChatCompletionAudio,
13
+)
14
+from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
15
+from pydantic import (
16
+ Field,
17
+ model_validator,
18
+)
19
+
20
+from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
21
+from vllm.entrypoints.openai.engine.protocol import (
22
+ AnyResponseFormat,
23
+ DeltaMessage,
24
+ FunctionCall,
25
+ FunctionDefinition,
26
+ LegacyStructuralTagResponseFormat,
27
+ LogitsProcessors,
28
+ OpenAIBaseModel,
29
+ StreamOptions,
30
+ StructuralTagResponseFormat,
31
+ ToolCall,
32
+ UsageInfo,
33
+ get_logits_processors,
34
+)
35
+from vllm.exceptions import VLLMValidationError
36
+from vllm.logger import init_logger
37
+from vllm.logprobs import Logprob
38
+from vllm.sampling_params import (
39
+ BeamSearchParams,
40
+ RequestOutputKind,
41
+ SamplingParams,
42
+ StructuredOutputsParams,
43
+)
44
+from vllm.utils import random_uuid
45
+
46
+logger = init_logger(__name__)
47
+
48
+
49
+_LONG_INFO = torch.iinfo(torch.long)
50
+
51
+
52
+class ChatMessage(OpenAIBaseModel):
53
+ role: str
54
+ content: str | None = None
55
+ refusal: str | None = None
56
+ annotations: OpenAIAnnotation | None = None
57
+ audio: OpenAIChatCompletionAudio | None = None
58
+ function_call: FunctionCall | None = None
59
+ tool_calls: list[ToolCall] = Field(default_factory=list)
60
+
61
+ # vLLM-specific fields that are not in OpenAI spec
62
+ reasoning: str | None = None
63
+ reasoning_content: str | None = None
64
+ """Deprecated: use `reasoning` instead."""
65
+
66
+ @model_validator(mode="after")
67
+ def handle_deprecated_reasoning_content(self):
68
+ """Copy reasoning to reasoning_content for backward compatibility."""
69
+ self.reasoning_content = self.reasoning
70
+ return self
71
+
72
+
73
+class ChatCompletionLogProb(OpenAIBaseModel):
74
+ token: str
75
+ logprob: float = -9999.0
76
+ bytes: list[int] | None = None
77
+
78
+
79
+class ChatCompletionLogProbsContent(ChatCompletionLogProb):
80
+ # Workaround: redefine fields name cache so that it's not
81
+ # shared with the super class.
82
+ field_names: ClassVar[set[str] | None] = None
83
+ top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
84
+
85
+
86
+class ChatCompletionLogProbs(OpenAIBaseModel):
87
+ content: list[ChatCompletionLogProbsContent] | None = None
88
+
89
+
90
+class ChatCompletionResponseChoice(OpenAIBaseModel):
91
+ index: int
92
+ message: ChatMessage
93
+ logprobs: ChatCompletionLogProbs | None = None
94
+ # per OpenAI spec this is the default
95
+ finish_reason: str | None = "stop"
96
+ # not part of the OpenAI spec but included in vLLM for legacy reasons
97
+ stop_reason: int | str | None = None
98
+ # not part of the OpenAI spec but is useful for tracing the tokens
99
+ # in agent scenarios
100
+ token_ids: list[int] | None = None
101
+
102
+
103
+class ChatCompletionResponse(OpenAIBaseModel):
104
+ id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
105
+ object: Literal["chat.completion"] = "chat.completion"
106
+ created: int = Field(default_factory=lambda: int(time.time()))
107
+ model: str
108
+ choices: list[ChatCompletionResponseChoice]
109
+ service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
110
+ system_fingerprint: str | None = None
111
+ usage: UsageInfo
112
+
113
+ # vLLM-specific fields that are not in OpenAI spec
114
+ prompt_logprobs: list[dict[int, Logprob] | None] | None = None
115
+ prompt_token_ids: list[int] | None = None
116
+ kv_transfer_params: dict[str, Any] | None = Field(
117
+ default=None, description="KVTransfer parameters."
118
+ )
119
+
120
+
121
+class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
122
+ index: int
123
+ delta: DeltaMessage
124
+ logprobs: ChatCompletionLogProbs | None = None
125
+ finish_reason: str | None = None
126
+ stop_reason: int | str | None = None
127
+ # not part of the OpenAI spec but for tracing the tokens
128
+ token_ids: list[int] | None = None
129
+
130
+
131
+class ChatCompletionStreamResponse(OpenAIBaseModel):
132
+ id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
133
+ object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
134
+ created: int = Field(default_factory=lambda: int(time.time()))
135
+ model: str
136
+ choices: list[ChatCompletionResponseStreamChoice]
137
+ usage: UsageInfo | None = Field(default=None)
138
+ # not part of the OpenAI spec but for tracing the tokens
139
+ prompt_token_ids: list[int] | None = None
140
+
141
+
142
+class ChatCompletionToolsParam(OpenAIBaseModel):
143
+ type: Literal["function"] = "function"
144
+ function: FunctionDefinition
145
+
146
+
147
+class ChatCompletionNamedFunction(OpenAIBaseModel):
148
+ name: str
149
+
150
+
151
+class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
152
+ function: ChatCompletionNamedFunction
153
+ type: Literal["function"] = "function"
154
+
155
+
156
+class ChatCompletionRequest(OpenAIBaseModel):
157
+ # Ordered by official OpenAI API documentation
158
+ # https://platform.openai.com/docs/api-reference/chat/create
159
+ messages: list[ChatCompletionMessageParam]
160
+ model: str | None = None
161
+ frequency_penalty: float | None = 0.0
162
+ logit_bias: dict[str, float] | None = None
163
+ logprobs: bool | None = False
164
+ top_logprobs: int | None = 0
165
+ max_tokens: int | None = Field(
166
+ default=None,
167
+ deprecated="max_tokens is deprecated in favor of "
168
+ "the max_completion_tokens field",
169
+ )
170
+ max_completion_tokens: int | None = None
171
+ n: int | None = 1
172
+ presence_penalty: float | None = 0.0
173
+ response_format: AnyResponseFormat | None = None
174
+ seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
175
+ stop: str | list[str] | None = []
176
+ stream: bool | None = False
177
+ stream_options: StreamOptions | None = None
178
+ temperature: float | None = None
179
+ top_p: float | None = None
180
+ tools: list[ChatCompletionToolsParam] | None = None
181
+ tool_choice: (
182
+ Literal["none"]
183
+ | Literal["auto"]
184
+ | Literal["required"]
185
+ | ChatCompletionNamedToolChoiceParam
186
+ | None
187
+ ) = "none"
188
+ reasoning_effort: Literal["low", "medium", "high"] | None = None
189
+ include_reasoning: bool = True
190
+ parallel_tool_calls: bool | None = True
191
+
192
+ # NOTE this will be ignored by vLLM
193
+ user: str | None = None
194
+
195
+ # --8<-- [start:chat-completion-sampling-params]
196
+ use_beam_search: bool = False
197
+ top_k: int | None = None
198
+ min_p: float | None = None
199
+ repetition_penalty: float | None = None
200
+ length_penalty: float = 1.0
201
+ stop_token_ids: list[int] | None = []
202
+ include_stop_str_in_output: bool = False
203
+ ignore_eos: bool = False
204
+ min_tokens: int = 0
205
+ skip_special_tokens: bool = True
206
+ spaces_between_special_tokens: bool = True
207
+ truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
208
+ None
209
+ )
210
+ prompt_logprobs: int | None = None
211
+ allowed_token_ids: list[int] | None = None
212
+ bad_words: list[str] = Field(default_factory=list)
213
+ # --8<-- [end:chat-completion-sampling-params]
214
+
215
+ # --8<-- [start:chat-completion-extra-params]
216
+ echo: bool = Field(
217
+ default=False,
218
+ description=(
219
+ "If true, the new message will be prepended with the last message "
220
+ "if they belong to the same role."
221
+ ),
222
+ )
223
+ add_generation_prompt: bool = Field(
224
+ default=True,
225
+ description=(
226
+ "If true, the generation prompt will be added to the chat template. "
227
+ "This is a parameter used by chat template in tokenizer config of the "
228
+ "model."
229
+ ),
230
+ )
231
+ continue_final_message: bool = Field(
232
+ default=False,
233
+ description=(
234
+ "If this is set, the chat will be formatted so that the final "
235
+ "message in the chat is open-ended, without any EOS tokens. The "
236
+ "model will continue this message rather than starting a new one. "
237
+ 'This allows you to "prefill" part of the model\'s response for it. '
238
+ "Cannot be used at the same time as `add_generation_prompt`."
239
+ ),
240
+ )
241
+ add_special_tokens: bool = Field(
242
+ default=False,
243
+ description=(
244
+ "If true, special tokens (e.g. BOS) will be added to the prompt "
245
+ "on top of what is added by the chat template. "
246
+ "For most models, the chat template takes care of adding the "
247
+ "special tokens so this should be set to false (as is the "
248
+ "default)."
249
+ ),
250
+ )
251
+ documents: list[dict[str, str]] | None = Field(
252
+ default=None,
253
+ description=(
254
+ "A list of dicts representing documents that will be accessible to "
255
+ "the model if it is performing RAG (retrieval-augmented generation)."
256
+ " If the template does not support RAG, this argument will have no "
257
+ "effect. We recommend that each document should be a dict containing "
258
+ '"title" and "text" keys.'
259
+ ),
260
+ )
261
+ chat_template: str | None = Field(
262
+ default=None,
263
+ description=(
264
+ "A Jinja template to use for this conversion. "
265
+ "As of transformers v4.44, default chat template is no longer "
266
+ "allowed, so you must provide a chat template if the tokenizer "
267
+ "does not define one."
268
+ ),
269
+ )
270
+ chat_template_kwargs: dict[str, Any] | None = Field(
271
+ default=None,
272
+ description=(
273
+ "Additional keyword args to pass to the template renderer. "
274
+ "Will be accessible by the chat template."
275
+ ),
276
+ )
277
+ mm_processor_kwargs: dict[str, Any] | None = Field(
278
+ default=None,
279
+ description=("Additional kwargs to pass to the HF processor."),
280
+ )
281
+ structured_outputs: StructuredOutputsParams | None = Field(
282
+ default=None,
283
+ description="Additional kwargs for structured outputs",
284
+ )
285
+ priority: int = Field(
286
+ default=0,
287
+ description=(
288
+ "The priority of the request (lower means earlier handling; "
289
+ "default: 0). Any priority other than 0 will raise an error "
290
+ "if the served model does not use priority scheduling."
291
+ ),
292
+ )
293
+ request_id: str = Field(
294
+ default_factory=random_uuid,
295
+ description=(
296
+ "The request_id related to this request. If the caller does "
297
+ "not set it, a random_uuid will be generated. This id is used "
298
+ "through out the inference process and return in response."
299
+ ),
300
+ )
301
+ logits_processors: LogitsProcessors | None = Field(
302
+ default=None,
303
+ description=(
304
+ "A list of either qualified names of logits processors, or "
305
+ "constructor objects, to apply when sampling. A constructor is "
306
+ "a JSON object with a required 'qualname' field specifying the "
307
+ "qualified name of the processor class/factory, and optional "
308
+ "'args' and 'kwargs' fields containing positional and keyword "
309
+ "arguments. For example: {'qualname': "
310
+ "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
311
+ "{'param': 'value'}}."
312
+ ),
313
+ )
314
+ return_tokens_as_token_ids: bool | None = Field(
315
+ default=None,
316
+ description=(
317
+ "If specified with 'logprobs', tokens are represented "
318
+ " as strings of the form 'token_id:{token_id}' so that tokens "
319
+ "that are not JSON-encodable can be identified."
320
+ ),
321
+ )
322
+ return_token_ids: bool | None = Field(
323
+ default=None,
324
+ description=(
325
+ "If specified, the result will include token IDs alongside the "
326
+ "generated text. In streaming mode, prompt_token_ids is included "
327
+ "only in the first chunk, and token_ids contains the delta tokens "
328
+ "for each chunk. This is useful for debugging or when you "
329
+ "need to map generated text back to input tokens."
330
+ ),
331
+ )
332
+ cache_salt: str | None = Field(
333
+ default=None,
334
+ description=(
335
+ "If specified, the prefix cache will be salted with the provided "
336
+ "string to prevent an attacker to guess prompts in multi-user "
337
+ "environments. The salt should be random, protected from "
338
+ "access by 3rd parties, and long enough to be "
339
+ "unpredictable (e.g., 43 characters base64-encoded, corresponding "
340
+ "to 256 bit)."
341
+ ),
342
+ )
343
+ kv_transfer_params: dict[str, Any] | None = Field(
344
+ default=None,
345
+ description="KVTransfer parameters used for disaggregated serving.",
346
+ )
347
+
348
+ vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
349
+ default=None,
350
+ description=(
351
+ "Additional request parameters with (list of) string or "
352
+ "numeric values, used by custom extensions."
353
+ ),
354
+ )
355
+
356
+ # --8<-- [end:chat-completion-extra-params]
357
+
358
+ # Default sampling parameters for chat completion requests
359
+ _DEFAULT_SAMPLING_PARAMS: dict = {
360
+ "repetition_penalty": 1.0,
361
+ "temperature": 1.0,
362
+ "top_p": 1.0,
363
+ "top_k": 0,
364
+ "min_p": 0.0,
365
+ }
366
+
367
+ def to_beam_search_params(
368
+ self, max_tokens: int, default_sampling_params: dict
369
+ ) -> BeamSearchParams:
370
+ n = self.n if self.n is not None else 1
371
+ if (temperature := self.temperature) is None:
372
+ temperature = default_sampling_params.get(
373
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
374
+ )
375
+
376
+ return BeamSearchParams(
377
+ beam_width=n,
378
+ max_tokens=max_tokens,
379
+ ignore_eos=self.ignore_eos,
380
+ temperature=temperature,
381
+ length_penalty=self.length_penalty,
382
+ include_stop_str_in_output=self.include_stop_str_in_output,
383
+ )
384
+
385
+ def to_sampling_params(
386
+ self,
387
+ max_tokens: int,
388
+ logits_processor_pattern: str | None,
389
+ default_sampling_params: dict,
390
+ ) -> SamplingParams:
391
+ # Default parameters
392
+ if (repetition_penalty := self.repetition_penalty) is None:
393
+ repetition_penalty = default_sampling_params.get(
394
+ "repetition_penalty",
395
+ self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
396
+ )
397
+ if (temperature := self.temperature) is None:
398
+ temperature = default_sampling_params.get(
399
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
400
+ )
401
+ if (top_p := self.top_p) is None:
402
+ top_p = default_sampling_params.get(
403
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
404
+ )
405
+ if (top_k := self.top_k) is None:
406
+ top_k = default_sampling_params.get(
407
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
408
+ )
409
+ if (min_p := self.min_p) is None:
410
+ min_p = default_sampling_params.get(
411
+ "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
412
+ )
413
+
414
+ prompt_logprobs = self.prompt_logprobs
415
+ if prompt_logprobs is None and self.echo:
416
+ prompt_logprobs = self.top_logprobs
417
+
418
+ response_format = self.response_format
419
+ if response_format is not None:
420
+ # If structured outputs wasn't already enabled,
421
+ # we must enable it for these features to work
422
+ if self.structured_outputs is None:
423
+ self.structured_outputs = StructuredOutputsParams()
424
+
425
+ # Set structured output params for response format
426
+ if response_format.type == "json_object":
427
+ self.structured_outputs.json_object = True
428
+ elif response_format.type == "json_schema":
429
+ json_schema = response_format.json_schema
430
+ assert json_schema is not None
431
+ self.structured_outputs.json = json_schema.json_schema
432
+ elif response_format.type == "structural_tag":
433
+ structural_tag = response_format
434
+ assert structural_tag is not None and isinstance(
435
+ structural_tag,
436
+ (
437
+ LegacyStructuralTagResponseFormat,
438
+ StructuralTagResponseFormat,
439
+ ),
440
+ )
441
+ s_tag_obj = structural_tag.model_dump(by_alias=True)
442
+ self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
443
+
444
+ extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
445
+ if self.kv_transfer_params:
446
+ # Pass in kv_transfer_params via extra_args
447
+ extra_args["kv_transfer_params"] = self.kv_transfer_params
448
+ return SamplingParams.from_optional(
449
+ n=self.n,
450
+ presence_penalty=self.presence_penalty,
451
+ frequency_penalty=self.frequency_penalty,
452
+ repetition_penalty=repetition_penalty,
453
+ temperature=temperature,
454
+ top_p=top_p,
455
+ top_k=top_k,
456
+ min_p=min_p,
457
+ seed=self.seed,
458
+ stop=self.stop,
459
+ stop_token_ids=self.stop_token_ids,
460
+ logprobs=self.top_logprobs if self.logprobs else None,
461
+ prompt_logprobs=prompt_logprobs,
462
+ ignore_eos=self.ignore_eos,
463
+ max_tokens=max_tokens,
464
+ min_tokens=self.min_tokens,
465
+ skip_special_tokens=self.skip_special_tokens,
466
+ spaces_between_special_tokens=self.spaces_between_special_tokens,
467
+ logits_processors=get_logits_processors(
468
+ self.logits_processors, logits_processor_pattern
469
+ ),
470
+ include_stop_str_in_output=self.include_stop_str_in_output,
471
+ truncate_prompt_tokens=self.truncate_prompt_tokens,
472
+ output_kind=RequestOutputKind.DELTA
473
+ if self.stream
474
+ else RequestOutputKind.FINAL_ONLY,
475
+ structured_outputs=self.structured_outputs,
476
+ logit_bias=self.logit_bias,
477
+ bad_words=self.bad_words,
478
+ allowed_token_ids=self.allowed_token_ids,
479
+ extra_args=extra_args or None,
480
+ skip_clone=True, # Created fresh per request, safe to skip clone
481
+ )
482
+
483
+ @model_validator(mode="before")
484
+ @classmethod
485
+ def validate_stream_options(cls, data):
486
+ if data.get("stream_options") and not data.get("stream"):
487
+ raise VLLMValidationError(
488
+ "Stream options can only be defined when `stream=True`.",
489
+ parameter="stream_options",
490
+ )
491
+
492
+ return data
493
+
494
+ @model_validator(mode="before")
495
+ @classmethod
496
+ def check_logprobs(cls, data):
497
+ if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
498
+ if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
499
+ raise VLLMValidationError(
500
+ "`prompt_logprobs` are not available when `stream=True`.",
501
+ parameter="prompt_logprobs",
502
+ )
503
+
504
+ if prompt_logprobs < 0 and prompt_logprobs != -1:
505
+ raise VLLMValidationError(
506
+ "`prompt_logprobs` must be a positive value or -1.",
507
+ parameter="prompt_logprobs",
508
+ value=prompt_logprobs,
509
+ )
510
+ if (top_logprobs := data.get("top_logprobs")) is not None:
511
+ if top_logprobs < 0 and top_logprobs != -1:
512
+ raise VLLMValidationError(
513
+ "`top_logprobs` must be a positive value or -1.",
514
+ parameter="top_logprobs",
515
+ value=top_logprobs,
516
+ )
517
+
518
+ if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
519
+ raise VLLMValidationError(
520
+ "when using `top_logprobs`, `logprobs` must be set to true.",
521
+ parameter="top_logprobs",
522
+ )
523
+
524
+ return data
525
+
526
+ @model_validator(mode="before")
527
+ @classmethod
528
+ def check_structured_outputs_count(cls, data):
529
+ if isinstance(data, ValueError):
530
+ raise data
531
+
532
+ if data.get("structured_outputs", None) is None:
533
+ return data
534
+
535
+ structured_outputs_kwargs = data["structured_outputs"]
536
+ count = sum(
537
+ structured_outputs_kwargs.get(k) is not None
538
+ for k in ("json", "regex", "choice")
539
+ )
540
+ # you can only use one kind of constraints for structured outputs
541
+ if count > 1:
542
+ raise ValueError(
543
+ "You can only use one kind of constraints for structured "
544
+ "outputs ('json', 'regex' or 'choice')."
545
+ )
546
+ # you can only either use structured outputs or tools, not both
547
+ if count > 1 and data.get("tool_choice", "none") not in (
548
+ "none",
549
+ "auto",
550
+ "required",
551
+ ):
552
+ raise ValueError(
553
+ "You can only either use constraints for structured outputs "
554
+ "or tools, not both."
555
+ )
556
+ return data
557
+
558
+ @model_validator(mode="before")
559
+ @classmethod
560
+ def check_tool_usage(cls, data):
561
+ # if "tool_choice" is not specified but tools are provided,
562
+ # default to "auto" tool_choice
563
+ if "tool_choice" not in data and data.get("tools"):
564
+ data["tool_choice"] = "auto"
565
+
566
+ # if "tool_choice" is "none" -- no validation is needed for tools
567
+ if "tool_choice" in data and data["tool_choice"] == "none":
568
+ return data
569
+
570
+ # if "tool_choice" is specified -- validation
571
+ if "tool_choice" in data and data["tool_choice"] is not None:
572
+ # ensure that if "tool choice" is specified, tools are present
573
+ if "tools" not in data or data["tools"] is None:
574
+ raise ValueError("When using `tool_choice`, `tools` must be set.")
575
+
576
+ # make sure that tool choice is either a named tool
577
+ # OR that it's set to "auto" or "required"
578
+ if data["tool_choice"] not in ["auto", "required"] and not isinstance(
579
+ data["tool_choice"], dict
580
+ ):
581
+ raise ValueError(
582
+ f"Invalid value for `tool_choice`: {data['tool_choice']}! "
583
+ 'Only named tools, "none", "auto" or "required" '
584
+ "are supported."
585
+ )
586
+
587
+ # if tool_choice is "required" but the "tools" list is empty,
588
+ # override the data to behave like "none" to align with
589
+ # OpenAI’s behavior.
590
+ if (
591
+ data["tool_choice"] == "required"
592
+ and isinstance(data["tools"], list)
593
+ and len(data["tools"]) == 0
594
+ ):
595
+ data["tool_choice"] = "none"
596
+ del data["tools"]
597
+ return data
598
+
599
+ # ensure that if "tool_choice" is specified as an object,
600
+ # it matches a valid tool
601
+ correct_usage_message = (
602
+ 'Correct usage: `{"type": "function",'
603
+ ' "function": {"name": "my_function"}}`'
604
+ )
605
+ if isinstance(data["tool_choice"], dict):
606
+ valid_tool = False
607
+ function = data["tool_choice"].get("function")
608
+ if not isinstance(function, dict):
609
+ raise ValueError(
610
+ f"Invalid value for `function`: `{function}` in "
611
+ f"`tool_choice`! {correct_usage_message}"
612
+ )
613
+ if "name" not in function:
614
+ raise ValueError(
615
+ f"Expected field `name` in `function` in "
616
+ f"`tool_choice`! {correct_usage_message}"
617
+ )
618
+ function_name = function["name"]
619
+ if not isinstance(function_name, str) or len(function_name) == 0:
620
+ raise ValueError(
621
+ f"Invalid `name` in `function`: `{function_name}`"
622
+ f" in `tool_choice`! {correct_usage_message}"
623
+ )
624
+ for tool in data["tools"]:
625
+ if tool["function"]["name"] == function_name:
626
+ valid_tool = True
627
+ break
628
+ if not valid_tool:
629
+ raise ValueError(
630
+ "The tool specified in `tool_choice` does not match any"
631
+ " of the specified `tools`"
632
+ )
633
+ return data
634
+
635
+ @model_validator(mode="before")
636
+ @classmethod
637
+ def check_generation_prompt(cls, data):
638
+ if data.get("continue_final_message") and data.get("add_generation_prompt"):
639
+ raise ValueError(
640
+ "Cannot set both `continue_final_message` and "
641
+ "`add_generation_prompt` to True."
642
+ )
643
+ return data
644
+
645
+ @model_validator(mode="before")
646
+ @classmethod
647
+ def check_cache_salt_support(cls, data):
648
+ if data.get("cache_salt") is not None and (
649
+ not isinstance(data["cache_salt"], str) or not data["cache_salt"]
650
+ ):
651
+ raise ValueError(
652
+ "Parameter 'cache_salt' must be a non-empty string if provided."
653
+ )
654
+ return data
vllm/entrypoints/openai/chat_completion/serving.py
renamed
+16
-14
@@ -23,16 +23,7 @@ from vllm.entrypoints.chat_utils import (
23
make_tool_call_id,
24
)
25
from vllm.entrypoints.logger import RequestLogger
26
-from vllm.entrypoints.openai.parser.harmony_utils import (
27
- get_developer_message,
28
- get_stop_tokens_for_assistant_actions,
29
- get_streamable_parser_for_assistant,
30
- get_system_message,
31
- parse_chat_inputs_to_harmony_messages,
32
- parse_chat_output,
33
- render_for_completion,
34
-)
35
-from vllm.entrypoints.openai.protocol import (
26
+from vllm.entrypoints.openai.chat_completion.protocol import (
27
ChatCompletionLogProb,
28
ChatCompletionLogProbs,
29
ChatCompletionLogProbsContent,
@@ -43,6 +34,11 @@ from vllm.entrypoints.openai.protocol import (
34
ChatCompletionResponseStreamChoice,
35
ChatCompletionStreamResponse,
36
ChatMessage,
37
+)
38
+from vllm.entrypoints.openai.chat_completion.stream_harmony import (
39
+ extract_harmony_streaming_delta,
40
+)
41
+from vllm.entrypoints.openai.engine.protocol import (
42
DeltaFunctionCall,
43
DeltaMessage,
44
DeltaToolCall,
@@ -52,14 +48,20 @@ from vllm.entrypoints.openai.protocol import (
48
ToolCall,
49
UsageInfo,
50
)
55
-from vllm.entrypoints.openai.serving_chat_stream_harmony import (
56
- extract_harmony_streaming_delta,
57
-)
58
-from vllm.entrypoints.openai.serving_engine import (
51
+from vllm.entrypoints.openai.engine.serving import (
52
GenerationError,
53
OpenAIServing,
54
clamp_prompt_logprobs,
55
)
56
+from vllm.entrypoints.openai.parser.harmony_utils import (
57
+ get_developer_message,
58
+ get_stop_tokens_for_assistant_actions,
59
+ get_streamable_parser_for_assistant,
60
+ get_system_message,
61
+ parse_chat_inputs_to_harmony_messages,
62
+ parse_chat_output,
63
+ render_for_completion,
64
+)
65
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
66
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
67
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
vllm/entrypoints/openai/chat_completion/stream_harmony.py
renamed
+1
-1
@@ -10,7 +10,7 @@ harmony parser state during streaming chat completions.
10
from openai_harmony import StreamableParser
11
12
from vllm.entrypoints.chat_utils import make_tool_call_id
13
-from vllm.entrypoints.openai.protocol import (
13
+from vllm.entrypoints.openai.engine.protocol import (
14
DeltaFunctionCall,
15
DeltaMessage,
16
DeltaToolCall,
vllm/entrypoints/openai/engine/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/openai/engine/protocol.py
renamed
+89
-847
@@ -11,10 +11,6 @@ from typing import Annotated, Any, ClassVar, Literal, TypeAlias
11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
-from openai.types.chat.chat_completion_audio import (
15
- ChatCompletionAudio as OpenAIChatCompletionAudio,
16
-)
17
-from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
14
from openai.types.responses import (
15
ResponseCodeInterpreterCallCodeDeltaEvent,
16
ResponseCodeInterpreterCallCodeDoneEvent,
@@ -234,20 +230,6 @@ class FunctionDefinition(OpenAIBaseModel):
230
parameters: dict[str, Any] | None = None
231
232
237
-class ChatCompletionToolsParam(OpenAIBaseModel):
238
- type: Literal["function"] = "function"
239
- function: FunctionDefinition
240
-
241
-
242
-class ChatCompletionNamedFunction(OpenAIBaseModel):
243
- name: str
244
-
245
-
246
-class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
247
- function: ChatCompletionNamedFunction
248
- type: Literal["function"] = "function"
249
-
250
-
233
# extra="forbid" is a workaround to have kwargs as a field,
234
# see https://github.com/pydantic/pydantic/issues/3125
235
class LogitsProcessorConstructor(BaseModel):
@@ -414,609 +396,66 @@ class ResponsesRequest(OpenAIBaseModel):
396
)
397
if (top_k := self.top_k) is None:
398
top_k = default_sampling_params.get(
417
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
418
- )
419
- stop_token_ids = default_sampling_params.get("stop_token_ids")
420
-
421
- # Structured output
422
- structured_outputs = None
423
- if self.text is not None and self.text.format is not None:
424
- response_format = self.text.format
425
- if (
426
- response_format.type == "json_schema"
427
- and response_format.schema_ is not None
428
- ):
429
- structured_outputs = StructuredOutputsParams(
430
- json=response_format.schema_
431
- )
432
- elif response_format.type == "json_object":
433
- raise NotImplementedError("json_object is not supported")
434
-
435
- # TODO: add more parameters
436
- return SamplingParams.from_optional(
437
- temperature=temperature,
438
- top_p=top_p,
439
- top_k=top_k,
440
- max_tokens=max_tokens,
441
- logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
442
- stop_token_ids=stop_token_ids,
443
- output_kind=(
444
- RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
445
- ),
446
- structured_outputs=structured_outputs,
447
- logit_bias=self.logit_bias,
448
- skip_clone=True, # Created fresh per request, safe to skip clone
449
- )
450
-
451
- def is_include_output_logprobs(self) -> bool:
452
- """Check if the request includes output logprobs."""
453
- if self.include is None:
454
- return False
455
- return (
456
- isinstance(self.include, list)
457
- and "message.output_text.logprobs" in self.include
458
- )
459
-
460
- @model_validator(mode="before")
461
- def validate_background(cls, data):
462
- if not data.get("background"):
463
- return data
464
- if not data.get("store", True):
465
- raise ValueError("background can only be used when `store` is true")
466
- return data
467
-
468
- @model_validator(mode="before")
469
- def validate_prompt(cls, data):
470
- if data.get("prompt") is not None:
471
- raise VLLMValidationError(
472
- "prompt template is not supported", parameter="prompt"
473
- )
474
- return data
475
-
476
- @model_validator(mode="before")
477
- def check_cache_salt_support(cls, data):
478
- if data.get("cache_salt") is not None and (
479
- not isinstance(data["cache_salt"], str) or not data["cache_salt"]
480
- ):
481
- raise ValueError(
482
- "Parameter 'cache_salt' must be a non-empty string if provided."
483
- )
484
- return data
485
-
486
- @model_validator(mode="before")
487
- def function_call_parsing(cls, data):
488
- """Parse function_call dictionaries into ResponseFunctionToolCall objects.
489
- This ensures Pydantic can properly resolve union types in the input field.
490
- Function calls provided as dicts are converted to ResponseFunctionToolCall
491
- objects before validation, while invalid structures are left for Pydantic
492
- to reject with appropriate error messages.
493
- """
494
-
495
- input_data = data.get("input")
496
-
497
- # Early return for None, strings, or bytes
498
- # (strings are iterable but shouldn't be processed)
499
- if input_data is None or isinstance(input_data, (str, bytes)):
500
- return data
501
-
502
- # Convert iterators (like ValidatorIterator) to list
503
- if not isinstance(input_data, list):
504
- try:
505
- input_data = list(input_data)
506
- except TypeError:
507
- # Not iterable, leave as-is for Pydantic to handle
508
- return data
509
-
510
- processed_input = []
511
- for item in input_data:
512
- if isinstance(item, dict) and item.get("type") == "function_call":
513
- try:
514
- processed_input.append(ResponseFunctionToolCall(**item))
515
- except ValidationError:
516
- # Let Pydantic handle validation for malformed function calls
517
- logger.debug(
518
- "Failed to parse function_call to ResponseFunctionToolCall, "
519
- "leaving for Pydantic validation"
520
- )
521
- processed_input.append(item)
522
- else:
523
- processed_input.append(item)
524
-
525
- data["input"] = processed_input
526
- return data
527
-
528
-
529
-class ChatCompletionRequest(OpenAIBaseModel):
530
- # Ordered by official OpenAI API documentation
531
- # https://platform.openai.com/docs/api-reference/chat/create
532
- messages: list[ChatCompletionMessageParam]
533
- model: str | None = None
534
- frequency_penalty: float | None = 0.0
535
- logit_bias: dict[str, float] | None = None
536
- logprobs: bool | None = False
537
- top_logprobs: int | None = 0
538
- max_tokens: int | None = Field(
539
- default=None,
540
- deprecated="max_tokens is deprecated in favor of "
541
- "the max_completion_tokens field",
542
- )
543
- max_completion_tokens: int | None = None
544
- n: int | None = 1
545
- presence_penalty: float | None = 0.0
546
- response_format: AnyResponseFormat | None = None
547
- seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
548
- stop: str | list[str] | None = []
549
- stream: bool | None = False
550
- stream_options: StreamOptions | None = None
551
- temperature: float | None = None
552
- top_p: float | None = None
553
- tools: list[ChatCompletionToolsParam] | None = None
554
- tool_choice: (
555
- Literal["none"]
556
- | Literal["auto"]
557
- | Literal["required"]
558
- | ChatCompletionNamedToolChoiceParam
559
- | None
560
- ) = "none"
561
- reasoning_effort: Literal["low", "medium", "high"] | None = None
562
- include_reasoning: bool = True
563
- parallel_tool_calls: bool | None = True
564
-
565
- # NOTE this will be ignored by vLLM
566
- user: str | None = None
567
-
568
- # --8<-- [start:chat-completion-sampling-params]
569
- use_beam_search: bool = False
570
- top_k: int | None = None
571
- min_p: float | None = None
572
- repetition_penalty: float | None = None
573
- length_penalty: float = 1.0
574
- stop_token_ids: list[int] | None = []
575
- include_stop_str_in_output: bool = False
576
- ignore_eos: bool = False
577
- min_tokens: int = 0
578
- skip_special_tokens: bool = True
579
- spaces_between_special_tokens: bool = True
580
- truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
581
- None
582
- )
583
- prompt_logprobs: int | None = None
584
- allowed_token_ids: list[int] | None = None
585
- bad_words: list[str] = Field(default_factory=list)
586
- # --8<-- [end:chat-completion-sampling-params]
587
-
588
- # --8<-- [start:chat-completion-extra-params]
589
- echo: bool = Field(
590
- default=False,
591
- description=(
592
- "If true, the new message will be prepended with the last message "
593
- "if they belong to the same role."
594
- ),
595
- )
596
- add_generation_prompt: bool = Field(
597
- default=True,
598
- description=(
599
- "If true, the generation prompt will be added to the chat template. "
600
- "This is a parameter used by chat template in tokenizer config of the "
601
- "model."
602
- ),
603
- )
604
- continue_final_message: bool = Field(
605
- default=False,
606
- description=(
607
- "If this is set, the chat will be formatted so that the final "
608
- "message in the chat is open-ended, without any EOS tokens. The "
609
- "model will continue this message rather than starting a new one. "
610
- 'This allows you to "prefill" part of the model\'s response for it. '
611
- "Cannot be used at the same time as `add_generation_prompt`."
612
- ),
613
- )
614
- add_special_tokens: bool = Field(
615
- default=False,
616
- description=(
617
- "If true, special tokens (e.g. BOS) will be added to the prompt "
618
- "on top of what is added by the chat template. "
619
- "For most models, the chat template takes care of adding the "
620
- "special tokens so this should be set to false (as is the "
621
- "default)."
622
- ),
623
- )
624
- documents: list[dict[str, str]] | None = Field(
625
- default=None,
626
- description=(
627
- "A list of dicts representing documents that will be accessible to "
628
- "the model if it is performing RAG (retrieval-augmented generation)."
629
- " If the template does not support RAG, this argument will have no "
630
- "effect. We recommend that each document should be a dict containing "
631
- '"title" and "text" keys.'
632
- ),
633
- )
634
- chat_template: str | None = Field(
635
- default=None,
636
- description=(
637
- "A Jinja template to use for this conversion. "
638
- "As of transformers v4.44, default chat template is no longer "
639
- "allowed, so you must provide a chat template if the tokenizer "
640
- "does not define one."
641
- ),
642
- )
643
- chat_template_kwargs: dict[str, Any] | None = Field(
644
- default=None,
645
- description=(
646
- "Additional keyword args to pass to the template renderer. "
647
- "Will be accessible by the chat template."
648
- ),
649
- )
650
- mm_processor_kwargs: dict[str, Any] | None = Field(
651
- default=None,
652
- description=("Additional kwargs to pass to the HF processor."),
653
- )
654
- structured_outputs: StructuredOutputsParams | None = Field(
655
- default=None,
656
- description="Additional kwargs for structured outputs",
657
- )
658
- priority: int = Field(
659
- default=0,
660
- description=(
661
- "The priority of the request (lower means earlier handling; "
662
- "default: 0). Any priority other than 0 will raise an error "
663
- "if the served model does not use priority scheduling."
664
- ),
665
- )
666
- request_id: str = Field(
667
- default_factory=random_uuid,
668
- description=(
669
- "The request_id related to this request. If the caller does "
670
- "not set it, a random_uuid will be generated. This id is used "
671
- "through out the inference process and return in response."
672
- ),
673
- )
674
- logits_processors: LogitsProcessors | None = Field(
675
- default=None,
676
- description=(
677
- "A list of either qualified names of logits processors, or "
678
- "constructor objects, to apply when sampling. A constructor is "
679
- "a JSON object with a required 'qualname' field specifying the "
680
- "qualified name of the processor class/factory, and optional "
681
- "'args' and 'kwargs' fields containing positional and keyword "
682
- "arguments. For example: {'qualname': "
683
- "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
684
- "{'param': 'value'}}."
685
- ),
686
- )
687
- return_tokens_as_token_ids: bool | None = Field(
688
- default=None,
689
- description=(
690
- "If specified with 'logprobs', tokens are represented "
691
- " as strings of the form 'token_id:{token_id}' so that tokens "
692
- "that are not JSON-encodable can be identified."
693
- ),
694
- )
695
- return_token_ids: bool | None = Field(
696
- default=None,
697
- description=(
698
- "If specified, the result will include token IDs alongside the "
699
- "generated text. In streaming mode, prompt_token_ids is included "
700
- "only in the first chunk, and token_ids contains the delta tokens "
701
- "for each chunk. This is useful for debugging or when you "
702
- "need to map generated text back to input tokens."
703
- ),
704
- )
705
- cache_salt: str | None = Field(
706
- default=None,
707
- description=(
708
- "If specified, the prefix cache will be salted with the provided "
709
- "string to prevent an attacker to guess prompts in multi-user "
710
- "environments. The salt should be random, protected from "
711
- "access by 3rd parties, and long enough to be "
712
- "unpredictable (e.g., 43 characters base64-encoded, corresponding "
713
- "to 256 bit)."
714
- ),
715
- )
716
- kv_transfer_params: dict[str, Any] | None = Field(
717
- default=None,
718
- description="KVTransfer parameters used for disaggregated serving.",
719
- )
720
-
721
- vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
722
- default=None,
723
- description=(
724
- "Additional request parameters with (list of) string or "
725
- "numeric values, used by custom extensions."
726
- ),
727
- )
728
-
729
- # --8<-- [end:chat-completion-extra-params]
730
-
731
- # Default sampling parameters for chat completion requests
732
- _DEFAULT_SAMPLING_PARAMS: dict = {
733
- "repetition_penalty": 1.0,
734
- "temperature": 1.0,
735
- "top_p": 1.0,
736
- "top_k": 0,
737
- "min_p": 0.0,
738
- }
739
-
740
- def to_beam_search_params(
741
- self, max_tokens: int, default_sampling_params: dict
742
- ) -> BeamSearchParams:
743
- n = self.n if self.n is not None else 1
744
- if (temperature := self.temperature) is None:
745
- temperature = default_sampling_params.get(
746
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
747
- )
748
-
749
- return BeamSearchParams(
750
- beam_width=n,
751
- max_tokens=max_tokens,
752
- ignore_eos=self.ignore_eos,
753
- temperature=temperature,
754
- length_penalty=self.length_penalty,
755
- include_stop_str_in_output=self.include_stop_str_in_output,
756
- )
757
-
758
- def to_sampling_params(
759
- self,
760
- max_tokens: int,
761
- logits_processor_pattern: str | None,
762
- default_sampling_params: dict,
763
- ) -> SamplingParams:
764
- # Default parameters
765
- if (repetition_penalty := self.repetition_penalty) is None:
766
- repetition_penalty = default_sampling_params.get(
767
- "repetition_penalty",
768
- self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
769
- )
770
- if (temperature := self.temperature) is None:
771
- temperature = default_sampling_params.get(
772
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
773
- )
774
- if (top_p := self.top_p) is None:
775
- top_p = default_sampling_params.get(
776
- "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
777
- )
778
- if (top_k := self.top_k) is None:
779
- top_k = default_sampling_params.get(
780
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
781
- )
782
- if (min_p := self.min_p) is None:
783
- min_p = default_sampling_params.get(
784
- "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
785
- )
786
-
787
- prompt_logprobs = self.prompt_logprobs
788
- if prompt_logprobs is None and self.echo:
789
- prompt_logprobs = self.top_logprobs
790
-
791
- response_format = self.response_format
792
- if response_format is not None:
793
- # If structured outputs wasn't already enabled,
794
- # we must enable it for these features to work
795
- if self.structured_outputs is None:
796
- self.structured_outputs = StructuredOutputsParams()
797
-
798
- # Set structured output params for response format
799
- if response_format.type == "json_object":
800
- self.structured_outputs.json_object = True
801
- elif response_format.type == "json_schema":
802
- json_schema = response_format.json_schema
803
- assert json_schema is not None
804
- self.structured_outputs.json = json_schema.json_schema
805
- elif response_format.type == "structural_tag":
806
- structural_tag = response_format
807
- assert structural_tag is not None and isinstance(
808
- structural_tag,
809
- (
810
- LegacyStructuralTagResponseFormat,
811
- StructuralTagResponseFormat,
812
- ),
813
- )
814
- s_tag_obj = structural_tag.model_dump(by_alias=True)
815
- self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
816
-
817
- extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
818
- if self.kv_transfer_params:
819
- # Pass in kv_transfer_params via extra_args
820
- extra_args["kv_transfer_params"] = self.kv_transfer_params
821
- return SamplingParams.from_optional(
822
- n=self.n,
823
- presence_penalty=self.presence_penalty,
824
- frequency_penalty=self.frequency_penalty,
825
- repetition_penalty=repetition_penalty,
826
- temperature=temperature,
827
- top_p=top_p,
828
- top_k=top_k,
829
- min_p=min_p,
830
- seed=self.seed,
831
- stop=self.stop,
832
- stop_token_ids=self.stop_token_ids,
833
- logprobs=self.top_logprobs if self.logprobs else None,
834
- prompt_logprobs=prompt_logprobs,
835
- ignore_eos=self.ignore_eos,
836
- max_tokens=max_tokens,
837
- min_tokens=self.min_tokens,
838
- skip_special_tokens=self.skip_special_tokens,
839
- spaces_between_special_tokens=self.spaces_between_special_tokens,
840
- logits_processors=get_logits_processors(
841
- self.logits_processors, logits_processor_pattern
842
- ),
843
- include_stop_str_in_output=self.include_stop_str_in_output,
844
- truncate_prompt_tokens=self.truncate_prompt_tokens,
845
- output_kind=RequestOutputKind.DELTA
846
- if self.stream
847
- else RequestOutputKind.FINAL_ONLY,
848
- structured_outputs=self.structured_outputs,
849
- logit_bias=self.logit_bias,
850
- bad_words=self.bad_words,
851
- allowed_token_ids=self.allowed_token_ids,
852
- extra_args=extra_args or None,
853
- skip_clone=True, # Created fresh per request, safe to skip clone
854
- )
855
-
856
- @model_validator(mode="before")
857
- @classmethod
858
- def validate_stream_options(cls, data):
859
- if data.get("stream_options") and not data.get("stream"):
860
- raise VLLMValidationError(
861
- "Stream options can only be defined when `stream=True`.",
862
- parameter="stream_options",
863
- )
864
-
865
- return data
866
-
867
- @model_validator(mode="before")
868
- @classmethod
869
- def check_logprobs(cls, data):
870
- if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
871
- if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
872
- raise VLLMValidationError(
873
- "`prompt_logprobs` are not available when `stream=True`.",
874
- parameter="prompt_logprobs",
875
- )
876
-
877
- if prompt_logprobs < 0 and prompt_logprobs != -1:
878
- raise VLLMValidationError(
879
- "`prompt_logprobs` must be a positive value or -1.",
880
- parameter="prompt_logprobs",
881
- value=prompt_logprobs,
882
- )
883
- if (top_logprobs := data.get("top_logprobs")) is not None:
884
- if top_logprobs < 0 and top_logprobs != -1:
885
- raise VLLMValidationError(
886
- "`top_logprobs` must be a positive value or -1.",
887
- parameter="top_logprobs",
888
- value=top_logprobs,
889
- )
890
-
891
- if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
892
- raise VLLMValidationError(
893
- "when using `top_logprobs`, `logprobs` must be set to true.",
894
- parameter="top_logprobs",
895
- )
896
-
897
- return data
898
-
899
- @model_validator(mode="before")
900
- @classmethod
901
- def check_structured_outputs_count(cls, data):
902
- if isinstance(data, ValueError):
903
- raise data
904
-
905
- if data.get("structured_outputs", None) is None:
906
- return data
907
-
908
- structured_outputs_kwargs = data["structured_outputs"]
909
- count = sum(
910
- structured_outputs_kwargs.get(k) is not None
911
- for k in ("json", "regex", "choice")
912
- )
913
- # you can only use one kind of constraints for structured outputs
914
- if count > 1:
915
- raise ValueError(
916
- "You can only use one kind of constraints for structured "
917
- "outputs ('json', 'regex' or 'choice')."
918
- )
919
- # you can only either use structured outputs or tools, not both
920
- if count > 1 and data.get("tool_choice", "none") not in (
921
- "none",
922
- "auto",
923
- "required",
924
- ):
925
- raise ValueError(
926
- "You can only either use constraints for structured outputs "
927
- "or tools, not both."
399
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
400
)
929
- return data
930
-
931
- @model_validator(mode="before")
932
- @classmethod
933
- def check_tool_usage(cls, data):
934
- # if "tool_choice" is not specified but tools are provided,
935
- # default to "auto" tool_choice
936
- if "tool_choice" not in data and data.get("tools"):
937
- data["tool_choice"] = "auto"
938
-
939
- # if "tool_choice" is "none" -- no validation is needed for tools
940
- if "tool_choice" in data and data["tool_choice"] == "none":
941
- return data
942
-
943
- # if "tool_choice" is specified -- validation
944
- if "tool_choice" in data and data["tool_choice"] is not None:
945
- # ensure that if "tool choice" is specified, tools are present
946
- if "tools" not in data or data["tools"] is None:
947
- raise ValueError("When using `tool_choice`, `tools` must be set.")
401
+ stop_token_ids = default_sampling_params.get("stop_token_ids")
402
949
- # make sure that tool choice is either a named tool
950
- # OR that it's set to "auto" or "required"
951
- if data["tool_choice"] not in ["auto", "required"] and not isinstance(
952
- data["tool_choice"], dict
403
+ # Structured output
404
+ structured_outputs = None
405
+ if self.text is not None and self.text.format is not None:
406
+ response_format = self.text.format
407
+ if (
408
+ response_format.type == "json_schema"
409
+ and response_format.schema_ is not None
410
):
954
- raise ValueError(
955
- f"Invalid value for `tool_choice`: {data['tool_choice']}! "
956
- 'Only named tools, "none", "auto" or "required" '
957
- "are supported."
411
+ structured_outputs = StructuredOutputsParams(
412
+ json=response_format.schema_
413
)
414
+ elif response_format.type == "json_object":
415
+ raise NotImplementedError("json_object is not supported")
416
960
- # if tool_choice is "required" but the "tools" list is empty,
961
- # override the data to behave like "none" to align with
962
- # OpenAI’s behavior.
963
- if (
964
- data["tool_choice"] == "required"
965
- and isinstance(data["tools"], list)
966
- and len(data["tools"]) == 0
967
- ):
968
- data["tool_choice"] = "none"
969
- del data["tools"]
970
- return data
417
+ # TODO: add more parameters
418
+ return SamplingParams.from_optional(
419
+ temperature=temperature,
420
+ top_p=top_p,
421
+ top_k=top_k,
422
+ max_tokens=max_tokens,
423
+ logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
424
+ stop_token_ids=stop_token_ids,
425
+ output_kind=(
426
+ RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
427
+ ),
428
+ structured_outputs=structured_outputs,
429
+ logit_bias=self.logit_bias,
430
+ skip_clone=True, # Created fresh per request, safe to skip clone
431
+ )
432
972
- # ensure that if "tool_choice" is specified as an object,
973
- # it matches a valid tool
974
- correct_usage_message = (
975
- 'Correct usage: `{"type": "function",'
976
- ' "function": {"name": "my_function"}}`'
977
- )
978
- if isinstance(data["tool_choice"], dict):
979
- valid_tool = False
980
- function = data["tool_choice"].get("function")
981
- if not isinstance(function, dict):
982
- raise ValueError(
983
- f"Invalid value for `function`: `{function}` in "
984
- f"`tool_choice`! {correct_usage_message}"
985
- )
986
- if "name" not in function:
987
- raise ValueError(
988
- f"Expected field `name` in `function` in "
989
- f"`tool_choice`! {correct_usage_message}"
990
- )
991
- function_name = function["name"]
992
- if not isinstance(function_name, str) or len(function_name) == 0:
993
- raise ValueError(
994
- f"Invalid `name` in `function`: `{function_name}`"
995
- f" in `tool_choice`! {correct_usage_message}"
996
- )
997
- for tool in data["tools"]:
998
- if tool["function"]["name"] == function_name:
999
- valid_tool = True
1000
- break
1001
- if not valid_tool:
1002
- raise ValueError(
1003
- "The tool specified in `tool_choice` does not match any"
1004
- " of the specified `tools`"
1005
- )
433
+ def is_include_output_logprobs(self) -> bool:
434
+ """Check if the request includes output logprobs."""
435
+ if self.include is None:
436
+ return False
437
+ return (
438
+ isinstance(self.include, list)
439
+ and "message.output_text.logprobs" in self.include
440
+ )
441
+
442
+ @model_validator(mode="before")
443
+ def validate_background(cls, data):
444
+ if not data.get("background"):
445
+ return data
446
+ if not data.get("store", True):
447
+ raise ValueError("background can only be used when `store` is true")
448
return data
449
450
@model_validator(mode="before")
1009
- @classmethod
1010
- def check_generation_prompt(cls, data):
1011
- if data.get("continue_final_message") and data.get("add_generation_prompt"):
1012
- raise ValueError(
1013
- "Cannot set both `continue_final_message` and "
1014
- "`add_generation_prompt` to True."
451
+ def validate_prompt(cls, data):
452
+ if data.get("prompt") is not None:
453
+ raise VLLMValidationError(
454
+ "prompt template is not supported", parameter="prompt"
455
)
456
return data
457
458
@model_validator(mode="before")
1019
- @classmethod
459
def check_cache_salt_support(cls, data):
460
if data.get("cache_salt") is not None and (
461
not isinstance(data["cache_salt"], str) or not data["cache_salt"]
@@ -1026,6 +465,48 @@ class ChatCompletionRequest(OpenAIBaseModel):
465
)
466
return data
467
468
+ @model_validator(mode="before")
469
+ def function_call_parsing(cls, data):
470
+ """Parse function_call dictionaries into ResponseFunctionToolCall objects.
471
+ This ensures Pydantic can properly resolve union types in the input field.
472
+ Function calls provided as dicts are converted to ResponseFunctionToolCall
473
+ objects before validation, while invalid structures are left for Pydantic
474
+ to reject with appropriate error messages.
475
+ """
476
+
477
+ input_data = data.get("input")
478
+
479
+ # Early return for None, strings, or bytes
480
+ # (strings are iterable but shouldn't be processed)
481
+ if input_data is None or isinstance(input_data, (str, bytes)):
482
+ return data
483
+
484
+ # Convert iterators (like ValidatorIterator) to list
485
+ if not isinstance(input_data, list):
486
+ try:
487
+ input_data = list(input_data)
488
+ except TypeError:
489
+ # Not iterable, leave as-is for Pydantic to handle
490
+ return data
491
+
492
+ processed_input = []
493
+ for item in input_data:
494
+ if isinstance(item, dict) and item.get("type") == "function_call":
495
+ try:
496
+ processed_input.append(ResponseFunctionToolCall(**item))
497
+ except ValidationError:
498
+ # Let Pydantic handle validation for malformed function calls
499
+ logger.debug(
500
+ "Failed to parse function_call to ResponseFunctionToolCall, "
501
+ "leaving for Pydantic validation"
502
+ )
503
+ processed_input.append(item)
504
+ else:
505
+ processed_input.append(item)
506
+
507
+ data["input"] = processed_input
508
+ return data
509
+
510
511
class CompletionRequest(OpenAIBaseModel):
512
# Ordered by official OpenAI API documentation
@@ -1486,75 +967,6 @@ class ExtractedToolCallInformation(BaseModel):
967
content: str | None = None
968
969
1489
-class ChatMessage(OpenAIBaseModel):
1490
- role: str
1491
- content: str | None = None
1492
- refusal: str | None = None
1493
- annotations: OpenAIAnnotation | None = None
1494
- audio: OpenAIChatCompletionAudio | None = None
1495
- function_call: FunctionCall | None = None
1496
- tool_calls: list[ToolCall] = Field(default_factory=list)
1497
-
1498
- # vLLM-specific fields that are not in OpenAI spec
1499
- reasoning: str | None = None
1500
- reasoning_content: str | None = None
1501
- """Deprecated: use `reasoning` instead."""
1502
-
1503
- @model_validator(mode="after")
1504
- def handle_deprecated_reasoning_content(self):
1505
- """Copy reasoning to reasoning_content for backward compatibility."""
1506
- self.reasoning_content = self.reasoning
1507
- return self
1508
-
1509
-
1510
-class ChatCompletionLogProb(OpenAIBaseModel):
1511
- token: str
1512
- logprob: float = -9999.0
1513
- bytes: list[int] | None = None
1514
-
1515
-
1516
-class ChatCompletionLogProbsContent(ChatCompletionLogProb):
1517
- # Workaround: redefine fields name cache so that it's not
1518
- # shared with the super class.
1519
- field_names: ClassVar[set[str] | None] = None
1520
- top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
1521
-
1522
-
1523
-class ChatCompletionLogProbs(OpenAIBaseModel):
1524
- content: list[ChatCompletionLogProbsContent] | None = None
1525
-
1526
-
1527
-class ChatCompletionResponseChoice(OpenAIBaseModel):
1528
- index: int
1529
- message: ChatMessage
1530
- logprobs: ChatCompletionLogProbs | None = None
1531
- # per OpenAI spec this is the default
1532
- finish_reason: str | None = "stop"
1533
- # not part of the OpenAI spec but included in vLLM for legacy reasons
1534
- stop_reason: int | str | None = None
1535
- # not part of the OpenAI spec but is useful for tracing the tokens
1536
- # in agent scenarios
1537
- token_ids: list[int] | None = None
1538
-
1539
-
1540
-class ChatCompletionResponse(OpenAIBaseModel):
1541
- id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1542
- object: Literal["chat.completion"] = "chat.completion"
1543
- created: int = Field(default_factory=lambda: int(time.time()))
1544
- model: str
1545
- choices: list[ChatCompletionResponseChoice]
1546
- service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
1547
- system_fingerprint: str | None = None
1548
- usage: UsageInfo
1549
-
1550
- # vLLM-specific fields that are not in OpenAI spec
1551
- prompt_logprobs: list[dict[int, Logprob] | None] | None = None
1552
- prompt_token_ids: list[int] | None = None
1553
- kv_transfer_params: dict[str, Any] | None = Field(
1554
- default=None, description="KVTransfer parameters."
1555
- )
1556
-
1557
-
970
class DeltaMessage(OpenAIBaseModel):
971
role: str | None = None
972
content: str | None = None
@@ -1570,27 +982,6 @@ class DeltaMessage(OpenAIBaseModel):
982
return self
983
984
1573
-class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
1574
- index: int
1575
- delta: DeltaMessage
1576
- logprobs: ChatCompletionLogProbs | None = None
1577
- finish_reason: str | None = None
1578
- stop_reason: int | str | None = None
1579
- # not part of the OpenAI spec but for tracing the tokens
1580
- token_ids: list[int] | None = None
1581
-
1582
-
1583
-class ChatCompletionStreamResponse(OpenAIBaseModel):
1584
- id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1585
- object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
1586
- created: int = Field(default_factory=lambda: int(time.time()))
1587
- model: str
1588
- choices: list[ChatCompletionResponseStreamChoice]
1589
- usage: UsageInfo | None = Field(default=None)
1590
- # not part of the OpenAI spec but for tracing the tokens
1591
- prompt_token_ids: list[int] | None = None
1592
-
1593
-
985
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
986
delta: DeltaMessage
987
finish_reason: str | None = None
@@ -1856,128 +1247,6 @@ StreamingResponsesResponse: TypeAlias = (
1247
)
1248
1249
1859
-class TokenizeCompletionRequest(OpenAIBaseModel):
1860
- model: str | None = None
1861
- prompt: str
1862
-
1863
- add_special_tokens: bool = Field(
1864
- default=True,
1865
- description=(
1866
- "If true (the default), special tokens (e.g. BOS) will be added to "
1867
- "the prompt."
1868
- ),
1869
- )
1870
- return_token_strs: bool | None = Field(
1871
- default=False,
1872
- description=(
1873
- "If true, also return the token strings corresponding to the token ids."
1874
- ),
1875
- )
1876
-
1877
-
1878
-class TokenizeChatRequest(OpenAIBaseModel):
1879
- model: str | None = None
1880
- messages: list[ChatCompletionMessageParam]
1881
-
1882
- add_generation_prompt: bool = Field(
1883
- default=True,
1884
- description=(
1885
- "If true, the generation prompt will be added to the chat template. "
1886
- "This is a parameter used by chat template in tokenizer config of the "
1887
- "model."
1888
- ),
1889
- )
1890
- return_token_strs: bool | None = Field(
1891
- default=False,
1892
- description=(
1893
- "If true, also return the token strings corresponding to the token ids."
1894
- ),
1895
- )
1896
- continue_final_message: bool = Field(
1897
- default=False,
1898
- description=(
1899
- "If this is set, the chat will be formatted so that the final "
1900
- "message in the chat is open-ended, without any EOS tokens. The "
1901
- "model will continue this message rather than starting a new one. "
1902
- 'This allows you to "prefill" part of the model\'s response for it. '
1903
- "Cannot be used at the same time as `add_generation_prompt`."
1904
- ),
1905
- )
1906
- add_special_tokens: bool = Field(
1907
- default=False,
1908
- description=(
1909
- "If true, special tokens (e.g. BOS) will be added to the prompt "
1910
- "on top of what is added by the chat template. "
1911
- "For most models, the chat template takes care of adding the "
1912
- "special tokens so this should be set to false (as is the "
1913
- "default)."
1914
- ),
1915
- )
1916
- chat_template: str | None = Field(
1917
- default=None,
1918
- description=(
1919
- "A Jinja template to use for this conversion. "
1920
- "As of transformers v4.44, default chat template is no longer "
1921
- "allowed, so you must provide a chat template if the tokenizer "
1922
- "does not define one."
1923
- ),
1924
- )
1925
- chat_template_kwargs: dict[str, Any] | None = Field(
1926
- default=None,
1927
- description=(
1928
- "Additional keyword args to pass to the template renderer. "
1929
- "Will be accessible by the chat template."
1930
- ),
1931
- )
1932
- mm_processor_kwargs: dict[str, Any] | None = Field(
1933
- default=None,
1934
- description=("Additional kwargs to pass to the HF processor."),
1935
- )
1936
- tools: list[ChatCompletionToolsParam] | None = Field(
1937
- default=None,
1938
- description=("A list of tools the model may call."),
1939
- )
1940
-
1941
- @model_validator(mode="before")
1942
- @classmethod
1943
- def check_generation_prompt(cls, data):
1944
- if data.get("continue_final_message") and data.get("add_generation_prompt"):
1945
- raise ValueError(
1946
- "Cannot set both `continue_final_message` and "
1947
- "`add_generation_prompt` to True."
1948
- )
1949
- return data
1950
-
1951
-
1952
-TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
1953
-
1954
-
1955
-class TokenizeResponse(OpenAIBaseModel):
1956
- count: int
1957
- max_model_len: int
1958
- tokens: list[int]
1959
- token_strs: list[str] | None = None
1960
-
1961
-
1962
-class DetokenizeRequest(OpenAIBaseModel):
1963
- model: str | None = None
1964
- tokens: list[int]
1965
-
1966
-
1967
-class DetokenizeResponse(OpenAIBaseModel):
1968
- prompt: str
1969
-
1970
-
1971
-class TokenizerInfoResponse(OpenAIBaseModel):
1972
- """
1973
- Response containing tokenizer configuration
1974
- equivalent to tokenizer_config.json
1975
- """
1976
-
1977
- model_config = ConfigDict(extra="allow")
1978
- tokenizer_class: str
1979
-
1980
-
1250
class LoadLoRAAdapterRequest(BaseModel):
1251
lora_name: str
1252
lora_path: str
@@ -2537,30 +1806,3 @@ class GenerateRequest(BaseModel):
1806
default=None,
1807
description="KVTransfer parameters used for disaggregated serving.",
1808
)
2540
-
2541
-
2542
-class GenerateResponseChoice(BaseModel):
2543
- index: int
2544
- logprobs: ChatCompletionLogProbs | None = None
2545
- # per OpenAI spec this is the default
2546
- finish_reason: str | None = "stop"
2547
- token_ids: list[int] | None = None
2548
-
2549
-
2550
-class GenerateResponse(BaseModel):
2551
- request_id: str = Field(
2552
- default_factory=random_uuid,
2553
- description=(
2554
- "The request_id related to this request. If the caller does "
2555
- "not set it, a random_uuid will be generated. This id is used "
2556
- "through out the inference process and return in response."
2557
- ),
2558
- )
2559
- choices: list[GenerateResponseChoice]
2560
-
2561
- prompt_logprobs: list[dict[int, Logprob] | None] | None = None
2562
-
2563
- kv_transfer_params: dict[str, Any] | None = Field(
2564
- default=None,
2565
- description="KVTransfer parameters used for disaggregated serving.",
2566
- )
vllm/entrypoints/openai/engine/serving.py
renamed
+9
-5
@@ -38,22 +38,20 @@ from vllm.entrypoints.context import (
38
StreamingHarmonyContext,
39
)
40
from vllm.entrypoints.logger import RequestLogger
41
-from vllm.entrypoints.openai.protocol import (
41
+from vllm.entrypoints.openai.chat_completion.protocol import (
42
ChatCompletionNamedToolChoiceParam,
43
ChatCompletionRequest,
44
ChatCompletionResponse,
45
+)
46
+from vllm.entrypoints.openai.engine.protocol import (
47
CompletionRequest,
48
CompletionResponse,
47
- DetokenizeRequest,
49
ErrorInfo,
50
ErrorResponse,
51
FunctionCall,
52
FunctionDefinition,
53
ResponseInputOutputItem,
54
ResponsesRequest,
54
- TokenizeChatRequest,
55
- TokenizeCompletionRequest,
56
- TokenizeResponse,
55
TranscriptionRequest,
56
TranscriptionResponse,
57
TranslationRequest,
@@ -86,6 +84,12 @@ from vllm.entrypoints.responses_utils import (
84
construct_input_messages,
85
)
86
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
87
+from vllm.entrypoints.serve.tokenize.protocol import (
88
+ DetokenizeRequest,
89
+ TokenizeChatRequest,
90
+ TokenizeCompletionRequest,
91
+ TokenizeResponse,
92
+)
93
from vllm.entrypoints.utils import _validate_truncation_size
94
from vllm.inputs.data import PromptType, TokensPrompt
95
from vllm.inputs.parse import (
vllm/entrypoints/openai/parser/harmony_utils.py
+2
-2
@@ -43,8 +43,8 @@ from openai_harmony import Message as OpenAIHarmonyMessage
43
from openai_harmony import Role as OpenAIHarmonyRole
44
45
from vllm import envs
46
-from vllm.entrypoints.openai.protocol import (
47
- ChatCompletionToolsParam,
46
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam
47
+from vllm.entrypoints.openai.engine.protocol import (
48
ResponseInputOutputItem,
49
ResponsesRequest,
50
)
vllm/entrypoints/openai/parser/responses_parser.py
+4
-1
@@ -16,7 +16,10 @@ from openai.types.responses.response_reasoning_item import (
16
)
17
18
from vllm.entrypoints.constants import MCP_PREFIX
19
-from vllm.entrypoints.openai.protocol import ResponseInputOutputItem, ResponsesRequest
19
+from vllm.entrypoints.openai.engine.protocol import (
20
+ ResponseInputOutputItem,
21
+ ResponsesRequest,
22
+)
23
from vllm.outputs import CompletionOutput
24
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
25
from vllm.tokenizers import TokenizerLike
vllm/entrypoints/openai/run_batch.py
+4
-2
@@ -19,13 +19,15 @@ from tqdm import tqdm
19
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
20
from vllm.engine.protocol import EngineClient
21
from vllm.entrypoints.logger import RequestLogger
22
-from vllm.entrypoints.openai.protocol import (
22
+from vllm.entrypoints.openai.chat_completion.protocol import (
23
ChatCompletionRequest,
24
ChatCompletionResponse,
25
+)
26
+from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
27
+from vllm.entrypoints.openai.engine.protocol import (
28
ErrorResponse,
29
OpenAIBaseModel,
30
)
28
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
31
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
32
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest, EmbeddingResponse
33
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
vllm/entrypoints/openai/serving_completion.py
+2
-2
@@ -12,7 +12,7 @@ from fastapi import Request
12
13
from vllm.engine.protocol import EngineClient
14
from vllm.entrypoints.logger import RequestLogger
15
-from vllm.entrypoints.openai.protocol import (
15
+from vllm.entrypoints.openai.engine.protocol import (
16
CompletionLogProbs,
17
CompletionRequest,
18
CompletionResponse,
@@ -25,7 +25,7 @@ from vllm.entrypoints.openai.protocol import (
25
UsageInfo,
26
VLLMValidationError,
27
)
28
-from vllm.entrypoints.openai.serving_engine import (
28
+from vllm.entrypoints.openai.engine.serving import (
29
GenerationError,
30
OpenAIServing,
31
clamp_prompt_logprobs,
vllm/entrypoints/openai/serving_models.py
+1
-1
@@ -7,7 +7,7 @@ from dataclasses import dataclass
7
from http import HTTPStatus
8
9
from vllm.engine.protocol import EngineClient
10
-from vllm.entrypoints.openai.protocol import (
10
+from vllm.entrypoints.openai.engine.protocol import (
11
ErrorInfo,
12
ErrorResponse,
13
LoadLoRAAdapterRequest,
vllm/entrypoints/openai/serving_responses.py
+14
-14
@@ -72,19 +72,7 @@ from vllm.entrypoints.context import (
72
StreamingHarmonyContext,
73
)
74
from vllm.entrypoints.logger import RequestLogger
75
-from vllm.entrypoints.openai.parser.harmony_utils import (
76
- construct_harmony_previous_input_messages,
77
- get_developer_message,
78
- get_stop_tokens_for_assistant_actions,
79
- get_system_message,
80
- get_user_message,
81
- has_custom_tools,
82
- parse_output_message,
83
- parse_remaining_state,
84
- parse_response_input,
85
- render_for_completion,
86
-)
87
-from vllm.entrypoints.openai.protocol import (
75
+from vllm.entrypoints.openai.engine.protocol import (
76
DeltaMessage,
77
ErrorResponse,
78
InputTokensDetails,
@@ -102,10 +90,22 @@ from vllm.entrypoints.openai.protocol import (
90
StreamingResponsesResponse,
91
VLLMValidationError,
92
)
105
-from vllm.entrypoints.openai.serving_engine import (
93
+from vllm.entrypoints.openai.engine.serving import (
94
GenerationError,
95
OpenAIServing,
96
)
97
+from vllm.entrypoints.openai.parser.harmony_utils import (
98
+ construct_harmony_previous_input_messages,
99
+ get_developer_message,
100
+ get_stop_tokens_for_assistant_actions,
101
+ get_system_message,
102
+ get_user_message,
103
+ has_custom_tools,
104
+ parse_output_message,
105
+ parse_remaining_state,
106
+ parse_response_input,
107
+ render_for_completion,
108
+)
109
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
110
from vllm.entrypoints.responses_utils import (
111
construct_input_messages,
vllm/entrypoints/openai/serving_transcription.py
+1
-1
@@ -6,7 +6,7 @@ from fastapi import Request
6
7
from vllm.engine.protocol import EngineClient
8
from vllm.entrypoints.logger import RequestLogger
9
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.engine.protocol import (
10
ErrorResponse,
11
RequestResponseMetadata,
12
TranscriptionRequest,
vllm/entrypoints/openai/speech_to_text.py
+2
-2
@@ -15,7 +15,7 @@ from transformers import PreTrainedTokenizerBase
15
import vllm.envs as envs
16
from vllm.engine.protocol import EngineClient
17
from vllm.entrypoints.logger import RequestLogger
18
-from vllm.entrypoints.openai.protocol import (
18
+from vllm.entrypoints.openai.engine.protocol import (
19
DeltaMessage,
20
ErrorResponse,
21
RequestResponseMetadata,
@@ -32,7 +32,7 @@ from vllm.entrypoints.openai.protocol import (
32
UsageInfo,
33
VLLMValidationError,
34
)
35
-from vllm.entrypoints.openai.serving_engine import OpenAIServing, SpeechToTextRequest
35
+from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
36
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
37
from vllm.inputs.data import PromptType
38
from vllm.logger import init_logger
vllm/entrypoints/openai/utils.py
+1
-1
@@ -5,7 +5,7 @@ from typing import TypeVar
5
from fastapi import Request
6
from fastapi.exceptions import RequestValidationError
7
8
-from vllm.entrypoints.openai.protocol import (
8
+from vllm.entrypoints.openai.chat_completion.protocol import (
9
ChatCompletionRequest,
10
ChatCompletionResponseChoice,
11
ChatCompletionResponseStreamChoice,
vllm/entrypoints/pooling/classify/api_router.py
+1
-1
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
6
from starlette.responses import JSONResponse
7
from typing_extensions import assert_never
8
9
-from vllm.entrypoints.openai.protocol import ErrorResponse
9
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
from vllm.entrypoints.openai.utils import validate_json_request
11
from vllm.entrypoints.pooling.classify.protocol import (
12
ClassificationRequest,
vllm/entrypoints/pooling/classify/protocol.py
+1
-1
@@ -11,7 +11,7 @@ from pydantic import (
11
from vllm import PoolingParams
12
from vllm.config.pooler import get_use_activation
13
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
14
-from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
14
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
15
from vllm.utils import random_uuid
16
17
vllm/entrypoints/pooling/classify/serving.py
+4
-2
@@ -11,12 +11,14 @@ from fastapi import Request
11
from vllm.engine.protocol import EngineClient
12
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
13
from vllm.entrypoints.logger import RequestLogger
14
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
ChatCompletionRequest,
16
+)
17
+from vllm.entrypoints.openai.engine.protocol import (
18
ErrorResponse,
19
UsageInfo,
20
)
19
-from vllm.entrypoints.openai.serving_engine import (
21
+from vllm.entrypoints.openai.engine.serving import (
22
ClassificationServeContext,
23
OpenAIServing,
24
ServeContext,
vllm/entrypoints/pooling/embed/api_router.py
+1
-1
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
6
from fastapi.responses import JSONResponse, StreamingResponse
7
from typing_extensions import assert_never
8
9
-from vllm.entrypoints.openai.protocol import ErrorResponse
9
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
from vllm.entrypoints.openai.utils import validate_json_request
11
from vllm.entrypoints.pooling.embed.protocol import (
12
EmbeddingBytesResponse,
vllm/entrypoints/pooling/embed/protocol.py
+1
-1
@@ -10,7 +10,7 @@ from pydantic import (
10
11
from vllm import PoolingParams
12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
-from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
13
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
14
from vllm.utils import random_uuid
15
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
16
vllm/entrypoints/pooling/embed/serving.py
+2
-2
@@ -12,11 +12,11 @@ from typing_extensions import assert_never, override
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
14
from vllm.entrypoints.logger import RequestLogger
15
-from vllm.entrypoints.openai.protocol import (
15
+from vllm.entrypoints.openai.engine.protocol import (
16
ErrorResponse,
17
UsageInfo,
18
)
19
-from vllm.entrypoints.openai.serving_engine import (
19
+from vllm.entrypoints.openai.engine.serving import (
20
EmbeddingServeContext,
21
OpenAIServing,
22
ServeContext,
vllm/entrypoints/pooling/pooling/api_router.py
+1
-1
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
6
from fastapi.responses import JSONResponse, StreamingResponse
7
from typing_extensions import assert_never
8
9
-from vllm.entrypoints.openai.protocol import ErrorResponse
9
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
from vllm.entrypoints.openai.utils import validate_json_request
11
from vllm.entrypoints.pooling.pooling.protocol import (
12
IOProcessorResponse,
vllm/entrypoints/pooling/pooling/protocol.py
+1
-1
@@ -9,7 +9,7 @@ from pydantic import (
9
10
from vllm import PoolingParams
11
from vllm.config.pooler import get_use_activation
12
-from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
12
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
13
from vllm.entrypoints.pooling.embed.protocol import (
14
EmbeddingChatRequest,
15
EmbeddingCompletionRequest,
vllm/entrypoints/pooling/pooling/serving.py
+2
-2
@@ -14,11 +14,11 @@ from typing_extensions import assert_never
14
from vllm.engine.protocol import EngineClient
15
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
16
from vllm.entrypoints.logger import RequestLogger
17
-from vllm.entrypoints.openai.protocol import (
17
+from vllm.entrypoints.openai.engine.protocol import (
18
ErrorResponse,
19
UsageInfo,
20
)
21
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
21
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
22
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
23
from vllm.entrypoints.pooling.pooling.protocol import (
24
IOProcessorRequest,
vllm/entrypoints/pooling/score/api_router.py
+1
-1
@@ -6,7 +6,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
6
from fastapi.responses import JSONResponse
7
from typing_extensions import assert_never
8
9
-from vllm.entrypoints.openai.protocol import ErrorResponse
9
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
from vllm.entrypoints.openai.utils import validate_json_request
11
from vllm.entrypoints.pooling.score.protocol import (
12
RerankRequest,
vllm/entrypoints/pooling/score/protocol.py
+1
-1
@@ -10,7 +10,7 @@ from pydantic import (
10
11
from vllm import PoolingParams
12
from vllm.config.pooler import get_use_activation
13
-from vllm.entrypoints.openai.protocol import OpenAIBaseModel, UsageInfo
13
+from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
14
from vllm.entrypoints.score_utils import ScoreContentPartParam, ScoreMultiModalParam
15
from vllm.utils import random_uuid
16
vllm/entrypoints/pooling/score/serving.py
+2
-2
@@ -9,11 +9,11 @@ from fastapi import Request
9
10
from vllm.engine.protocol import EngineClient
11
from vllm.entrypoints.logger import RequestLogger
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.engine.protocol import (
13
ErrorResponse,
14
UsageInfo,
15
)
16
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
16
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
17
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
18
from vllm.entrypoints.pooling.score.protocol import (
19
RerankDocument,
vllm/entrypoints/responses_utils.py
+1
-1
@@ -22,7 +22,7 @@ from openai.types.responses.tool import Tool
22
23
from vllm import envs
24
from vllm.entrypoints.constants import MCP_PREFIX
25
-from vllm.entrypoints.openai.protocol import (
25
+from vllm.entrypoints.openai.engine.protocol import (
26
ChatCompletionMessageParam,
27
ResponseInputOutputItem,
28
)
vllm/entrypoints/sagemaker/routes.py
+7
-3
@@ -14,16 +14,20 @@ from vllm.entrypoints.openai.api_server import (
14
base,
15
chat,
16
completion,
17
- create_chat_completion,
17
create_completion,
18
validate_json_request,
19
)
21
-from vllm.entrypoints.openai.protocol import (
20
+from vllm.entrypoints.openai.chat_completion.api_router import (
21
+ create_chat_completion,
22
+)
23
+from vllm.entrypoints.openai.chat_completion.protocol import (
24
ChatCompletionRequest,
25
+)
26
+from vllm.entrypoints.openai.engine.protocol import (
27
CompletionRequest,
28
ErrorResponse,
29
)
26
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
30
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
31
from vllm.entrypoints.pooling.classify.api_router import classify, create_classify
32
from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
33
from vllm.entrypoints.pooling.embed.api_router import create_embedding, embedding
vllm/entrypoints/serve/disagg/api_router.py
+1
-1
@@ -11,7 +11,7 @@ from fastapi.responses import JSONResponse, StreamingResponse
11
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.openai.api_server import validate_json_request
14
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.engine.protocol import (
15
ErrorResponse,
16
)
17
from vllm.entrypoints.serve.disagg.protocol import (
vllm/entrypoints/serve/disagg/protocol.py
+2
-2
@@ -4,8 +4,8 @@ from typing import Any
4
5
from pydantic import BaseModel, Field
6
7
-from vllm.entrypoints.openai.protocol import (
8
- ChatCompletionLogProbs,
7
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
8
+from vllm.entrypoints.openai.engine.protocol import (
9
Logprob,
10
SamplingParams,
11
StreamOptions,
vllm/entrypoints/serve/disagg/serving.py
+4
-2
@@ -11,16 +11,18 @@ from fastapi import Request
11
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.logger import RequestLogger
14
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
ChatCompletionLogProb,
16
ChatCompletionLogProbs,
17
ChatCompletionLogProbsContent,
18
+)
19
+from vllm.entrypoints.openai.engine.protocol import (
20
ErrorResponse,
21
PromptTokenUsageInfo,
22
RequestResponseMetadata,
23
UsageInfo,
24
)
23
-from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
25
+from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
26
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
27
from vllm.entrypoints.serve.disagg.protocol import (
28
GenerateRequest,
vllm/entrypoints/serve/elastic_ep/api_router.py
+1
-1
@@ -10,7 +10,7 @@ from fastapi.responses import JSONResponse
10
11
from vllm.engine.protocol import EngineClient
12
from vllm.entrypoints.openai.api_server import validate_json_request
13
-from vllm.entrypoints.openai.protocol import (
13
+from vllm.entrypoints.openai.engine.protocol import (
14
ErrorResponse,
15
)
16
from vllm.entrypoints.serve.elastic_ep.middleware import (
vllm/entrypoints/serve/lora/api_router.py
+1
-1
@@ -8,7 +8,7 @@ from fastapi.responses import JSONResponse, Response
8
9
from vllm import envs
10
from vllm.entrypoints.openai.api_server import models, validate_json_request
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.engine.protocol import (
12
ErrorResponse,
13
LoadLoRAAdapterRequest,
14
UnloadLoRAAdapterRequest,
vllm/entrypoints/serve/tokenize/api_router.py
+4
-2
@@ -10,10 +10,12 @@ from fastapi.responses import JSONResponse
10
from typing_extensions import assert_never
11
12
from vllm.entrypoints.openai.api_server import validate_json_request
13
-from vllm.entrypoints.openai.protocol import (
13
+from vllm.entrypoints.openai.engine.protocol import (
14
+ ErrorResponse,
15
+)
16
+from vllm.entrypoints.serve.tokenize.protocol import (
17
DetokenizeRequest,
18
DetokenizeResponse,
16
- ErrorResponse,
19
TokenizeRequest,
20
TokenizeResponse,
21
)
vllm/entrypoints/serve/tokenize/protocol.py
new
+139
@@ -0,0 +1,139 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from typing import Any, TypeAlias
6
+
7
+from pydantic import ConfigDict, Field, model_validator
8
+
9
+from vllm.entrypoints.chat_utils import (
10
+ ChatCompletionMessageParam,
11
+)
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
+ ChatCompletionToolsParam,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
+ OpenAIBaseModel,
17
+)
18
+
19
+
20
+class TokenizeCompletionRequest(OpenAIBaseModel):
21
+ model: str | None = None
22
+ prompt: str
23
+
24
+ add_special_tokens: bool = Field(
25
+ default=True,
26
+ description=(
27
+ "If true (the default), special tokens (e.g. BOS) will be added to "
28
+ "the prompt."
29
+ ),
30
+ )
31
+ return_token_strs: bool | None = Field(
32
+ default=False,
33
+ description=(
34
+ "If true, also return the token strings corresponding to the token ids."
35
+ ),
36
+ )
37
+
38
+
39
+class TokenizeChatRequest(OpenAIBaseModel):
40
+ model: str | None = None
41
+ messages: list[ChatCompletionMessageParam]
42
+
43
+ add_generation_prompt: bool = Field(
44
+ default=True,
45
+ description=(
46
+ "If true, the generation prompt will be added to the chat template. "
47
+ "This is a parameter used by chat template in tokenizer config of the "
48
+ "model."
49
+ ),
50
+ )
51
+ return_token_strs: bool | None = Field(
52
+ default=False,
53
+ description=(
54
+ "If true, also return the token strings corresponding to the token ids."
55
+ ),
56
+ )
57
+ continue_final_message: bool = Field(
58
+ default=False,
59
+ description=(
60
+ "If this is set, the chat will be formatted so that the final "
61
+ "message in the chat is open-ended, without any EOS tokens. The "
62
+ "model will continue this message rather than starting a new one. "
63
+ 'This allows you to "prefill" part of the model\'s response for it. '
64
+ "Cannot be used at the same time as `add_generation_prompt`."
65
+ ),
66
+ )
67
+ add_special_tokens: bool = Field(
68
+ default=False,
69
+ description=(
70
+ "If true, special tokens (e.g. BOS) will be added to the prompt "
71
+ "on top of what is added by the chat template. "
72
+ "For most models, the chat template takes care of adding the "
73
+ "special tokens so this should be set to false (as is the "
74
+ "default)."
75
+ ),
76
+ )
77
+ chat_template: str | None = Field(
78
+ default=None,
79
+ description=(
80
+ "A Jinja template to use for this conversion. "
81
+ "As of transformers v4.44, default chat template is no longer "
82
+ "allowed, so you must provide a chat template if the tokenizer "
83
+ "does not define one."
84
+ ),
85
+ )
86
+ chat_template_kwargs: dict[str, Any] | None = Field(
87
+ default=None,
88
+ description=(
89
+ "Additional keyword args to pass to the template renderer. "
90
+ "Will be accessible by the chat template."
91
+ ),
92
+ )
93
+ mm_processor_kwargs: dict[str, Any] | None = Field(
94
+ default=None,
95
+ description=("Additional kwargs to pass to the HF processor."),
96
+ )
97
+ tools: list[ChatCompletionToolsParam] | None = Field(
98
+ default=None,
99
+ description=("A list of tools the model may call."),
100
+ )
101
+
102
+ @model_validator(mode="before")
103
+ @classmethod
104
+ def check_generation_prompt(cls, data):
105
+ if data.get("continue_final_message") and data.get("add_generation_prompt"):
106
+ raise ValueError(
107
+ "Cannot set both `continue_final_message` and "
108
+ "`add_generation_prompt` to True."
109
+ )
110
+ return data
111
+
112
+
113
+TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
114
+
115
+
116
+class TokenizeResponse(OpenAIBaseModel):
117
+ count: int
118
+ max_model_len: int
119
+ tokens: list[int]
120
+ token_strs: list[str] | None = None
121
+
122
+
123
+class DetokenizeRequest(OpenAIBaseModel):
124
+ model: str | None = None
125
+ tokens: list[int]
126
+
127
+
128
+class DetokenizeResponse(OpenAIBaseModel):
129
+ prompt: str
130
+
131
+
132
+class TokenizerInfoResponse(OpenAIBaseModel):
133
+ """
134
+ Response containing tokenizer configuration
135
+ equivalent to tokenizer_config.json
136
+ """
137
+
138
+ model_config = ConfigDict(extra="allow")
139
+ tokenizer_class: str
vllm/entrypoints/serve/tokenize/serving.py
+7
-5
@@ -9,18 +9,20 @@ from fastapi import Request
9
from vllm.engine.protocol import EngineClient
10
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
11
from vllm.entrypoints.logger import RequestLogger
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.engine.protocol import (
13
+ ErrorResponse,
14
+)
15
+from vllm.entrypoints.openai.engine.serving import OpenAIServing
16
+from vllm.entrypoints.openai.serving_models import OpenAIServingModels
17
+from vllm.entrypoints.renderer import RenderConfig
18
+from vllm.entrypoints.serve.tokenize.protocol import (
19
DetokenizeRequest,
20
DetokenizeResponse,
15
- ErrorResponse,
21
TokenizeChatRequest,
22
TokenizeRequest,
23
TokenizeResponse,
24
TokenizerInfoResponse,
25
)
21
-from vllm.entrypoints.openai.serving_engine import OpenAIServing
22
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
23
-from vllm.entrypoints.renderer import RenderConfig
26
from vllm.inputs import TokensPrompt
27
from vllm.logger import init_logger
28
from vllm.tokenizers import TokenizerLike
vllm/entrypoints/utils.py
+4
-2
@@ -22,9 +22,11 @@ from vllm.entrypoints.chat_utils import (
22
resolve_hf_chat_template,
23
resolve_mistral_chat_template,
24
)
25
-from vllm.entrypoints.openai.cli_args import make_arg_parser
26
-from vllm.entrypoints.openai.protocol import (
25
+from vllm.entrypoints.openai.chat_completion.protocol import (
26
ChatCompletionRequest,
27
+)
28
+from vllm.entrypoints.openai.cli_args import make_arg_parser
29
+from vllm.entrypoints.openai.engine.protocol import (
30
CompletionRequest,
31
StreamOptions,
32
)
vllm/reasoning/abs_reasoning_parsers.py
+3
-1
@@ -14,8 +14,10 @@ from vllm.utils.collection_utils import is_list_of
14
from vllm.utils.import_utils import import_from_path
15
16
if TYPE_CHECKING:
17
- from vllm.entrypoints.openai.protocol import (
17
+ from vllm.entrypoints.openai.chat_completion.protocol import (
18
ChatCompletionRequest,
19
+ )
20
+ from vllm.entrypoints.openai.engine.protocol import (
21
DeltaMessage,
22
ResponsesRequest,
23
)
vllm/reasoning/basic_parsers.py
+4
-2
@@ -5,13 +5,15 @@ from abc import abstractmethod
5
from collections.abc import Sequence
6
from typing import TYPE_CHECKING, Any
7
8
-from vllm.entrypoints.openai.protocol import DeltaMessage
8
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
9
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
10
from vllm.tokenizers import TokenizerLike
11
12
if TYPE_CHECKING:
13
- from vllm.entrypoints.openai.protocol import (
13
+ from vllm.entrypoints.openai.chat_completion.protocol import (
14
ChatCompletionRequest,
15
+ )
16
+ from vllm.entrypoints.openai.engine.protocol import (
17
ResponsesRequest,
18
)
19
else:
vllm/reasoning/deepseek_r1_reasoning_parser.py
+1
-1
@@ -3,7 +3,7 @@
3
4
from collections.abc import Sequence
5
6
-from vllm.entrypoints.openai.protocol import DeltaMessage
6
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
7
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
8
9
vllm/reasoning/deepseek_v3_reasoning_parser.py
+4
-1
@@ -5,7 +5,10 @@ from collections.abc import Sequence
5
6
from transformers import PreTrainedTokenizerBase
7
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
8
+from vllm.entrypoints.openai.chat_completion.protocol import (
9
+ ChatCompletionRequest,
10
+)
11
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
12
from vllm.logger import init_logger
13
from vllm.reasoning import ReasoningParser
14
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
vllm/reasoning/ernie45_reasoning_parser.py
+4
-1
@@ -5,7 +5,10 @@ from collections.abc import Sequence
5
6
from transformers import PreTrainedTokenizerBase
7
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
8
+from vllm.entrypoints.openai.chat_completion.protocol import (
9
+ ChatCompletionRequest,
10
+)
11
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
12
from vllm.logger import init_logger
13
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
14
vllm/reasoning/gptoss_reasoning_parser.py
+4
-1
@@ -5,8 +5,11 @@ from collections.abc import Sequence
5
6
from transformers import PreTrainedTokenizerBase
7
8
+from vllm.entrypoints.openai.chat_completion.protocol import (
9
+ ChatCompletionRequest,
10
+)
11
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
12
from vllm.entrypoints.openai.parser.harmony_utils import parse_chat_output
9
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
13
from vllm.entrypoints.tool_server import ToolServer
14
from vllm.logger import init_logger
15
from vllm.reasoning import ReasoningParser
vllm/reasoning/granite_reasoning_parser.py
+4
-1
@@ -6,7 +6,10 @@ from collections.abc import Sequence
6
import regex as re
7
from transformers import PreTrainedTokenizerBase
8
9
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
+ ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
13
from vllm.logger import init_logger
14
from vllm.reasoning import ReasoningParser
15
vllm/reasoning/holo2_reasoning_parser.py
+4
-1
@@ -3,7 +3,10 @@
3
4
from collections.abc import Sequence
5
6
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
6
+from vllm.entrypoints.openai.chat_completion.protocol import (
7
+ ChatCompletionRequest,
8
+)
9
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
10
from vllm.logger import init_logger
11
from vllm.reasoning import (
12
ReasoningParser,
vllm/reasoning/hunyuan_a13b_reasoning_parser.py
+4
-1
@@ -6,7 +6,10 @@ from collections.abc import Sequence
6
import regex as re
7
from transformers import PreTrainedTokenizerBase
8
9
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
+ ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
13
from vllm.logger import init_logger
14
from vllm.reasoning import ReasoningParser
15
vllm/reasoning/identity_reasoning_parser.py
+4
-1
@@ -5,7 +5,10 @@ from collections.abc import Sequence
5
6
from transformers import PreTrainedTokenizerBase
7
8
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
8
+from vllm.entrypoints.openai.chat_completion.protocol import (
9
+ ChatCompletionRequest,
10
+)
11
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
12
from vllm.logger import init_logger
13
from vllm.reasoning import ReasoningParser
14
vllm/reasoning/minimax_m2_reasoning_parser.py
+3
-1
@@ -3,8 +3,10 @@
3
4
from collections.abc import Sequence
5
6
-from vllm.entrypoints.openai.protocol import (
6
+from vllm.entrypoints.openai.chat_completion.protocol import (
7
ChatCompletionRequest,
8
+)
9
+from vllm.entrypoints.openai.engine.protocol import (
10
DeltaMessage,
11
ResponsesRequest,
12
)
vllm/reasoning/mistral_reasoning_parser.py
+3
-1
@@ -3,8 +3,10 @@
3
4
from functools import cached_property
5
6
-from vllm.entrypoints.openai.protocol import (
6
+from vllm.entrypoints.openai.chat_completion.protocol import (
7
ChatCompletionRequest,
8
+)
9
+from vllm.entrypoints.openai.engine.protocol import (
10
ResponsesRequest,
11
)
12
from vllm.logger import init_logger
vllm/reasoning/olmo3_reasoning_parser.py
+3
-2
@@ -10,9 +10,10 @@ import regex as re
10
11
if TYPE_CHECKING:
12
from vllm.tokenizers import TokenizerLike
13
-
14
-from vllm.entrypoints.openai.protocol import (
13
+from vllm.entrypoints.openai.chat_completion.protocol import (
14
ChatCompletionRequest,
15
+)
16
+from vllm.entrypoints.openai.engine.protocol import (
17
DeltaMessage,
18
ResponsesRequest,
19
)
vllm/reasoning/qwen3_reasoning_parser.py
+4
-2
@@ -1,8 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-
5
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, ResponsesRequest
4
+from vllm.entrypoints.openai.chat_completion.protocol import (
5
+ ChatCompletionRequest,
6
+)
7
+from vllm.entrypoints.openai.engine.protocol import ResponsesRequest
8
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
9
10
vllm/reasoning/step3_reasoning_parser.py
+4
-1
@@ -6,7 +6,10 @@ from collections.abc import Sequence
6
import regex as re
7
from transformers import PreTrainedTokenizerBase
8
9
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
+ ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
13
from vllm.logger import init_logger
14
from vllm.reasoning import ReasoningParser
15
vllm/tokenizers/mistral.py
+1
-1
@@ -19,7 +19,7 @@ from mistral_common.tokens.tokenizers.sentencepiece import (
19
from mistral_common.tokens.tokenizers.tekken import Tekkenizer
20
21
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
22
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest
22
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
23
from vllm.logger import init_logger
24
25
from .protocol import TokenizerLike
vllm/tool_parsers/abstract_tool_parser.py
+2
-2
@@ -10,8 +10,8 @@ from openai.types.responses.response_format_text_json_schema_config import (
10
ResponseFormatTextJSONSchemaConfig,
11
)
12
13
-from vllm.entrypoints.openai.protocol import (
14
- ChatCompletionRequest,
13
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaMessage,
16
ExtractedToolCallInformation,
17
ResponsesRequest,
vllm/tool_parsers/deepseekv31_tool_parser.py
+3
-1
@@ -6,8 +6,10 @@ from collections.abc import Sequence
6
import regex as re
7
8
from vllm.entrypoints.chat_utils import make_tool_call_id
9
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import (
13
DeltaFunctionCall,
14
DeltaMessage,
15
DeltaToolCall,
vllm/tool_parsers/deepseekv32_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ from typing import Any
8
9
import regex as re
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/deepseekv3_tool_parser.py
+3
-1
@@ -6,8 +6,10 @@ from collections.abc import Sequence
6
import regex as re
7
8
from vllm.entrypoints.chat_utils import make_tool_call_id
9
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import (
13
DeltaFunctionCall,
14
DeltaMessage,
15
DeltaToolCall,
vllm/tool_parsers/ernie45_tool_parser.py
+3
-1
@@ -6,8 +6,10 @@ from collections.abc import Sequence
6
7
import regex as re
8
9
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import (
13
DeltaFunctionCall,
14
DeltaMessage,
15
DeltaToolCall,
vllm/tool_parsers/functiongemma_tool_parser.py
+3
-1
@@ -7,8 +7,10 @@ from collections.abc import Sequence
7
import regex as re
8
9
from vllm.entrypoints.chat_utils import make_tool_call_id
10
-from vllm.entrypoints.openai.protocol import (
10
+from vllm.entrypoints.openai.chat_completion.protocol import (
11
ChatCompletionRequest,
12
+)
13
+from vllm.entrypoints.openai.engine.protocol import (
14
DeltaFunctionCall,
15
DeltaMessage,
16
DeltaToolCall,
vllm/tool_parsers/gigachat3_tool_parser.py
+3
-1
@@ -7,8 +7,10 @@ from collections.abc import Sequence
7
import regex as re
8
9
from vllm.entrypoints.chat_utils import make_tool_call_id
10
-from vllm.entrypoints.openai.protocol import (
10
+from vllm.entrypoints.openai.chat_completion.protocol import (
11
ChatCompletionRequest,
12
+)
13
+from vllm.entrypoints.openai.engine.protocol import (
14
DeltaFunctionCall,
15
DeltaMessage,
16
DeltaToolCall,
vllm/tool_parsers/glm4_moe_tool_parser.py
+3
-1
@@ -8,9 +8,11 @@ from typing import Any
8
9
import regex as re
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
ChatCompletionToolsParam,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaFunctionCall,
17
DeltaMessage,
18
DeltaToolCall,
vllm/tool_parsers/granite_20b_fc_tool_parser.py
+3
-1
@@ -10,8 +10,10 @@ import regex as re
10
from partial_json_parser.core.options import Allow
11
12
from vllm.entrypoints.chat_utils import make_tool_call_id
13
-from vllm.entrypoints.openai.protocol import (
13
+from vllm.entrypoints.openai.chat_completion.protocol import (
14
ChatCompletionRequest,
15
+)
16
+from vllm.entrypoints.openai.engine.protocol import (
17
DeltaFunctionCall,
18
DeltaMessage,
19
DeltaToolCall,
vllm/tool_parsers/granite_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ import partial_json_parser
8
from partial_json_parser.core.options import Allow
9
10
from vllm.entrypoints.chat_utils import make_tool_call_id
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/hermes_tool_parser.py
+3
-1
@@ -9,8 +9,10 @@ import regex as re
9
from partial_json_parser.core.options import Allow
10
11
from vllm.entrypoints.chat_utils import make_tool_call_id
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
ChatCompletionRequest,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaFunctionCall,
17
DeltaMessage,
18
DeltaToolCall,
vllm/tool_parsers/hunyuan_a13b_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ from typing import Any
8
9
import regex as re
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/internlm2_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ import partial_json_parser
8
from partial_json_parser.core.options import Allow
9
10
from vllm.entrypoints.chat_utils import make_tool_call_id
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/jamba_tool_parser.py
+3
-1
@@ -9,8 +9,10 @@ import regex as re
9
from partial_json_parser.core.options import Allow
10
11
from vllm.entrypoints.chat_utils import make_tool_call_id
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
ChatCompletionRequest,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaFunctionCall,
17
DeltaMessage,
18
DeltaToolCall,
vllm/tool_parsers/kimi_k2_tool_parser.py
+3
-1
@@ -6,8 +6,10 @@ from collections.abc import Sequence
6
7
import regex as re
8
9
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.openai.engine.protocol import (
13
DeltaFunctionCall,
14
DeltaMessage,
15
DeltaToolCall,
vllm/tool_parsers/llama4_pythonic_tool_parser.py
+3
-1
@@ -9,8 +9,10 @@ import regex as re
9
from transformers import PreTrainedTokenizerBase
10
11
import vllm.envs as envs
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
ChatCompletionRequest,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaFunctionCall,
17
DeltaMessage,
18
DeltaToolCall,
vllm/tool_parsers/llama_tool_parser.py
+3
-1
@@ -11,8 +11,10 @@ from transformers import PreTrainedTokenizerBase
11
12
import vllm.envs as envs
13
from vllm.entrypoints.chat_utils import make_tool_call_id
14
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
ChatCompletionRequest,
16
+)
17
+from vllm.entrypoints.openai.engine.protocol import (
18
DeltaFunctionCall,
19
DeltaMessage,
20
DeltaToolCall,
vllm/tool_parsers/minimax_m2_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ from typing import Any
8
9
import regex as re
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/minimax_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ from typing import Any
8
import regex as re
9
10
from vllm.entrypoints.chat_utils import make_tool_call_id
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/mistral_tool_parser.py
+3
-1
@@ -12,8 +12,10 @@ import ijson
12
import regex as re
13
from pydantic import Field
14
15
-from vllm.entrypoints.openai.protocol import (
15
+from vllm.entrypoints.openai.chat_completion.protocol import (
16
ChatCompletionRequest,
17
+)
18
+from vllm.entrypoints.openai.engine.protocol import (
19
DeltaFunctionCall,
20
DeltaMessage,
21
DeltaToolCall,
vllm/tool_parsers/olmo3_tool_parser.py
+3
-1
@@ -9,8 +9,10 @@ import regex as re
9
from transformers import PreTrainedTokenizerBase
10
11
import vllm.envs as envs
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
ChatCompletionRequest,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaFunctionCall,
17
DeltaMessage,
18
DeltaToolCall,
vllm/tool_parsers/openai_tool_parser.py
+4
-2
@@ -4,14 +4,16 @@ import json
4
from collections.abc import Sequence
5
from typing import TYPE_CHECKING
6
7
-from vllm.entrypoints.openai.parser.harmony_utils import parse_output_into_messages
8
-from vllm.entrypoints.openai.protocol import (
7
+from vllm.entrypoints.openai.chat_completion.protocol import (
8
ChatCompletionRequest,
9
+)
10
+from vllm.entrypoints.openai.engine.protocol import (
11
DeltaMessage,
12
ExtractedToolCallInformation,
13
FunctionCall,
14
ToolCall,
15
)
16
+from vllm.entrypoints.openai.parser.harmony_utils import parse_output_into_messages
17
from vllm.logger import init_logger
18
from vllm.tool_parsers.abstract_tool_parser import (
19
ToolParser,
vllm/tool_parsers/phi4mini_tool_parser.py
+3
-1
@@ -9,8 +9,10 @@ import regex as re
9
from transformers import PreTrainedTokenizerBase
10
11
from vllm.entrypoints.chat_utils import make_tool_call_id
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
ChatCompletionRequest,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaMessage,
17
ExtractedToolCallInformation,
18
FunctionCall,
vllm/tool_parsers/pythonic_tool_parser.py
+3
-1
@@ -10,8 +10,10 @@ import regex as re
10
from transformers import PreTrainedTokenizerBase
11
12
import vllm.envs as envs
13
-from vllm.entrypoints.openai.protocol import (
13
+from vllm.entrypoints.openai.chat_completion.protocol import (
14
ChatCompletionRequest,
15
+)
16
+from vllm.entrypoints.openai.engine.protocol import (
17
DeltaFunctionCall,
18
DeltaMessage,
19
DeltaToolCall,
vllm/tool_parsers/qwen3coder_tool_parser.py
+3
-1
@@ -8,9 +8,11 @@ from typing import Any
8
9
import regex as re
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
ChatCompletionToolsParam,
14
+)
15
+from vllm.entrypoints.openai.engine.protocol import (
16
DeltaFunctionCall,
17
DeltaMessage,
18
DeltaToolCall,
vllm/tool_parsers/qwen3xml_tool_parser.py
+3
-1
@@ -9,9 +9,11 @@ from xml.parsers.expat import ParserCreate
9
import regex as re
10
11
from vllm.entrypoints.chat_utils import make_tool_call_id
12
-from vllm.entrypoints.openai.protocol import (
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
ChatCompletionRequest,
14
ChatCompletionToolsParam,
15
+)
16
+from vllm.entrypoints.openai.engine.protocol import (
17
DeltaFunctionCall,
18
DeltaMessage,
19
DeltaToolCall,
vllm/tool_parsers/seed_oss_tool_parser.py
+3
-1
@@ -11,9 +11,11 @@ from typing import Any
11
12
import regex as re
13
14
-from vllm.entrypoints.openai.protocol import (
14
+from vllm.entrypoints.openai.chat_completion.protocol import (
15
ChatCompletionRequest,
16
ChatCompletionToolsParam,
17
+)
18
+from vllm.entrypoints.openai.engine.protocol import (
19
DeltaFunctionCall,
20
DeltaMessage,
21
DeltaToolCall,
vllm/tool_parsers/step3_tool_parser.py
+3
-1
@@ -8,8 +8,10 @@ from typing import Any
8
9
import regex as re
10
11
-from vllm.entrypoints.openai.protocol import (
11
+from vllm.entrypoints.openai.chat_completion.protocol import (
12
ChatCompletionRequest,
13
+)
14
+from vllm.entrypoints.openai.engine.protocol import (
15
DeltaFunctionCall,
16
DeltaMessage,
17
DeltaToolCall,
vllm/tool_parsers/utils.py
+1
-1
@@ -13,7 +13,7 @@ from openai.types.responses import (
13
from openai.types.responses.tool import Tool
14
from partial_json_parser.core.options import Allow
15
16
-from vllm.entrypoints.openai.protocol import (
16
+from vllm.entrypoints.openai.chat_completion.protocol import (
17
ChatCompletionNamedToolChoiceParam,
18
ChatCompletionToolsParam,
19
)
vllm/tool_parsers/xlam_tool_parser.py
+4
-3
@@ -6,10 +6,11 @@ from collections.abc import Sequence
6
from typing import Any, Optional, Union
7
8
import regex as re
9
-
10
-from vllm.entrypoints.chat_utils import make_tool_call_id
11
-from vllm.entrypoints.openai.protocol import (
9
+from vllm.entrypoints.openai.chat_completion.protocol import (
10
ChatCompletionRequest,
11
+)
12
+from vllm.entrypoints.chat_utils import make_tool_call_id
13
+from vllm.entrypoints.openai.engine.protocol import (
14
DeltaFunctionCall,
15
DeltaMessage,
16
DeltaToolCall,