A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 04:00 UTC
[Refactor] [10/N] to simplify the vLLM openai completion serving architecture (#32369)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jan 15, 2026 at 15:41 UTC
4c1c501a7ee1d5efbad945ea62a702ce5cefb799
43 files changed
+777
-642
tests/entrypoints/openai/test_chat_error.py
+2
-1
@@ -12,7 +12,8 @@ from vllm.config.multimodal import MultiModalConfig
12
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
13
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
14
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
15
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
from vllm.outputs import CompletionOutput, RequestOutput
18
from vllm.tokenizers import get_tokenizer
19
from vllm.v1.engine.async_llm import AsyncLLM
tests/entrypoints/openai/test_cli_args.py
+1
-1
@@ -6,7 +6,7 @@ import json
6
import pytest
7
8
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
9
-from vllm.entrypoints.openai.serving_models import LoRAModulePath
9
+from vllm.entrypoints.openai.models.protocol import LoRAModulePath
10
from vllm.utils.argparse_utils import FlexibleArgumentParser
11
12
from ...utils import VLLM_PATH
tests/entrypoints/openai/test_completion_error.py
+5
-3
@@ -9,9 +9,11 @@ from unittest.mock import AsyncMock, MagicMock
9
import pytest
10
11
from vllm.config.multimodal import MultiModalConfig
12
-from vllm.entrypoints.openai.engine.protocol import CompletionRequest, ErrorResponse
13
-from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
14
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
12
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
13
+from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
from vllm.outputs import CompletionOutput, RequestOutput
18
from vllm.tokenizers import get_tokenizer
19
from vllm.v1.engine.async_llm import AsyncLLM
tests/entrypoints/openai/test_gptoss_structural_tags_integration.py
+1
-3
@@ -8,13 +8,11 @@ from unittest.mock import Mock
8
9
import pytest
10
11
-from vllm.entrypoints.openai.engine.protocol import (
12
- StructuredOutputsParams,
13
-)
11
from vllm.entrypoints.tool_server import ToolServer
12
from vllm.reasoning.gptoss_reasoning_parser import (
13
GptOssReasoningParser,
14
)
15
+from vllm.sampling_params import StructuredOutputsParams
16
17
18
class TestGptOssStructuralTagsIntegration:
tests/entrypoints/openai/test_lora_resolvers.py
+5
-3
@@ -9,9 +9,11 @@ from unittest.mock import AsyncMock, MagicMock
9
import pytest
10
11
from vllm.config.multimodal import MultiModalConfig
12
-from vllm.entrypoints.openai.engine.protocol import CompletionRequest, ErrorResponse
13
-from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
14
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
12
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
13
+from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
14
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
15
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
16
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
from vllm.lora.request import LoRARequest
18
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
19
from vllm.tokenizers import get_tokenizer
tests/entrypoints/openai/test_serving_chat.py
+1
-1
@@ -20,8 +20,8 @@ from vllm.entrypoints.openai.engine.protocol import (
20
ErrorResponse,
21
RequestResponseMetadata,
22
)
23
+from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels
24
from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
24
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
25
from vllm.outputs import CompletionOutput, RequestOutput
26
from vllm.tokenizers import get_tokenizer
27
from vllm.tool_parsers import ToolParserManager
tests/entrypoints/openai/test_serving_engine.py
+1
-1
@@ -9,7 +9,7 @@ import pytest
9
10
from vllm.config import ModelConfig
11
from vllm.entrypoints.openai.engine.serving import OpenAIServing
12
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
12
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
13
from vllm.tokenizers.mistral import MistralTokenizer
14
15
tests/entrypoints/openai/test_serving_models.py
+2
-1
@@ -11,7 +11,8 @@ from vllm.engine.protocol import EngineClient
11
from vllm.entrypoints.openai.engine.protocol import (
12
ErrorResponse,
13
)
14
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
14
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
15
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
16
from vllm.entrypoints.serve.lora.protocol import (
17
LoadLoRAAdapterRequest,
18
UnloadLoRAAdapterRequest,
tests/v1/engine/test_async_llm.py
+2
-1
@@ -19,7 +19,8 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
19
from vllm.entrypoints.openai.engine.protocol import (
20
ErrorResponse,
21
)
22
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
22
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
23
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
24
from vllm.inputs import PromptType
25
from vllm.outputs import RequestOutput
26
from vllm.platforms import current_platform
vllm/entrypoints/anthropic/__init__.py
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/anthropic/api_router.py
new
+92
@@ -0,0 +1,92 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+
7
+from fastapi import APIRouter, Depends, FastAPI, Request
8
+from fastapi.responses import JSONResponse, StreamingResponse
9
+
10
+from vllm.entrypoints.anthropic.protocol import (
11
+ AnthropicError,
12
+ AnthropicErrorResponse,
13
+ AnthropicMessagesRequest,
14
+ AnthropicMessagesResponse,
15
+)
16
+from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
17
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
18
+from vllm.entrypoints.openai.utils import validate_json_request
19
+from vllm.entrypoints.utils import (
20
+ load_aware_call,
21
+ with_cancellation,
22
+)
23
+from vllm.logger import init_logger
24
+
25
+logger = init_logger(__name__)
26
+
27
+router = APIRouter()
28
+
29
+
30
+def messages(request: Request) -> AnthropicServingMessages:
31
+ return request.app.state.anthropic_serving_messages
32
+
33
+
34
+@router.post(
35
+ "/v1/messages",
36
+ dependencies=[Depends(validate_json_request)],
37
+ responses={
38
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
39
+ HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
40
+ HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
41
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
42
+ },
43
+)
44
+@with_cancellation
45
+@load_aware_call
46
+async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
47
+ def translate_error_response(response: ErrorResponse) -> JSONResponse:
48
+ anthropic_error = AnthropicErrorResponse(
49
+ error=AnthropicError(
50
+ type=response.error.type,
51
+ message=response.error.message,
52
+ )
53
+ )
54
+ return JSONResponse(
55
+ status_code=response.error.code, content=anthropic_error.model_dump()
56
+ )
57
+
58
+ handler = messages(raw_request)
59
+ if handler is None:
60
+ base_server = raw_request.app.state.openai_serving_tokenization
61
+ error = base_server.create_error_response(
62
+ message="The model does not support Messages API"
63
+ )
64
+ return translate_error_response(error)
65
+
66
+ try:
67
+ generator = await handler.create_messages(request, raw_request)
68
+ except Exception as e:
69
+ logger.exception("Error in create_messages: %s", e)
70
+ return JSONResponse(
71
+ status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
72
+ content=AnthropicErrorResponse(
73
+ error=AnthropicError(
74
+ type="internal_error",
75
+ message=str(e),
76
+ )
77
+ ).model_dump(),
78
+ )
79
+
80
+ if isinstance(generator, ErrorResponse):
81
+ return translate_error_response(generator)
82
+
83
+ elif isinstance(generator, AnthropicMessagesResponse):
84
+ resp = generator.model_dump(exclude_none=True)
85
+ logger.debug("Anthropic Messages Response: %s", resp)
86
+ return JSONResponse(content=resp)
87
+
88
+ return StreamingResponse(content=generator, media_type="text/event-stream")
89
+
90
+
91
+def attach_router(app: FastAPI):
92
+ app.include_router(router)
vllm/entrypoints/anthropic/serving.py
renamed
+1
-1
@@ -37,7 +37,7 @@ from vllm.entrypoints.openai.engine.protocol import (
37
ErrorResponse,
38
StreamOptions,
39
)
40
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
40
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
41
42
logger = logging.getLogger(__name__)
43
vllm/entrypoints/openai/api_server.py
+24
-141
@@ -22,10 +22,10 @@ from typing import Any
22
import model_hosting_container_standards.sagemaker as sagemaker_standards
23
import pydantic
24
import uvloop
25
-from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
25
+from fastapi import APIRouter, FastAPI, HTTPException, Request
26
from fastapi.exceptions import RequestValidationError
27
from fastapi.middleware.cors import CORSMiddleware
28
-from fastapi.responses import JSONResponse, StreamingResponse
28
+from fastapi.responses import JSONResponse
29
from starlette.concurrency import iterate_in_threadpool
30
from starlette.datastructures import URL, Headers, MutableHeaders, State
31
from starlette.types import ASGIApp, Message, Receive, Scope, Send
@@ -33,36 +33,26 @@ from starlette.types import ASGIApp, Message, Receive, Scope, Send
33
import vllm.envs as envs
34
from vllm.engine.arg_utils import AsyncEngineArgs
35
from vllm.engine.protocol import EngineClient
36
-from vllm.entrypoints.anthropic.protocol import (
37
- AnthropicError,
38
- AnthropicErrorResponse,
39
- AnthropicMessagesRequest,
40
- AnthropicMessagesResponse,
41
-)
42
-from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
36
+from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
37
from vllm.entrypoints.launcher import serve_http
38
from vllm.entrypoints.logger import RequestLogger
39
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
40
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
41
+from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
42
from vllm.entrypoints.openai.engine.protocol import (
48
- CompletionRequest,
49
- CompletionResponse,
43
ErrorInfo,
44
ErrorResponse,
45
)
46
from vllm.entrypoints.openai.engine.serving import OpenAIServing
54
-from vllm.entrypoints.openai.orca_metrics import metrics_header
55
-from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
56
-from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
57
-from vllm.entrypoints.openai.serving_models import (
58
- BaseModelPath,
47
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
48
+from vllm.entrypoints.openai.models.serving import (
49
OpenAIServingModels,
50
)
51
+from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
52
from vllm.entrypoints.openai.translations.serving import (
53
OpenAIServingTranscription,
54
OpenAIServingTranslation,
55
)
65
-from vllm.entrypoints.openai.utils import validate_json_request
56
from vllm.entrypoints.pooling.classify.serving import ServingClassification
57
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
58
from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling
@@ -75,12 +65,10 @@ from vllm.entrypoints.serve.tokenize.serving import OpenAIServingTokenization
65
from vllm.entrypoints.tool_server import DemoToolServer, MCPToolServer, ToolServer
66
from vllm.entrypoints.utils import (
67
cli_env_setup,
78
- load_aware_call,
68
log_non_default_args,
69
process_chat_template,
70
process_lora_modules,
71
sanitize_message,
83
- with_cancellation,
72
)
73
from vllm.exceptions import VLLMValidationError
74
from vllm.logger import init_logger
@@ -99,7 +87,6 @@ prometheus_multiproc_dir: tempfile.TemporaryDirectory
87
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
88
logger = init_logger("vllm.entrypoints.openai.api_server")
89
102
-ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
90
91
_running_tasks: set[asyncio.Task] = set()
92
@@ -231,22 +218,6 @@ def base(request: Request) -> OpenAIServing:
218
return tokenization(request)
219
220
234
-def models(request: Request) -> OpenAIServingModels:
235
- return request.app.state.openai_serving_models
236
-
237
-
238
-def messages(request: Request) -> AnthropicServingMessages:
239
- return request.app.state.anthropic_serving_messages
240
-
241
-
242
-def chat(request: Request) -> OpenAIServingChat | None:
243
- return request.app.state.openai_serving_chat
244
-
245
-
246
-def completion(request: Request) -> OpenAIServingCompletion | None:
247
- return request.app.state.openai_serving_completion
248
-
249
-
221
def tokenization(request: Request) -> OpenAIServingTokenization:
222
return request.app.state.openai_serving_tokenization
223
@@ -278,116 +249,12 @@ async def get_server_load_metrics(request: Request):
249
return JSONResponse(content={"server_load": request.app.state.server_load_metrics})
250
251
281
-@router.get("/v1/models")
282
-async def show_available_models(raw_request: Request):
283
- handler = models(raw_request)
284
-
285
- models_ = await handler.show_available_models()
286
- return JSONResponse(content=models_.model_dump())
287
-
288
-
252
@router.get("/version")
253
async def show_version():
254
ver = {"version": VLLM_VERSION}
255
return JSONResponse(content=ver)
256
257
295
-@router.post(
296
- "/v1/messages",
297
- dependencies=[Depends(validate_json_request)],
298
- responses={
299
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
300
- HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
301
- HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
302
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
303
- },
304
-)
305
-@with_cancellation
306
-@load_aware_call
307
-async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
308
- def translate_error_response(response: ErrorResponse) -> JSONResponse:
309
- anthropic_error = AnthropicErrorResponse(
310
- error=AnthropicError(
311
- type=response.error.type,
312
- message=response.error.message,
313
- )
314
- )
315
- return JSONResponse(
316
- status_code=response.error.code, content=anthropic_error.model_dump()
317
- )
318
-
319
- handler = messages(raw_request)
320
- if handler is None:
321
- error = base(raw_request).create_error_response(
322
- message="The model does not support Messages API"
323
- )
324
- return translate_error_response(error)
325
-
326
- try:
327
- generator = await handler.create_messages(request, raw_request)
328
- except Exception as e:
329
- logger.exception("Error in create_messages: %s", e)
330
- return JSONResponse(
331
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
332
- content=AnthropicErrorResponse(
333
- error=AnthropicError(
334
- type="internal_error",
335
- message=str(e),
336
- )
337
- ).model_dump(),
338
- )
339
-
340
- if isinstance(generator, ErrorResponse):
341
- return translate_error_response(generator)
342
-
343
- elif isinstance(generator, AnthropicMessagesResponse):
344
- resp = generator.model_dump(exclude_none=True)
345
- logger.debug("Anthropic Messages Response: %s", resp)
346
- return JSONResponse(content=resp)
347
-
348
- return StreamingResponse(content=generator, media_type="text/event-stream")
349
-
350
-
351
-@router.post(
352
- "/v1/completions",
353
- dependencies=[Depends(validate_json_request)],
354
- responses={
355
- HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
356
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
357
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
358
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
359
- },
360
-)
361
-@with_cancellation
362
-@load_aware_call
363
-async def create_completion(request: CompletionRequest, raw_request: Request):
364
- metrics_header_format = raw_request.headers.get(
365
- ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
366
- )
367
- handler = completion(raw_request)
368
- if handler is None:
369
- return base(raw_request).create_error_response(
370
- message="The model does not support Completions API"
371
- )
372
-
373
- try:
374
- generator = await handler.create_completion(request, raw_request)
375
- except Exception as e:
376
- return handler.create_error_response(e)
377
-
378
- if isinstance(generator, ErrorResponse):
379
- return JSONResponse(
380
- content=generator.model_dump(), status_code=generator.error.code
381
- )
382
- elif isinstance(generator, CompletionResponse):
383
- return JSONResponse(
384
- content=generator.model_dump(),
385
- headers=metrics_header(metrics_header_format),
386
- )
387
-
388
- return StreamingResponse(content=generator, media_type="text/event-stream")
389
-
390
-
258
def load_log_config(log_config_file: str | None) -> dict | None:
259
if not log_config_file:
260
return None
@@ -486,7 +353,7 @@ def _extract_content_from_chunk(chunk_data: dict) -> str:
353
from vllm.entrypoints.openai.chat_completion.protocol import (
354
ChatCompletionStreamResponse,
355
)
489
- from vllm.entrypoints.openai.engine.protocol import (
356
+ from vllm.entrypoints.openai.completion.protocol import (
357
CompletionStreamResponse,
358
)
359
@@ -646,6 +513,22 @@ def build_app(args: Namespace) -> FastAPI:
513
)
514
515
register_translations_api_router(app)
516
+
517
+ from vllm.entrypoints.openai.completion.api_router import (
518
+ attach_router as register_completion_api_router,
519
+ )
520
+
521
+ register_completion_api_router(app)
522
+ from vllm.entrypoints.anthropic.api_router import (
523
+ attach_router as register_anthropic_api_router,
524
+ )
525
+
526
+ register_anthropic_api_router(app)
527
+ from vllm.entrypoints.openai.models.api_router import (
528
+ attach_router as register_models_api_router,
529
+ )
530
+
531
+ register_models_api_router(app)
532
from vllm.entrypoints.sagemaker.routes import register_sagemaker_routes
533
534
register_sagemaker_routes(router)
vllm/entrypoints/openai/chat_completion/serving.py
+1
-1
@@ -54,6 +54,7 @@ from vllm.entrypoints.openai.engine.serving import (
54
OpenAIServing,
55
clamp_prompt_logprobs,
56
)
57
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
58
from vllm.entrypoints.openai.parser.harmony_utils import (
59
get_developer_message,
60
get_stop_tokens_for_assistant_actions,
@@ -63,7 +64,6 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
64
parse_chat_output,
65
render_for_completion,
66
)
66
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
67
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
68
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
69
from vllm.inputs.data import TokensPrompt
vllm/entrypoints/openai/cli_args.py
+1
-1
@@ -26,7 +26,7 @@ from vllm.entrypoints.constants import (
26
H11_MAX_HEADER_COUNT_DEFAULT,
27
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT,
28
)
29
-from vllm.entrypoints.openai.serving_models import LoRAModulePath
29
+from vllm.entrypoints.openai.models.protocol import LoRAModulePath
30
from vllm.logger import init_logger
31
from vllm.tool_parsers import ToolParserManager
32
from vllm.utils.argparse_utils import FlexibleArgumentParser
vllm/entrypoints/openai/completion/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
vllm/entrypoints/openai/completion/api_router.py
new
+76
@@ -0,0 +1,76 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from http import HTTPStatus
6
+
7
+from fastapi import APIRouter, Depends, FastAPI, Request
8
+from fastapi.responses import JSONResponse, StreamingResponse
9
+
10
+from vllm.entrypoints.openai.completion.protocol import (
11
+ CompletionRequest,
12
+ CompletionResponse,
13
+)
14
+from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
15
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
16
+from vllm.entrypoints.openai.orca_metrics import metrics_header
17
+from vllm.entrypoints.openai.utils import validate_json_request
18
+from vllm.entrypoints.utils import (
19
+ load_aware_call,
20
+ with_cancellation,
21
+)
22
+from vllm.logger import init_logger
23
+
24
+logger = init_logger(__name__)
25
+
26
+router = APIRouter()
27
+ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL = "endpoint-load-metrics-format"
28
+
29
+
30
+def completion(request: Request) -> OpenAIServingCompletion | None:
31
+ return request.app.state.openai_serving_completion
32
+
33
+
34
+@router.post(
35
+ "/v1/completions",
36
+ dependencies=[Depends(validate_json_request)],
37
+ responses={
38
+ HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
39
+ HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
40
+ HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
41
+ HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
42
+ },
43
+)
44
+@with_cancellation
45
+@load_aware_call
46
+async def create_completion(request: CompletionRequest, raw_request: Request):
47
+ metrics_header_format = raw_request.headers.get(
48
+ ENDPOINT_LOAD_METRICS_FORMAT_HEADER_LABEL, ""
49
+ )
50
+ handler = completion(raw_request)
51
+ if handler is None:
52
+ base_server = raw_request.app.state.openai_serving_tokenization
53
+ return base_server.create_error_response(
54
+ message="The model does not support Completions API"
55
+ )
56
+
57
+ try:
58
+ generator = await handler.create_completion(request, raw_request)
59
+ except Exception as e:
60
+ return handler.create_error_response(e)
61
+
62
+ if isinstance(generator, ErrorResponse):
63
+ return JSONResponse(
64
+ content=generator.model_dump(), status_code=generator.error.code
65
+ )
66
+ elif isinstance(generator, CompletionResponse):
67
+ return JSONResponse(
68
+ content=generator.model_dump(),
69
+ headers=metrics_header(metrics_header_format),
70
+ )
71
+
72
+ return StreamingResponse(content=generator, media_type="text/event-stream")
73
+
74
+
75
+def attach_router(app: FastAPI):
76
+ app.include_router(router)
vllm/entrypoints/openai/completion/protocol.py
new
+463
@@ -0,0 +1,463 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
+import json
7
+import time
8
+from typing import Annotated, Any, Literal
9
+
10
+import torch
11
+from pydantic import (
12
+ Field,
13
+ model_validator,
14
+)
15
+
16
+from vllm.entrypoints.openai.engine.protocol import (
17
+ AnyResponseFormat,
18
+ LegacyStructuralTagResponseFormat,
19
+ LogitsProcessors,
20
+ OpenAIBaseModel,
21
+ StreamOptions,
22
+ StructuralTagResponseFormat,
23
+ UsageInfo,
24
+ get_logits_processors,
25
+)
26
+from vllm.exceptions import VLLMValidationError
27
+from vllm.logger import init_logger
28
+from vllm.logprobs import Logprob
29
+from vllm.sampling_params import (
30
+ BeamSearchParams,
31
+ RequestOutputKind,
32
+ SamplingParams,
33
+ StructuredOutputsParams,
34
+)
35
+from vllm.utils import random_uuid
36
+
37
+logger = init_logger(__name__)
38
+
39
+
40
+_LONG_INFO = torch.iinfo(torch.long)
41
+
42
+
43
+class CompletionRequest(OpenAIBaseModel):
44
+ # Ordered by official OpenAI API documentation
45
+ # https://platform.openai.com/docs/api-reference/completions/create
46
+ model: str | None = None
47
+ prompt: list[int] | list[list[int]] | str | list[str] | None = None
48
+ echo: bool | None = False
49
+ frequency_penalty: float | None = 0.0
50
+ logit_bias: dict[str, float] | None = None
51
+ logprobs: int | None = None
52
+ max_tokens: int | None = 16
53
+ n: int = 1
54
+ presence_penalty: float | None = 0.0
55
+ seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
56
+ stop: str | list[str] | None = []
57
+ stream: bool | None = False
58
+ stream_options: StreamOptions | None = None
59
+ suffix: str | None = None
60
+ temperature: float | None = None
61
+ top_p: float | None = None
62
+ user: str | None = None
63
+
64
+ # --8<-- [start:completion-sampling-params]
65
+ use_beam_search: bool = False
66
+ top_k: int | None = None
67
+ min_p: float | None = None
68
+ repetition_penalty: float | None = None
69
+ length_penalty: float = 1.0
70
+ stop_token_ids: list[int] | None = []
71
+ include_stop_str_in_output: bool = False
72
+ ignore_eos: bool = False
73
+ min_tokens: int = 0
74
+ skip_special_tokens: bool = True
75
+ spaces_between_special_tokens: bool = True
76
+ truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
77
+ None
78
+ )
79
+ allowed_token_ids: list[int] | None = None
80
+ prompt_logprobs: int | None = None
81
+ # --8<-- [end:completion-sampling-params]
82
+
83
+ # --8<-- [start:completion-extra-params]
84
+ prompt_embeds: bytes | list[bytes] | None = None
85
+ add_special_tokens: bool = Field(
86
+ default=True,
87
+ description=(
88
+ "If true (the default), special tokens (e.g. BOS) will be added to "
89
+ "the prompt."
90
+ ),
91
+ )
92
+ response_format: AnyResponseFormat | None = Field(
93
+ default=None,
94
+ description=(
95
+ "Similar to chat completion, this parameter specifies the format "
96
+ "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
97
+ ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
98
+ ),
99
+ )
100
+ structured_outputs: StructuredOutputsParams | None = Field(
101
+ default=None,
102
+ description="Additional kwargs for structured outputs",
103
+ )
104
+ priority: int = Field(
105
+ default=0,
106
+ description=(
107
+ "The priority of the request (lower means earlier handling; "
108
+ "default: 0). Any priority other than 0 will raise an error "
109
+ "if the served model does not use priority scheduling."
110
+ ),
111
+ )
112
+ request_id: str = Field(
113
+ default_factory=random_uuid,
114
+ description=(
115
+ "The request_id related to this request. If the caller does "
116
+ "not set it, a random_uuid will be generated. This id is used "
117
+ "through out the inference process and return in response."
118
+ ),
119
+ )
120
+ logits_processors: LogitsProcessors | None = Field(
121
+ default=None,
122
+ description=(
123
+ "A list of either qualified names of logits processors, or "
124
+ "constructor objects, to apply when sampling. A constructor is "
125
+ "a JSON object with a required 'qualname' field specifying the "
126
+ "qualified name of the processor class/factory, and optional "
127
+ "'args' and 'kwargs' fields containing positional and keyword "
128
+ "arguments. For example: {'qualname': "
129
+ "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
130
+ "{'param': 'value'}}."
131
+ ),
132
+ )
133
+
134
+ return_tokens_as_token_ids: bool | None = Field(
135
+ default=None,
136
+ description=(
137
+ "If specified with 'logprobs', tokens are represented "
138
+ " as strings of the form 'token_id:{token_id}' so that tokens "
139
+ "that are not JSON-encodable can be identified."
140
+ ),
141
+ )
142
+ return_token_ids: bool | None = Field(
143
+ default=None,
144
+ description=(
145
+ "If specified, the result will include token IDs alongside the "
146
+ "generated text. In streaming mode, prompt_token_ids is included "
147
+ "only in the first chunk, and token_ids contains the delta tokens "
148
+ "for each chunk. This is useful for debugging or when you "
149
+ "need to map generated text back to input tokens."
150
+ ),
151
+ )
152
+
153
+ cache_salt: str | None = Field(
154
+ default=None,
155
+ description=(
156
+ "If specified, the prefix cache will be salted with the provided "
157
+ "string to prevent an attacker to guess prompts in multi-user "
158
+ "environments. The salt should be random, protected from "
159
+ "access by 3rd parties, and long enough to be "
160
+ "unpredictable (e.g., 43 characters base64-encoded, corresponding "
161
+ "to 256 bit)."
162
+ ),
163
+ )
164
+
165
+ kv_transfer_params: dict[str, Any] | None = Field(
166
+ default=None,
167
+ description="KVTransfer parameters used for disaggregated serving.",
168
+ )
169
+
170
+ vllm_xargs: dict[str, str | int | float] | None = Field(
171
+ default=None,
172
+ description=(
173
+ "Additional request parameters with string or "
174
+ "numeric values, used by custom extensions."
175
+ ),
176
+ )
177
+
178
+ # --8<-- [end:completion-extra-params]
179
+
180
+ # Default sampling parameters for completion requests
181
+ _DEFAULT_SAMPLING_PARAMS: dict = {
182
+ "repetition_penalty": 1.0,
183
+ "temperature": 1.0,
184
+ "top_p": 1.0,
185
+ "top_k": 0,
186
+ "min_p": 0.0,
187
+ }
188
+
189
+ def to_beam_search_params(
190
+ self,
191
+ max_tokens: int,
192
+ default_sampling_params: dict | None = None,
193
+ ) -> BeamSearchParams:
194
+ if default_sampling_params is None:
195
+ default_sampling_params = {}
196
+ n = self.n if self.n is not None else 1
197
+
198
+ if (temperature := self.temperature) is None:
199
+ temperature = default_sampling_params.get("temperature", 1.0)
200
+
201
+ return BeamSearchParams(
202
+ beam_width=n,
203
+ max_tokens=max_tokens,
204
+ ignore_eos=self.ignore_eos,
205
+ temperature=temperature,
206
+ length_penalty=self.length_penalty,
207
+ include_stop_str_in_output=self.include_stop_str_in_output,
208
+ )
209
+
210
+ def to_sampling_params(
211
+ self,
212
+ max_tokens: int,
213
+ logits_processor_pattern: str | None,
214
+ default_sampling_params: dict | None = None,
215
+ ) -> SamplingParams:
216
+ if default_sampling_params is None:
217
+ default_sampling_params = {}
218
+
219
+ # Default parameters
220
+ if (repetition_penalty := self.repetition_penalty) is None:
221
+ repetition_penalty = default_sampling_params.get(
222
+ "repetition_penalty",
223
+ self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
224
+ )
225
+ if (temperature := self.temperature) is None:
226
+ temperature = default_sampling_params.get(
227
+ "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
228
+ )
229
+ if (top_p := self.top_p) is None:
230
+ top_p = default_sampling_params.get(
231
+ "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
232
+ )
233
+ if (top_k := self.top_k) is None:
234
+ top_k = default_sampling_params.get(
235
+ "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
236
+ )
237
+ if (min_p := self.min_p) is None:
238
+ min_p = default_sampling_params.get(
239
+ "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
240
+ )
241
+
242
+ prompt_logprobs = self.prompt_logprobs
243
+ if prompt_logprobs is None and self.echo:
244
+ prompt_logprobs = self.logprobs
245
+
246
+ echo_without_generation = self.echo and self.max_tokens == 0
247
+
248
+ response_format = self.response_format
249
+ if response_format is not None:
250
+ # If structured outputs wasn't already enabled,
251
+ # we must enable it for these features to work
252
+ if self.structured_outputs is None:
253
+ self.structured_outputs = StructuredOutputsParams()
254
+
255
+ # Set structured output params for response format
256
+ if response_format.type == "json_object":
257
+ self.structured_outputs.json_object = True
258
+ elif response_format.type == "json_schema":
259
+ json_schema = response_format.json_schema
260
+ assert json_schema is not None
261
+ self.structured_outputs.json = json_schema.json_schema
262
+ elif response_format.type == "structural_tag":
263
+ structural_tag = response_format
264
+ assert structural_tag is not None and isinstance(
265
+ structural_tag,
266
+ (
267
+ LegacyStructuralTagResponseFormat,
268
+ StructuralTagResponseFormat,
269
+ ),
270
+ )
271
+ s_tag_obj = structural_tag.model_dump(by_alias=True)
272
+ self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
273
+
274
+ extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
275
+ if self.kv_transfer_params:
276
+ # Pass in kv_transfer_params via extra_args
277
+ extra_args["kv_transfer_params"] = self.kv_transfer_params
278
+ return SamplingParams.from_optional(
279
+ n=self.n,
280
+ presence_penalty=self.presence_penalty,
281
+ frequency_penalty=self.frequency_penalty,
282
+ repetition_penalty=repetition_penalty,
283
+ temperature=temperature,
284
+ top_p=top_p,
285
+ top_k=top_k,
286
+ min_p=min_p,
287
+ seed=self.seed,
288
+ stop=self.stop,
289
+ stop_token_ids=self.stop_token_ids,
290
+ logprobs=self.logprobs,
291
+ ignore_eos=self.ignore_eos,
292
+ max_tokens=max_tokens if not echo_without_generation else 1,
293
+ min_tokens=self.min_tokens,
294
+ prompt_logprobs=prompt_logprobs,
295
+ skip_special_tokens=self.skip_special_tokens,
296
+ spaces_between_special_tokens=self.spaces_between_special_tokens,
297
+ include_stop_str_in_output=self.include_stop_str_in_output,
298
+ logits_processors=get_logits_processors(
299
+ self.logits_processors, logits_processor_pattern
300
+ ),
301
+ truncate_prompt_tokens=self.truncate_prompt_tokens,
302
+ output_kind=RequestOutputKind.DELTA
303
+ if self.stream
304
+ else RequestOutputKind.FINAL_ONLY,
305
+ structured_outputs=self.structured_outputs,
306
+ logit_bias=self.logit_bias,
307
+ allowed_token_ids=self.allowed_token_ids,
308
+ extra_args=extra_args or None,
309
+ skip_clone=True, # Created fresh per request, safe to skip clone
310
+ )
311
+
312
+ @model_validator(mode="before")
313
+ @classmethod
314
+ def check_structured_outputs_count(cls, data):
315
+ if data.get("structured_outputs", None) is None:
316
+ return data
317
+
318
+ structured_outputs_kwargs = data["structured_outputs"]
319
+ count = sum(
320
+ structured_outputs_kwargs.get(k) is not None
321
+ for k in ("json", "regex", "choice")
322
+ )
323
+ if count > 1:
324
+ raise VLLMValidationError(
325
+ "You can only use one kind of constraints for structured "
326
+ "outputs ('json', 'regex' or 'choice').",
327
+ parameter="structured_outputs",
328
+ )
329
+ return data
330
+
331
+ @model_validator(mode="before")
332
+ @classmethod
333
+ def check_logprobs(cls, data):
334
+ if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
335
+ if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
336
+ raise VLLMValidationError(
337
+ "`prompt_logprobs` are not available when `stream=True`.",
338
+ parameter="prompt_logprobs",
339
+ )
340
+
341
+ if prompt_logprobs < 0 and prompt_logprobs != -1:
342
+ raise VLLMValidationError(
343
+ "`prompt_logprobs` must be a positive value or -1.",
344
+ parameter="prompt_logprobs",
345
+ value=prompt_logprobs,
346
+ )
347
+ if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
348
+ raise VLLMValidationError(
349
+ "`logprobs` must be a positive value.",
350
+ parameter="logprobs",
351
+ value=logprobs,
352
+ )
353
+
354
+ return data
355
+
356
+ @model_validator(mode="before")
357
+ @classmethod
358
+ def validate_stream_options(cls, data):
359
+ if data.get("stream_options") and not data.get("stream"):
360
+ raise VLLMValidationError(
361
+ "Stream options can only be defined when `stream=True`.",
362
+ parameter="stream_options",
363
+ )
364
+
365
+ return data
366
+
367
+ @model_validator(mode="before")
368
+ @classmethod
369
+ def validate_prompt_and_prompt_embeds(cls, data):
370
+ prompt = data.get("prompt")
371
+ prompt_embeds = data.get("prompt_embeds")
372
+
373
+ prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
374
+ embeds_is_empty = prompt_embeds is None or (
375
+ isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
376
+ )
377
+
378
+ if prompt_is_empty and embeds_is_empty:
379
+ raise ValueError(
380
+ "Either prompt or prompt_embeds must be provided and non-empty."
381
+ )
382
+
383
+ return data
384
+
385
+ @model_validator(mode="before")
386
+ @classmethod
387
+ def check_cache_salt_support(cls, data):
388
+ if data.get("cache_salt") is not None and (
389
+ not isinstance(data["cache_salt"], str) or not data["cache_salt"]
390
+ ):
391
+ raise ValueError(
392
+ "Parameter 'cache_salt' must be a non-empty string if provided."
393
+ )
394
+ return data
395
+
396
+
397
+class CompletionLogProbs(OpenAIBaseModel):
398
+ text_offset: list[int] = Field(default_factory=list)
399
+ token_logprobs: list[float | None] = Field(default_factory=list)
400
+ tokens: list[str] = Field(default_factory=list)
401
+ top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
402
+
403
+
404
+class CompletionResponseChoice(OpenAIBaseModel):
405
+ index: int
406
+ text: str
407
+ logprobs: CompletionLogProbs | None = None
408
+ finish_reason: str | None = None
409
+ stop_reason: int | str | None = Field(
410
+ default=None,
411
+ description=(
412
+ "The stop string or token id that caused the completion "
413
+ "to stop, None if the completion finished for some other reason "
414
+ "including encountering the EOS token"
415
+ ),
416
+ )
417
+ token_ids: list[int] | None = None # For response
418
+ prompt_logprobs: list[dict[int, Logprob] | None] | None = None
419
+ prompt_token_ids: list[int] | None = None # For prompt
420
+
421
+
422
+class CompletionResponse(OpenAIBaseModel):
423
+ id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
424
+ object: Literal["text_completion"] = "text_completion"
425
+ created: int = Field(default_factory=lambda: int(time.time()))
426
+ model: str
427
+ choices: list[CompletionResponseChoice]
428
+ service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
429
+ system_fingerprint: str | None = None
430
+ usage: UsageInfo
431
+
432
+ # vLLM-specific fields that are not in OpenAI spec
433
+ kv_transfer_params: dict[str, Any] | None = Field(
434
+ default=None, description="KVTransfer parameters."
435
+ )
436
+
437
+
438
+class CompletionResponseStreamChoice(OpenAIBaseModel):
439
+ index: int
440
+ text: str
441
+ logprobs: CompletionLogProbs | None = None
442
+ finish_reason: str | None = None
443
+ stop_reason: int | str | None = Field(
444
+ default=None,
445
+ description=(
446
+ "The stop string or token id that caused the completion "
447
+ "to stop, None if the completion finished for some other reason "
448
+ "including encountering the EOS token"
449
+ ),
450
+ )
451
+ # not part of the OpenAI spec but for tracing the tokens
452
+ # prompt tokens is put into choice to align with CompletionResponseChoice
453
+ prompt_token_ids: list[int] | None = None
454
+ token_ids: list[int] | None = None
455
+
456
+
457
+class CompletionStreamResponse(OpenAIBaseModel):
458
+ id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
459
+ object: str = "text_completion"
460
+ created: int = Field(default_factory=lambda: int(time.time()))
461
+ model: str
462
+ choices: list[CompletionResponseStreamChoice]
463
+ usage: UsageInfo | None = Field(default=None)
vllm/entrypoints/openai/completion/serving.py
renamed
+5
-3
@@ -12,27 +12,29 @@ from fastapi import Request
12
13
from vllm.engine.protocol import EngineClient
14
from vllm.entrypoints.logger import RequestLogger
15
-from vllm.entrypoints.openai.engine.protocol import (
15
+from vllm.entrypoints.openai.completion.protocol import (
16
CompletionLogProbs,
17
CompletionRequest,
18
CompletionResponse,
19
CompletionResponseChoice,
20
CompletionResponseStreamChoice,
21
CompletionStreamResponse,
22
+)
23
+from vllm.entrypoints.openai.engine.protocol import (
24
ErrorResponse,
25
PromptTokenUsageInfo,
26
RequestResponseMetadata,
27
UsageInfo,
26
- VLLMValidationError,
28
)
29
from vllm.entrypoints.openai.engine.serving import (
30
GenerationError,
31
OpenAIServing,
32
clamp_prompt_logprobs,
33
)
33
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
34
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
35
from vllm.entrypoints.renderer import RenderConfig
36
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
37
+from vllm.exceptions import VLLMValidationError
38
from vllm.inputs.data import EmbedsPrompt, TokensPrompt, is_embeds_prompt
39
from vllm.logger import init_logger
40
from vllm.logprobs import Logprob
vllm/entrypoints/openai/engine/protocol.py
+1
-430
@@ -3,9 +3,8 @@
3
4
# Adapted from
5
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
-import json
6
import time
8
-from typing import Annotated, Any, ClassVar, Literal, TypeAlias
7
+from typing import Any, ClassVar, Literal, TypeAlias
8
9
import regex as re
10
import torch
@@ -17,14 +16,9 @@ from pydantic import (
16
)
17
18
from vllm.entrypoints.chat_utils import make_tool_call_id
20
-from vllm.exceptions import VLLMValidationError
19
from vllm.logger import init_logger
22
-from vllm.logprobs import Logprob
20
from vllm.sampling_params import (
24
- BeamSearchParams,
25
- RequestOutputKind,
21
SamplingParams,
27
- StructuredOutputsParams,
22
)
23
from vllm.utils import random_uuid
24
from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -226,429 +220,6 @@ def get_logits_processors(
220
return None
221
222
229
-class CompletionRequest(OpenAIBaseModel):
230
- # Ordered by official OpenAI API documentation
231
- # https://platform.openai.com/docs/api-reference/completions/create
232
- model: str | None = None
233
- prompt: list[int] | list[list[int]] | str | list[str] | None = None
234
- echo: bool | None = False
235
- frequency_penalty: float | None = 0.0
236
- logit_bias: dict[str, float] | None = None
237
- logprobs: int | None = None
238
- max_tokens: int | None = 16
239
- n: int = 1
240
- presence_penalty: float | None = 0.0
241
- seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
242
- stop: str | list[str] | None = []
243
- stream: bool | None = False
244
- stream_options: StreamOptions | None = None
245
- suffix: str | None = None
246
- temperature: float | None = None
247
- top_p: float | None = None
248
- user: str | None = None
249
-
250
- # --8<-- [start:completion-sampling-params]
251
- use_beam_search: bool = False
252
- top_k: int | None = None
253
- min_p: float | None = None
254
- repetition_penalty: float | None = None
255
- length_penalty: float = 1.0
256
- stop_token_ids: list[int] | None = []
257
- include_stop_str_in_output: bool = False
258
- ignore_eos: bool = False
259
- min_tokens: int = 0
260
- skip_special_tokens: bool = True
261
- spaces_between_special_tokens: bool = True
262
- truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
263
- None
264
- )
265
- allowed_token_ids: list[int] | None = None
266
- prompt_logprobs: int | None = None
267
- # --8<-- [end:completion-sampling-params]
268
-
269
- # --8<-- [start:completion-extra-params]
270
- prompt_embeds: bytes | list[bytes] | None = None
271
- add_special_tokens: bool = Field(
272
- default=True,
273
- description=(
274
- "If true (the default), special tokens (e.g. BOS) will be added to "
275
- "the prompt."
276
- ),
277
- )
278
- response_format: AnyResponseFormat | None = Field(
279
- default=None,
280
- description=(
281
- "Similar to chat completion, this parameter specifies the format "
282
- "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
283
- ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
284
- ),
285
- )
286
- structured_outputs: StructuredOutputsParams | None = Field(
287
- default=None,
288
- description="Additional kwargs for structured outputs",
289
- )
290
- priority: int = Field(
291
- default=0,
292
- description=(
293
- "The priority of the request (lower means earlier handling; "
294
- "default: 0). Any priority other than 0 will raise an error "
295
- "if the served model does not use priority scheduling."
296
- ),
297
- )
298
- request_id: str = Field(
299
- default_factory=random_uuid,
300
- description=(
301
- "The request_id related to this request. If the caller does "
302
- "not set it, a random_uuid will be generated. This id is used "
303
- "through out the inference process and return in response."
304
- ),
305
- )
306
- logits_processors: LogitsProcessors | None = Field(
307
- default=None,
308
- description=(
309
- "A list of either qualified names of logits processors, or "
310
- "constructor objects, to apply when sampling. A constructor is "
311
- "a JSON object with a required 'qualname' field specifying the "
312
- "qualified name of the processor class/factory, and optional "
313
- "'args' and 'kwargs' fields containing positional and keyword "
314
- "arguments. For example: {'qualname': "
315
- "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
316
- "{'param': 'value'}}."
317
- ),
318
- )
319
-
320
- return_tokens_as_token_ids: bool | None = Field(
321
- default=None,
322
- description=(
323
- "If specified with 'logprobs', tokens are represented "
324
- " as strings of the form 'token_id:{token_id}' so that tokens "
325
- "that are not JSON-encodable can be identified."
326
- ),
327
- )
328
- return_token_ids: bool | None = Field(
329
- default=None,
330
- description=(
331
- "If specified, the result will include token IDs alongside the "
332
- "generated text. In streaming mode, prompt_token_ids is included "
333
- "only in the first chunk, and token_ids contains the delta tokens "
334
- "for each chunk. This is useful for debugging or when you "
335
- "need to map generated text back to input tokens."
336
- ),
337
- )
338
-
339
- cache_salt: str | None = Field(
340
- default=None,
341
- description=(
342
- "If specified, the prefix cache will be salted with the provided "
343
- "string to prevent an attacker to guess prompts in multi-user "
344
- "environments. The salt should be random, protected from "
345
- "access by 3rd parties, and long enough to be "
346
- "unpredictable (e.g., 43 characters base64-encoded, corresponding "
347
- "to 256 bit)."
348
- ),
349
- )
350
-
351
- kv_transfer_params: dict[str, Any] | None = Field(
352
- default=None,
353
- description="KVTransfer parameters used for disaggregated serving.",
354
- )
355
-
356
- vllm_xargs: dict[str, str | int | float] | None = Field(
357
- default=None,
358
- description=(
359
- "Additional request parameters with string or "
360
- "numeric values, used by custom extensions."
361
- ),
362
- )
363
-
364
- # --8<-- [end:completion-extra-params]
365
-
366
- # Default sampling parameters for completion requests
367
- _DEFAULT_SAMPLING_PARAMS: dict = {
368
- "repetition_penalty": 1.0,
369
- "temperature": 1.0,
370
- "top_p": 1.0,
371
- "top_k": 0,
372
- "min_p": 0.0,
373
- }
374
-
375
- def to_beam_search_params(
376
- self,
377
- max_tokens: int,
378
- default_sampling_params: dict | None = None,
379
- ) -> BeamSearchParams:
380
- if default_sampling_params is None:
381
- default_sampling_params = {}
382
- n = self.n if self.n is not None else 1
383
-
384
- if (temperature := self.temperature) is None:
385
- temperature = default_sampling_params.get("temperature", 1.0)
386
-
387
- return BeamSearchParams(
388
- beam_width=n,
389
- max_tokens=max_tokens,
390
- ignore_eos=self.ignore_eos,
391
- temperature=temperature,
392
- length_penalty=self.length_penalty,
393
- include_stop_str_in_output=self.include_stop_str_in_output,
394
- )
395
-
396
- def to_sampling_params(
397
- self,
398
- max_tokens: int,
399
- logits_processor_pattern: str | None,
400
- default_sampling_params: dict | None = None,
401
- ) -> SamplingParams:
402
- if default_sampling_params is None:
403
- default_sampling_params = {}
404
-
405
- # Default parameters
406
- if (repetition_penalty := self.repetition_penalty) is None:
407
- repetition_penalty = default_sampling_params.get(
408
- "repetition_penalty",
409
- self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
410
- )
411
- if (temperature := self.temperature) is None:
412
- temperature = default_sampling_params.get(
413
- "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
414
- )
415
- if (top_p := self.top_p) is None:
416
- top_p = default_sampling_params.get(
417
- "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
418
- )
419
- if (top_k := self.top_k) is None:
420
- top_k = default_sampling_params.get(
421
- "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
422
- )
423
- if (min_p := self.min_p) is None:
424
- min_p = default_sampling_params.get(
425
- "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
426
- )
427
-
428
- prompt_logprobs = self.prompt_logprobs
429
- if prompt_logprobs is None and self.echo:
430
- prompt_logprobs = self.logprobs
431
-
432
- echo_without_generation = self.echo and self.max_tokens == 0
433
-
434
- response_format = self.response_format
435
- if response_format is not None:
436
- # If structured outputs wasn't already enabled,
437
- # we must enable it for these features to work
438
- if self.structured_outputs is None:
439
- self.structured_outputs = StructuredOutputsParams()
440
-
441
- # Set structured output params for response format
442
- if response_format.type == "json_object":
443
- self.structured_outputs.json_object = True
444
- elif response_format.type == "json_schema":
445
- json_schema = response_format.json_schema
446
- assert json_schema is not None
447
- self.structured_outputs.json = json_schema.json_schema
448
- elif response_format.type == "structural_tag":
449
- structural_tag = response_format
450
- assert structural_tag is not None and isinstance(
451
- structural_tag,
452
- (
453
- LegacyStructuralTagResponseFormat,
454
- StructuralTagResponseFormat,
455
- ),
456
- )
457
- s_tag_obj = structural_tag.model_dump(by_alias=True)
458
- self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
459
-
460
- extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
461
- if self.kv_transfer_params:
462
- # Pass in kv_transfer_params via extra_args
463
- extra_args["kv_transfer_params"] = self.kv_transfer_params
464
- return SamplingParams.from_optional(
465
- n=self.n,
466
- presence_penalty=self.presence_penalty,
467
- frequency_penalty=self.frequency_penalty,
468
- repetition_penalty=repetition_penalty,
469
- temperature=temperature,
470
- top_p=top_p,
471
- top_k=top_k,
472
- min_p=min_p,
473
- seed=self.seed,
474
- stop=self.stop,
475
- stop_token_ids=self.stop_token_ids,
476
- logprobs=self.logprobs,
477
- ignore_eos=self.ignore_eos,
478
- max_tokens=max_tokens if not echo_without_generation else 1,
479
- min_tokens=self.min_tokens,
480
- prompt_logprobs=prompt_logprobs,
481
- skip_special_tokens=self.skip_special_tokens,
482
- spaces_between_special_tokens=self.spaces_between_special_tokens,
483
- include_stop_str_in_output=self.include_stop_str_in_output,
484
- logits_processors=get_logits_processors(
485
- self.logits_processors, logits_processor_pattern
486
- ),
487
- truncate_prompt_tokens=self.truncate_prompt_tokens,
488
- output_kind=RequestOutputKind.DELTA
489
- if self.stream
490
- else RequestOutputKind.FINAL_ONLY,
491
- structured_outputs=self.structured_outputs,
492
- logit_bias=self.logit_bias,
493
- allowed_token_ids=self.allowed_token_ids,
494
- extra_args=extra_args or None,
495
- skip_clone=True, # Created fresh per request, safe to skip clone
496
- )
497
-
498
- @model_validator(mode="before")
499
- @classmethod
500
- def check_structured_outputs_count(cls, data):
501
- if data.get("structured_outputs", None) is None:
502
- return data
503
-
504
- structured_outputs_kwargs = data["structured_outputs"]
505
- count = sum(
506
- structured_outputs_kwargs.get(k) is not None
507
- for k in ("json", "regex", "choice")
508
- )
509
- if count > 1:
510
- raise VLLMValidationError(
511
- "You can only use one kind of constraints for structured "
512
- "outputs ('json', 'regex' or 'choice').",
513
- parameter="structured_outputs",
514
- )
515
- return data
516
-
517
- @model_validator(mode="before")
518
- @classmethod
519
- def check_logprobs(cls, data):
520
- if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
521
- if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
522
- raise VLLMValidationError(
523
- "`prompt_logprobs` are not available when `stream=True`.",
524
- parameter="prompt_logprobs",
525
- )
526
-
527
- if prompt_logprobs < 0 and prompt_logprobs != -1:
528
- raise VLLMValidationError(
529
- "`prompt_logprobs` must be a positive value or -1.",
530
- parameter="prompt_logprobs",
531
- value=prompt_logprobs,
532
- )
533
- if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
534
- raise VLLMValidationError(
535
- "`logprobs` must be a positive value.",
536
- parameter="logprobs",
537
- value=logprobs,
538
- )
539
-
540
- return data
541
-
542
- @model_validator(mode="before")
543
- @classmethod
544
- def validate_stream_options(cls, data):
545
- if data.get("stream_options") and not data.get("stream"):
546
- raise VLLMValidationError(
547
- "Stream options can only be defined when `stream=True`.",
548
- parameter="stream_options",
549
- )
550
-
551
- return data
552
-
553
- @model_validator(mode="before")
554
- @classmethod
555
- def validate_prompt_and_prompt_embeds(cls, data):
556
- prompt = data.get("prompt")
557
- prompt_embeds = data.get("prompt_embeds")
558
-
559
- prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
560
- embeds_is_empty = prompt_embeds is None or (
561
- isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
562
- )
563
-
564
- if prompt_is_empty and embeds_is_empty:
565
- raise ValueError(
566
- "Either prompt or prompt_embeds must be provided and non-empty."
567
- )
568
-
569
- return data
570
-
571
- @model_validator(mode="before")
572
- @classmethod
573
- def check_cache_salt_support(cls, data):
574
- if data.get("cache_salt") is not None and (
575
- not isinstance(data["cache_salt"], str) or not data["cache_salt"]
576
- ):
577
- raise ValueError(
578
- "Parameter 'cache_salt' must be a non-empty string if provided."
579
- )
580
- return data
581
-
582
-
583
-class CompletionLogProbs(OpenAIBaseModel):
584
- text_offset: list[int] = Field(default_factory=list)
585
- token_logprobs: list[float | None] = Field(default_factory=list)
586
- tokens: list[str] = Field(default_factory=list)
587
- top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
588
-
589
-
590
-class CompletionResponseChoice(OpenAIBaseModel):
591
- index: int
592
- text: str
593
- logprobs: CompletionLogProbs | None = None
594
- finish_reason: str | None = None
595
- stop_reason: int | str | None = Field(
596
- default=None,
597
- description=(
598
- "The stop string or token id that caused the completion "
599
- "to stop, None if the completion finished for some other reason "
600
- "including encountering the EOS token"
601
- ),
602
- )
603
- token_ids: list[int] | None = None # For response
604
- prompt_logprobs: list[dict[int, Logprob] | None] | None = None
605
- prompt_token_ids: list[int] | None = None # For prompt
606
-
607
-
608
-class CompletionResponse(OpenAIBaseModel):
609
- id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
610
- object: Literal["text_completion"] = "text_completion"
611
- created: int = Field(default_factory=lambda: int(time.time()))
612
- model: str
613
- choices: list[CompletionResponseChoice]
614
- service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
615
- system_fingerprint: str | None = None
616
- usage: UsageInfo
617
-
618
- # vLLM-specific fields that are not in OpenAI spec
619
- kv_transfer_params: dict[str, Any] | None = Field(
620
- default=None, description="KVTransfer parameters."
621
- )
622
-
623
-
624
-class CompletionResponseStreamChoice(OpenAIBaseModel):
625
- index: int
626
- text: str
627
- logprobs: CompletionLogProbs | None = None
628
- finish_reason: str | None = None
629
- stop_reason: int | str | None = Field(
630
- default=None,
631
- description=(
632
- "The stop string or token id that caused the completion "
633
- "to stop, None if the completion finished for some other reason "
634
- "including encountering the EOS token"
635
- ),
636
- )
637
- # not part of the OpenAI spec but for tracing the tokens
638
- # prompt tokens is put into choice to align with CompletionResponseChoice
639
- prompt_token_ids: list[int] | None = None
640
- token_ids: list[int] | None = None
641
-
642
-
643
-class CompletionStreamResponse(OpenAIBaseModel):
644
- id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
645
- object: str = "text_completion"
646
- created: int = Field(default_factory=lambda: int(time.time()))
647
- model: str
648
- choices: list[CompletionResponseStreamChoice]
649
- usage: UsageInfo | None = Field(default=None)
650
-
651
-
223
class FunctionCall(OpenAIBaseModel):
224
name: str
225
arguments: str
vllm/entrypoints/openai/engine/serving.py
+5
-3
@@ -43,20 +43,21 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
43
ChatCompletionRequest,
44
ChatCompletionResponse,
45
)
46
-from vllm.entrypoints.openai.engine.protocol import (
46
+from vllm.entrypoints.openai.completion.protocol import (
47
CompletionRequest,
48
CompletionResponse,
49
+)
50
+from vllm.entrypoints.openai.engine.protocol import (
51
ErrorInfo,
52
ErrorResponse,
53
FunctionCall,
54
FunctionDefinition,
53
- VLLMValidationError,
55
)
56
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
57
from vllm.entrypoints.openai.responses.protocol import (
58
ResponseInputOutputItem,
59
ResponsesRequest,
60
)
59
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
61
from vllm.entrypoints.openai.translations.protocol import (
62
TranscriptionRequest,
63
TranscriptionResponse,
@@ -95,6 +96,7 @@ from vllm.entrypoints.serve.tokenize.protocol import (
96
TokenizeResponse,
97
)
98
from vllm.entrypoints.utils import _validate_truncation_size, sanitize_message
99
+from vllm.exceptions import VLLMValidationError
100
from vllm.inputs.data import PromptType, TokensPrompt
101
from vllm.inputs.parse import (
102
PromptComponents,
vllm/entrypoints/openai/models/__init__.py
vllm/entrypoints/openai/models/api_router.py
new
+29
@@ -0,0 +1,29 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from fastapi import APIRouter, FastAPI, Request
6
+from fastapi.responses import JSONResponse
7
+
8
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
9
+from vllm.logger import init_logger
10
+
11
+logger = init_logger(__name__)
12
+
13
+router = APIRouter()
14
+
15
+
16
+def models(request: Request) -> OpenAIServingModels:
17
+ return request.app.state.openai_serving_models
18
+
19
+
20
+@router.get("/v1/models")
21
+async def show_available_models(raw_request: Request):
22
+ handler = models(raw_request)
23
+
24
+ models_ = await handler.show_available_models()
25
+ return JSONResponse(content=models_.model_dump())
26
+
27
+
28
+def attach_router(app: FastAPI):
29
+ app.include_router(router)
vllm/entrypoints/openai/models/protocol.py
new
+18
@@ -0,0 +1,18 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+from dataclasses import dataclass
6
+
7
+
8
+@dataclass
9
+class BaseModelPath:
10
+ name: str
11
+ model_path: str
12
+
13
+
14
+@dataclass
15
+class LoRAModulePath:
16
+ name: str
17
+ path: str
18
+ base_model_name: str | None = None
vllm/entrypoints/openai/models/serving.py
renamed
+1
-14
@@ -3,7 +3,6 @@
3
4
from asyncio import Lock
5
from collections import defaultdict
6
-from dataclasses import dataclass
6
from http import HTTPStatus
7
8
from vllm.engine.protocol import EngineClient
@@ -14,6 +13,7 @@ from vllm.entrypoints.openai.engine.protocol import (
13
ModelList,
14
ModelPermission,
15
)
16
+from vllm.entrypoints.openai.models.protocol import BaseModelPath, LoRAModulePath
17
from vllm.entrypoints.serve.lora.protocol import (
18
LoadLoRAAdapterRequest,
19
UnloadLoRAAdapterRequest,
@@ -27,19 +27,6 @@ from vllm.utils.counter import AtomicCounter
27
logger = init_logger(__name__)
28
29
30
-@dataclass
31
-class BaseModelPath:
32
- name: str
33
- model_path: str
34
-
35
-
36
-@dataclass
37
-class LoRAModulePath:
38
- name: str
39
- path: str
40
- base_model_name: str | None = None
41
-
42
-
30
class OpenAIServingModels:
31
"""Shared instance to hold data about the loaded base model(s) and adapters.
32
vllm/entrypoints/openai/responses/serving.py
+2
-2
@@ -76,12 +76,12 @@ from vllm.entrypoints.openai.engine.protocol import (
76
DeltaMessage,
77
ErrorResponse,
78
RequestResponseMetadata,
79
- VLLMValidationError,
79
)
80
from vllm.entrypoints.openai.engine.serving import (
81
GenerationError,
82
OpenAIServing,
83
)
84
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
85
from vllm.entrypoints.openai.parser.harmony_utils import (
86
construct_harmony_previous_input_messages,
87
get_developer_message,
@@ -108,7 +108,6 @@ from vllm.entrypoints.openai.responses.protocol import (
108
ResponseUsage,
109
StreamingResponsesResponse,
110
)
111
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
111
from vllm.entrypoints.responses_utils import (
112
construct_input_messages,
113
construct_tool_dicts,
@@ -116,6 +115,7 @@ from vllm.entrypoints.responses_utils import (
115
should_continue_final_message,
116
)
117
from vllm.entrypoints.tool_server import ToolServer
118
+from vllm.exceptions import VLLMValidationError
119
from vllm.inputs.data import TokensPrompt
120
from vllm.logger import init_logger
121
from vllm.logprobs import Logprob as SampleLogprob
vllm/entrypoints/openai/run_batch.py
+2
-1
@@ -28,7 +28,8 @@ from vllm.entrypoints.openai.engine.protocol import (
28
ErrorResponse,
29
OpenAIBaseModel,
30
)
31
-from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
31
+from vllm.entrypoints.openai.models.protocol import BaseModelPath
32
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
33
from vllm.entrypoints.pooling.embed.protocol import EmbeddingRequest, EmbeddingResponse
34
from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
35
from vllm.entrypoints.pooling.score.protocol import (
vllm/entrypoints/openai/translations/api_router.py
+3
-8
@@ -5,7 +5,7 @@
5
from http import HTTPStatus
6
from typing import Annotated
7
8
-from fastapi import APIRouter, FastAPI, Form, HTTPException, Request
8
+from fastapi import APIRouter, FastAPI, Form, Request
9
from fastapi.responses import JSONResponse, StreamingResponse
10
11
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
@@ -63,10 +63,7 @@ async def create_transcriptions(
63
try:
64
generator = await handler.create_transcription(audio_data, request, raw_request)
65
except Exception as e:
66
- raise HTTPException(
67
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
68
- ) from e
69
-
66
+ return handler.create_error_response(e)
67
if isinstance(generator, ErrorResponse):
68
return JSONResponse(
69
content=generator.model_dump(), status_code=generator.error.code
@@ -103,9 +100,7 @@ async def create_translations(
100
try:
101
generator = await handler.create_translation(audio_data, request, raw_request)
102
except Exception as e:
106
- raise HTTPException(
107
- status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)
108
- ) from e
103
+ return handler.create_error_response(e)
104
105
if isinstance(generator, ErrorResponse):
106
return JSONResponse(
vllm/entrypoints/openai/translations/serving.py
+1
-1
@@ -10,7 +10,7 @@ from vllm.entrypoints.openai.engine.protocol import (
10
ErrorResponse,
11
RequestResponseMetadata,
12
)
13
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
13
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
14
from vllm.entrypoints.openai.translations.protocol import (
15
TranscriptionRequest,
16
TranscriptionResponse,
vllm/entrypoints/openai/translations/speech_to_text.py
+1
-1
@@ -22,7 +22,7 @@ from vllm.entrypoints.openai.engine.protocol import (
22
UsageInfo,
23
)
24
from vllm.entrypoints.openai.engine.serving import OpenAIServing, SpeechToTextRequest
25
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
25
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
26
from vllm.entrypoints.openai.translations.protocol import (
27
TranscriptionResponse,
28
TranscriptionResponseStreamChoice,
vllm/entrypoints/pooling/classify/serving.py
+1
-1
@@ -23,7 +23,7 @@ from vllm.entrypoints.openai.engine.serving import (
23
OpenAIServing,
24
ServeContext,
25
)
26
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
from vllm.entrypoints.pooling.classify.protocol import (
28
ClassificationChatRequest,
29
ClassificationCompletionRequest,
vllm/entrypoints/pooling/embed/serving.py
+1
-1
@@ -21,7 +21,7 @@ from vllm.entrypoints.openai.engine.serving import (
21
OpenAIServing,
22
ServeContext,
23
)
24
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
24
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
25
from vllm.entrypoints.pooling.embed.protocol import (
26
EmbeddingBytesResponse,
27
EmbeddingChatRequest,
vllm/entrypoints/pooling/pooling/serving.py
+1
-1
@@ -19,7 +19,7 @@ from vllm.entrypoints.openai.engine.protocol import (
19
UsageInfo,
20
)
21
from vllm.entrypoints.openai.engine.serving import OpenAIServing
22
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
22
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
23
from vllm.entrypoints.pooling.pooling.protocol import (
24
IOProcessorRequest,
25
IOProcessorResponse,
vllm/entrypoints/pooling/score/serving.py
+1
-1
@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
14
UsageInfo,
15
)
16
from vllm.entrypoints.openai.engine.serving import OpenAIServing
17
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
17
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
18
from vllm.entrypoints.pooling.score.protocol import (
19
RerankDocument,
20
RerankRequest,
vllm/entrypoints/sagemaker/routes.py
+9
-5
@@ -12,22 +12,26 @@ from fastapi.responses import JSONResponse, Response
12
13
from vllm.entrypoints.openai.api_server import (
14
base,
15
- chat,
16
- completion,
17
- create_completion,
18
- validate_json_request,
15
)
16
from vllm.entrypoints.openai.chat_completion.api_router import (
17
+ chat,
18
create_chat_completion,
19
)
20
from vllm.entrypoints.openai.chat_completion.protocol import (
21
ChatCompletionRequest,
22
)
26
-from vllm.entrypoints.openai.engine.protocol import (
23
+from vllm.entrypoints.openai.completion.api_router import (
24
+ completion,
25
+ create_completion,
26
+)
27
+from vllm.entrypoints.openai.completion.protocol import (
28
CompletionRequest,
29
+)
30
+from vllm.entrypoints.openai.engine.protocol import (
31
ErrorResponse,
32
)
33
from vllm.entrypoints.openai.engine.serving import OpenAIServing
34
+from vllm.entrypoints.openai.utils import validate_json_request
35
from vllm.entrypoints.pooling.classify.api_router import classify, create_classify
36
from vllm.entrypoints.pooling.classify.protocol import ClassificationRequest
37
from vllm.entrypoints.pooling.embed.api_router import create_embedding, embedding
vllm/entrypoints/serve/disagg/api_router.py
+1
-1
@@ -10,10 +10,10 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Respons
10
from fastapi.responses import JSONResponse, StreamingResponse
11
12
from vllm.engine.protocol import EngineClient
13
-from vllm.entrypoints.openai.api_server import validate_json_request
13
from vllm.entrypoints.openai.engine.protocol import (
14
ErrorResponse,
15
)
16
+from vllm.entrypoints.openai.utils import validate_json_request
17
from vllm.entrypoints.serve.disagg.protocol import (
18
GenerateRequest,
19
GenerateResponse,
vllm/entrypoints/serve/disagg/protocol.py
+1
-1
@@ -6,10 +6,10 @@ from pydantic import BaseModel, Field
6
7
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
8
from vllm.entrypoints.openai.engine.protocol import (
9
- Logprob,
9
SamplingParams,
10
StreamOptions,
11
)
12
+from vllm.logprobs import Logprob
13
from vllm.utils import random_uuid
14
15
vllm/entrypoints/serve/disagg/serving.py
+1
-1
@@ -23,7 +23,7 @@ from vllm.entrypoints.openai.engine.protocol import (
23
UsageInfo,
24
)
25
from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
26
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
26
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
27
from vllm.entrypoints.serve.disagg.protocol import (
28
GenerateRequest,
29
GenerateResponse,
vllm/entrypoints/serve/elastic_ep/api_router.py
+1
-1
@@ -9,10 +9,10 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
9
from fastapi.responses import JSONResponse
10
11
from vllm.engine.protocol import EngineClient
12
-from vllm.entrypoints.openai.api_server import validate_json_request
12
from vllm.entrypoints.openai.engine.protocol import (
13
ErrorResponse,
14
)
15
+from vllm.entrypoints.openai.utils import validate_json_request
16
from vllm.entrypoints.serve.elastic_ep.middleware import (
17
get_scaling_elastic_ep,
18
set_scaling_elastic_ep,
vllm/entrypoints/serve/lora/api_router.py
+3
-2
@@ -7,11 +7,12 @@ from fastapi import APIRouter, Depends, FastAPI, Request
7
from fastapi.responses import JSONResponse, Response
8
9
from vllm import envs
10
-from vllm.entrypoints.openai.api_server import models, validate_json_request
10
from vllm.entrypoints.openai.engine.protocol import (
11
ErrorResponse,
12
)
14
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
13
+from vllm.entrypoints.openai.models.api_router import models
14
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
15
+from vllm.entrypoints.openai.utils import validate_json_request
16
from vllm.entrypoints.serve.lora.protocol import (
17
LoadLoRAAdapterRequest,
18
UnloadLoRAAdapterRequest,
vllm/entrypoints/serve/tokenize/api_router.py
+1
-1
@@ -9,10 +9,10 @@ from fastapi.exceptions import RequestValidationError
9
from fastapi.responses import JSONResponse
10
from typing_extensions import assert_never
11
12
-from vllm.entrypoints.openai.api_server import validate_json_request
12
from vllm.entrypoints.openai.engine.protocol import (
13
ErrorResponse,
14
)
15
+from vllm.entrypoints.openai.utils import validate_json_request
16
from vllm.entrypoints.serve.tokenize.protocol import (
17
DetokenizeRequest,
18
DetokenizeResponse,
vllm/entrypoints/serve/tokenize/serving.py
+1
-1
@@ -13,7 +13,7 @@ from vllm.entrypoints.openai.engine.protocol import (
13
ErrorResponse,
14
)
15
from vllm.entrypoints.openai.engine.serving import OpenAIServing
16
-from vllm.entrypoints.openai.serving_models import OpenAIServingModels
16
+from vllm.entrypoints.openai.models.serving import OpenAIServingModels
17
from vllm.entrypoints.renderer import RenderConfig
18
from vllm.entrypoints.serve.tokenize.protocol import (
19
DetokenizeRequest,
vllm/entrypoints/utils.py
+5
-3
@@ -31,11 +31,13 @@ if TYPE_CHECKING:
31
from vllm.entrypoints.openai.chat_completion.protocol import (
32
ChatCompletionRequest,
33
)
34
- from vllm.entrypoints.openai.engine.protocol import (
34
+ from vllm.entrypoints.openai.completion.protocol import (
35
CompletionRequest,
36
+ )
37
+ from vllm.entrypoints.openai.engine.protocol import (
38
StreamOptions,
39
)
38
- from vllm.entrypoints.openai.serving_models import LoRAModulePath
40
+ from vllm.entrypoints.openai.models.protocol import LoRAModulePath
41
else:
42
ChatCompletionRequest = object
43
CompletionRequest = object
@@ -281,7 +283,7 @@ def should_include_usage(
283
def process_lora_modules(
284
args_lora_modules: list[LoRAModulePath], default_mm_loras: dict[str, str] | None
285
) -> list[LoRAModulePath]:
284
- from vllm.entrypoints.openai.serving_models import LoRAModulePath
286
+ from vllm.entrypoints.openai.models.serving import LoRAModulePath
287
288
lora_modules = args_lora_modules
289
if default_mm_loras: