A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 08:30 UTC
[Frontend] Resettle generative scoring entrypoint. (#44153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jun 1, 2026 at 15:54 UTC
0910f7e0e1f5ee9f2e5a6f76d0d09e68fddadc01
13 files changed
+38
-58
.buildkite/test-amd.yaml
+5
-1
@@ -1275,10 +1275,12 @@ steps:
1275
- vllm/
1276
- tests/entrypoints/openai
1277
- tests/entrypoints/test_chat_utils
1278
+ - tests/entrypoints/generate
1279
commands:
1280
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
1281
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
1282
- pytest -v -s entrypoints/test_chat_utils.py
1283
+ - pytest -v -s entrypoints/generate
1284
1285
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
1286
timeout_in_minutes: 180
@@ -1368,7 +1370,7 @@ steps:
1370
- vllm/platforms/rocm.py
1371
commands:
1372
- pytest -v -s entrypoints/openai/tool_parsers
1371
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling
1373
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
1374
1375
- label: OpenAI API correctness # TBD
1376
timeout_in_minutes: 180
@@ -2782,10 +2784,12 @@ steps:
2784
- vllm/
2785
- tests/entrypoints/openai
2786
- tests/entrypoints/test_chat_utils
2787
+ - tests/entrypoints/generate
2788
commands:
2789
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
2790
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
2791
- pytest -v -s entrypoints/test_chat_utils.py
2792
+ - pytest -v -s entrypoints/generate
2793
2794
- label: Entrypoints Integration (API Server openai - Part 3) # TBD
2795
timeout_in_minutes: 180
.buildkite/test_areas/entrypoints.yaml
+3
-1
@@ -11,7 +11,7 @@ steps:
11
- tests/entrypoints/
12
commands:
13
- pytest -v -s entrypoints/openai/tool_parsers
14
- - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text
14
+ - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
15
16
- label: Entrypoints Integration (LLM)
17
key: entrypoints-integration-llm
@@ -60,9 +60,11 @@ steps:
60
- vllm/
61
- tests/entrypoints/openai
62
- tests/entrypoints/test_chat_utils
63
+ - tests/entrypoints/generate
64
commands:
65
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
66
- pytest -v -s entrypoints/test_chat_utils.py
67
+ - pytest -v -s entrypoints/generate
68
mirror:
69
amd:
70
device: mi325_1
tests/entrypoints/generate/__init__.py
renamed
tests/entrypoints/generate/generative_scoring/__init__.py
renamed
tests/entrypoints/generate/generative_scoring/test_generative_scoring.py
renamed
+8
-8
@@ -18,13 +18,13 @@ from unittest.mock import MagicMock
18
import pytest
19
20
from vllm.config.multimodal import MultiModalConfig
21
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
22
-from vllm.entrypoints.openai.generative_scoring.serving import (
21
+from vllm.entrypoints.generate.generative_scoring.serving import (
22
GenerativeScoringItemResult,
23
GenerativeScoringRequest,
24
GenerativeScoringResponse,
26
- OpenAIServingGenerativeScoring,
25
+ ServingGenerativeScoring,
26
)
27
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
28
from vllm.entrypoints.openai.models.protocol import BaseModelPath
29
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
30
from vllm.logprobs import Logprob
@@ -86,13 +86,13 @@ def _create_mock_engine():
86
return mock_engine
87
88
89
-def _create_serving(mock_engine) -> OpenAIServingGenerativeScoring:
90
- """Create an OpenAIServingGenerativeScoring instance with mocks."""
89
+def _create_serving(mock_engine) -> ServingGenerativeScoring:
90
+ """Create an ServingGenerativeScoring instance with mocks."""
91
models = OpenAIServingModels(
92
engine_client=mock_engine,
93
base_model_paths=BASE_MODEL_PATHS,
94
)
95
- return OpenAIServingGenerativeScoring(mock_engine, models, request_logger=None)
95
+ return ServingGenerativeScoring(mock_engine, models, request_logger=None)
96
97
98
def _create_mock_request_output(logprobs_dict: dict[int, float]) -> RequestOutput:
@@ -186,7 +186,7 @@ class TestProbabilityComputation:
186
self, label_logprobs, apply_softmax, should_sum_to_one
187
):
188
"""Test probability computation for softmax and true probability modes."""
189
- serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring)
189
+ serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
190
probs = serving._compute_probabilities(
191
label_logprobs, apply_softmax=apply_softmax
192
)
@@ -211,7 +211,7 @@ class TestProbabilityComputation:
211
212
def test_score_formula(self):
213
"""Test the score formula: P(token[0]) / (P(token[0]) + P(token[1]))."""
214
- serving = OpenAIServingGenerativeScoring.__new__(OpenAIServingGenerativeScoring)
214
+ serving = ServingGenerativeScoring.__new__(ServingGenerativeScoring)
215
216
# With logprobs -0.5 and -2.0, softmax gives higher prob to first token
217
logprobs = {9454: -0.5, 2753: -2.0}
tests/entrypoints/generate/generative_scoring/test_generative_scoring_e2e.py
renamed
+1
-1
@@ -8,7 +8,7 @@ Tests verify the full HTTP request/response flow using RemoteOpenAIServer.
8
import pytest
9
import requests
10
11
-from ....utils import RemoteOpenAIServer
11
+from tests.utils import RemoteOpenAIServer
12
13
MODEL_NAME = "Qwen/Qwen3-0.6B"
14
vllm/entrypoints/generate/api_router.py
renamed
+12
@@ -41,6 +41,10 @@ def register_generate_api_routers(app: FastAPI):
41
42
register_anthropic_api_router(app)
43
44
+ from .generative_scoring.api_router import register_generative_scoring_api_router
45
+
46
+ register_generative_scoring_api_router(app)
47
+
48
49
async def init_generate_state(
50
engine_client: "EngineClient",
@@ -185,3 +189,11 @@ async def init_generate_state(
189
if "generate" in supported_tasks
190
else None
191
)
192
+
193
+ from .generative_scoring.serving import ServingGenerativeScoring
194
+
195
+ state.serving_generative_scoring = ServingGenerativeScoring(
196
+ engine_client,
197
+ state.openai_serving_models,
198
+ request_logger=request_logger,
199
+ )
vllm/entrypoints/generate/factories.py
renamed
vllm/entrypoints/generate/generative_scoring/__init__.py
renamed
vllm/entrypoints/generate/generative_scoring/api_router.py
renamed
+5
-31
@@ -1,34 +1,25 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from http import HTTPStatus
4
-from typing import TYPE_CHECKING
4
5
from fastapi import APIRouter, Depends, FastAPI, Request
6
from fastapi.responses import JSONResponse
7
9
-from vllm.entrypoints.openai.engine.protocol import ErrorResponse
10
-from vllm.entrypoints.openai.generative_scoring.serving import (
8
+from vllm.entrypoints.generate.generative_scoring.serving import (
9
GenerativeScoringResponse,
12
- OpenAIServingGenerativeScoring,
10
+ ServingGenerativeScoring,
11
)
12
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
13
from vllm.entrypoints.openai.utils import validate_json_request
14
from vllm.entrypoints.utils import load_aware_call, with_cancellation
15
from vllm.logger import init_logger
16
18
-if TYPE_CHECKING:
19
- from argparse import Namespace
20
-
21
- from starlette.datastructures import State
22
-
23
- from vllm.engine.protocol import EngineClient
24
- from vllm.entrypoints.logger import RequestLogger
25
-
17
router = APIRouter()
18
19
logger = init_logger(__name__)
20
21
31
-def generative_scoring(request: Request) -> OpenAIServingGenerativeScoring | None:
22
+def generative_scoring(request: Request) -> ServingGenerativeScoring | None:
23
return request.app.state.serving_generative_scoring
24
25
@@ -51,7 +42,7 @@ async def create_generative_scoring(raw_request: Request):
42
43
raw_body = await raw_request.json()
44
54
- from vllm.entrypoints.openai.generative_scoring.serving import (
45
+ from vllm.entrypoints.generate.generative_scoring.serving import (
46
GenerativeScoringRequest,
47
)
48
@@ -68,20 +59,3 @@ async def create_generative_scoring(raw_request: Request):
59
60
def register_generative_scoring_api_router(app: FastAPI):
61
app.include_router(router)
71
-
72
-
73
-async def init_generative_scoring_state(
74
- engine_client: "EngineClient",
75
- state: "State",
76
- args: "Namespace",
77
- request_logger: "RequestLogger | None",
78
-):
79
- from vllm.entrypoints.openai.generative_scoring.serving import (
80
- OpenAIServingGenerativeScoring,
81
- )
82
-
83
- state.serving_generative_scoring = OpenAIServingGenerativeScoring(
84
- engine_client,
85
- state.openai_serving_models,
86
- request_logger=request_logger,
87
- )
vllm/entrypoints/generate/generative_scoring/serving.py
renamed
+1
-1
@@ -142,7 +142,7 @@ class GenerativeScoringResponse(OpenAIBaseModel):
142
# ============================================================================
143
144
145
-class OpenAIServingGenerativeScoring(OpenAIServing):
145
+class ServingGenerativeScoring(OpenAIServing):
146
"""Serving class for generative scoring computation.
147
148
This class handles computing the probability of specified token IDs
vllm/entrypoints/openai/api_server.py
+2
-14
@@ -196,7 +196,7 @@ def build_app(
196
register_sagemaker_api_router(app, supported_tasks, model_config)
197
198
if "generate" in supported_tasks:
199
- from vllm.entrypoints.openai.generate.api_router import (
199
+ from vllm.entrypoints.generate.api_router import (
200
register_generate_api_routers,
201
)
202
@@ -220,12 +220,6 @@ def build_app(
220
221
elastic_ep_attach_router(app)
222
223
- from vllm.entrypoints.openai.generative_scoring.api_router import (
224
- register_generative_scoring_api_router,
225
- )
226
-
227
- register_generative_scoring_api_router(app)
228
-
223
if "generate" in supported_tasks or "render" in supported_tasks:
224
from vllm.entrypoints.serve.render.api_router import (
225
attach_router as attach_render_router,
@@ -402,18 +396,12 @@ async def init_app_state(
396
)
397
398
if "generate" in supported_tasks:
405
- from vllm.entrypoints.openai.generate.api_router import init_generate_state
399
+ from vllm.entrypoints.generate.api_router import init_generate_state
400
401
await init_generate_state(
402
engine_client, state, args, request_logger, supported_tasks
403
)
404
411
- from vllm.entrypoints.openai.generative_scoring.api_router import (
412
- init_generative_scoring_state,
413
- )
414
-
415
- await init_generative_scoring_state(engine_client, state, args, request_logger)
416
-
405
if "transcription" in supported_tasks or "realtime" in supported_tasks:
406
from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
407
vllm/entrypoints/sagemaker/api_router.py
+1
-1
@@ -11,9 +11,9 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
11
from fastapi.responses import JSONResponse, Response
12
13
from vllm.config import ModelConfig
14
+from vllm.entrypoints.generate.factories import get_generate_invocation_types
15
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
16
from vllm.entrypoints.openai.engine.serving import OpenAIServing
16
-from vllm.entrypoints.openai.generate.factories import get_generate_invocation_types
17
from vllm.entrypoints.openai.utils import validate_json_request
18
from vllm.entrypoints.pooling.base.serving import PoolingServingBase
19
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types