@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9cea90eab4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 11:00 UTC

[Frontend] Add /classify endpoint (#17032)

Signed-off-by: Frieda (Jingying) Huang <jingyingfhuang@gmail.com>

Frieda Huang committed May 11, 2025 at 03:57 UTC 9cea90eab4e26892a75e7b8faa3c0a2aa8d52923
9 files changed +961 -162
docs/source/models/pooling_models.md
+1
@@ -140,6 +140,7 @@ Our [OpenAI-Compatible Server](#openai-compatible-server) provides endpoints tha
140
141 - [Pooling API](#pooling-api) is similar to `LLM.encode`, being applicable to all types of pooling models.
142 - [Embeddings API](#embeddings-api) is similar to `LLM.embed`, accepting both text and [multi-modal inputs](#multimodal-inputs) for embedding models.
143 +- [Classification API](#classification-api) is similar to `LLM.classify` and is applicable to sequence classification models.
144 - [Score API](#score-api) is similar to `LLM.score` for cross-encoder models.
145
146 ## Matryoshka Embeddings
docs/source/serving/openai_compatible_server.md
+126
@@ -61,6 +61,8 @@ In addition, we have the following custom APIs:
61 - Applicable to any model with a tokenizer.
62 - [Pooling API](#pooling-api) (`/pooling`)
63 - Applicable to all [pooling models](../models/pooling_models.md).
64 +- [Classification API](#classification-api) (`/classify`)
65 + - Only applicable to [classification models](../models/pooling_models.md) (`--task classify`).
66 - [Score API](#score-api) (`/score`)
67 - Applicable to embedding models and [cross-encoder models](../models/pooling_models.md) (`--task score`).
68 - [Re-rank API](#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
@@ -443,6 +445,130 @@ The input format is the same as [Embeddings API](#embeddings-api), but the outpu
445
446 Code example: <gh-file:examples/online_serving/openai_pooling_client.py>
447
448 +(classification-api)=
449 +
450 +### Classification API
451 +
452 +Our Classification API directly supports Hugging Face sequence-classification models such as [ai21labs/Jamba-tiny-reward-dev](https://huggingface.co/ai21labs/Jamba-tiny-reward-dev) and [jason9693/Qwen2.5-1.5B-apeach](https://huggingface.co/jason9693/Qwen2.5-1.5B-apeach).
453 +
454 +We automatically wrap any other transformer via `as_classification_model()`, which pools on the last token, attaches a `RowParallelLinear` head, and applies a softmax to produce per-class probabilities.
455 +
456 +Code example: <gh-file:examples/online_serving/openai_classification_client.py>
457 +
458 +#### Example Requests
459 +
460 +You can classify multiple texts by passing an array of strings:
461 +
462 +Request:
463 +
464 +```bash
465 +curl -v "http://127.0.0.1:8000/classify" \
466 + -H "Content-Type: application/json" \
467 + -d '{
468 + "model": "jason9693/Qwen2.5-1.5B-apeach",
469 + "input": [
470 + "Loved the new café—coffee was great.",
471 + "This update broke everything. Frustrating."
472 + ]
473 + }'
474 +```
475 +
476 +Response:
477 +
478 +```bash
479 +{
480 + "id": "classify-7c87cac407b749a6935d8c7ce2a8fba2",
481 + "object": "list",
482 + "created": 1745383065,
483 + "model": "jason9693/Qwen2.5-1.5B-apeach",
484 + "data": [
485 + {
486 + "index": 0,
487 + "label": "Default",
488 + "probs": [
489 + 0.565970778465271,
490 + 0.4340292513370514
491 + ],
492 + "num_classes": 2
493 + },
494 + {
495 + "index": 1,
496 + "label": "Spoiled",
497 + "probs": [
498 + 0.26448777318000793,
499 + 0.7355121970176697
500 + ],
501 + "num_classes": 2
502 + }
503 + ],
504 + "usage": {
505 + "prompt_tokens": 20,
506 + "total_tokens": 20,
507 + "completion_tokens": 0,
508 + "prompt_tokens_details": null
509 + }
510 +}
511 +```
512 +
513 +You can also pass a string directly to the `input` field:
514 +
515 +Request:
516 +
517 +```bash
518 +curl -v "http://127.0.0.1:8000/classify" \
519 + -H "Content-Type: application/json" \
520 + -d '{
521 + "model": "jason9693/Qwen2.5-1.5B-apeach",
522 + "input": "Loved the new café—coffee was great."
523 + }'
524 +```
525 +
526 +Response:
527 +
528 +```bash
529 +{
530 + "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682",
531 + "object": "list",
532 + "created": 1745383213,
533 + "model": "jason9693/Qwen2.5-1.5B-apeach",
534 + "data": [
535 + {
536 + "index": 0,
537 + "label": "Default",
538 + "probs": [
539 + 0.565970778465271,
540 + 0.4340292513370514
541 + ],
542 + "num_classes": 2
543 + }
544 + ],
545 + "usage": {
546 + "prompt_tokens": 10,
547 + "total_tokens": 10,
548 + "completion_tokens": 0,
549 + "prompt_tokens_details": null
550 + }
551 +}
552 +```
553 +
554 +#### Extra parameters
555 +
556 +The following [pooling parameters](#pooling-params) are supported.
557 +
558 +:::{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
559 +:language: python
560 +:start-after: begin-classification-pooling-params
561 +:end-before: end-classification-pooling-params
562 +:::
563 +
564 +The following extra parameters are supported:
565 +
566 +:::{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
567 +:language: python
568 +:start-after: begin-classification-extra-params
569 +:end-before: end-classification-extra-params
570 +:::
571 +
572 (score-api)=
573
574 ### Score API
examples/online_serving/openai_classification_client.py new
+49
@@ -0,0 +1,49 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import argparse
4 +import pprint
5 +
6 +import requests
7 +
8 +
9 +def post_http_request(payload: dict, api_url: str) -> requests.Response:
10 + headers = {"User-Agent": "Test Client"}
11 + response = requests.post(api_url, headers=headers, json=payload)
12 + return response
13 +
14 +
15 +def parse_args():
16 + parse = argparse.ArgumentParser()
17 + parse.add_argument("--host", type=str, default="localhost")
18 + parse.add_argument("--port", type=int, default=8000)
19 + parse.add_argument("--model",
20 + type=str,
21 + default="jason9693/Qwen2.5-1.5B-apeach")
22 + return parse.parse_args()
23 +
24 +
25 +def main(args):
26 + host = args.host
27 + port = args.port
28 + model_name = args.model
29 +
30 + api_url = f"http://{host}:{port}/classify"
31 + prompts = [
32 + "Hello, my name is",
33 + "The president of the United States is",
34 + "The capital of France is",
35 + "The future of AI is",
36 + ]
37 +
38 + payload = {
39 + "model": model_name,
40 + "input": prompts,
41 + }
42 +
43 + classify_response = post_http_request(payload=payload, api_url=api_url)
44 + pprint.pprint(classify_response.json())
45 +
46 +
47 +if __name__ == "__main__":
48 + args = parse_args()
49 + main(args)
tests/entrypoints/openai/test_classification.py new
+156
@@ -0,0 +1,156 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import pytest
4 +import requests
5 +
6 +from vllm.entrypoints.openai.protocol import ClassificationResponse
7 +
8 +from ...utils import RemoteOpenAIServer
9 +
10 +MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
11 +DTYPE = "float32" # Use float32 to avoid NaN issue
12 +
13 +
14 +@pytest.fixture(scope="module")
15 +def server():
16 + args = [
17 + "--enforce-eager",
18 + "--max-model-len",
19 + "512",
20 + "--dtype",
21 + DTYPE,
22 + ]
23 +
24 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
25 + yield remote_server
26 +
27 +
28 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
29 +def test_single_input_classification(server: RemoteOpenAIServer,
30 + model_name: str):
31 + input_text = "This product was excellent and exceeded my expectations"
32 +
33 + classification_response = requests.post(
34 + server.url_for("classify"),
35 + json={
36 + "model": model_name,
37 + "input": input_text
38 + },
39 + )
40 +
41 + classification_response.raise_for_status()
42 + output = ClassificationResponse.model_validate(
43 + classification_response.json())
44 +
45 + assert output.object == "list"
46 + assert output.model == MODEL_NAME
47 + assert len(output.data) == 1
48 + assert hasattr(output.data[0], "label")
49 + assert hasattr(output.data[0], "probs")
50 +
51 +
52 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
53 +def test_multiple_inputs_classification(server: RemoteOpenAIServer,
54 + model_name: str):
55 + input_texts = [
56 + "The product arrived on time and works perfectly",
57 + "I'm very satisfied with my purchase, would buy again",
58 + "The customer service was helpful and resolved my issue quickly",
59 + "This product broke after one week, terrible quality",
60 + "I'm very disappointed with this purchase, complete waste of money",
61 + "The customer service was rude and unhelpful",
62 + ]
63 +
64 + classification_response = requests.post(
65 + server.url_for("classify"),
66 + json={
67 + "model": model_name,
68 + "input": input_texts
69 + },
70 + )
71 + output = ClassificationResponse.model_validate(
72 + classification_response.json())
73 +
74 + assert len(output.data) == len(input_texts)
75 + for i, item in enumerate(output.data):
76 + assert item.index == i
77 + assert hasattr(item, "label")
78 + assert hasattr(item, "probs")
79 + assert len(item.probs) == item.num_classes
80 + assert item.label in ["Default", "Spoiled"]
81 +
82 +
83 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
84 +def test_truncate_prompt_tokens(server: RemoteOpenAIServer, model_name: str):
85 + long_text = "hello " * 600
86 +
87 + classification_response = requests.post(
88 + server.url_for("classify"),
89 + json={
90 + "model": model_name,
91 + "input": long_text,
92 + "truncate_prompt_tokens": 5
93 + },
94 + )
95 +
96 + classification_response.raise_for_status()
97 + output = ClassificationResponse.model_validate(
98 + classification_response.json())
99 +
100 + assert len(output.data) == 1
101 + assert output.data[0].index == 0
102 + assert hasattr(output.data[0], "probs")
103 + assert output.usage.prompt_tokens == 5
104 + assert output.usage.total_tokens == 5
105 +
106 +
107 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
108 +def test_invalid_truncate_prompt_tokens_error(server: RemoteOpenAIServer,
109 + model_name: str):
110 + classification_response = requests.post(
111 + server.url_for("classify"),
112 + json={
113 + "model": model_name,
114 + "input": "test",
115 + "truncate_prompt_tokens": 513
116 + },
117 + )
118 +
119 + error = classification_response.json()
120 + assert classification_response.status_code == 400
121 + assert error["object"] == "error"
122 + assert "truncate_prompt_tokens" in error["message"]
123 +
124 +
125 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
126 +def test_empty_input_error(server: RemoteOpenAIServer, model_name: str):
127 + classification_response = requests.post(
128 + server.url_for("classify"),
129 + json={
130 + "model": model_name,
131 + "input": ""
132 + },
133 + )
134 +
135 + error = classification_response.json()
136 + assert classification_response.status_code == 400
137 + assert error["object"] == "error"
138 +
139 +
140 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
141 +def test_batch_classification_empty_list(server: RemoteOpenAIServer,
142 + model_name: str):
143 + classification_response = requests.post(
144 + server.url_for("classify"),
145 + json={
146 + "model": model_name,
147 + "input": []
148 + },
149 + )
150 + classification_response.raise_for_status()
151 + output = ClassificationResponse.model_validate(
152 + classification_response.json())
153 +
154 + assert output.object == "list"
155 + assert isinstance(output.data, list)
156 + assert len(output.data) == 0
vllm/entrypoints/openai/api_server.py
+36
@@ -48,6 +48,8 @@ from vllm.entrypoints.openai.cli_args import (log_non_default_args,
48 # yapf: disable
49 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
50 ChatCompletionResponse,
51 + ClassificationRequest,
52 + ClassificationResponse,
53 CompletionRequest,
54 CompletionResponse,
55 DetokenizeRequest,
@@ -71,6 +73,8 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
73 UnloadLoRAAdapterRequest)
74 # yapf: enable
75 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
76 +from vllm.entrypoints.openai.serving_classification import (
77 + ServingClassification)
78 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
79 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
80 from vllm.entrypoints.openai.serving_engine import OpenAIServing
@@ -373,6 +377,10 @@ def score(request: Request) -> Optional[ServingScores]:
377 return request.app.state.openai_serving_scores
378
379
380 +def classify(request: Request) -> Optional[ServingClassification]:
381 + return request.app.state.openai_serving_classification
382 +
383 +
384 def rerank(request: Request) -> Optional[ServingScores]:
385 return request.app.state.openai_serving_scores
386
@@ -405,6 +413,7 @@ async def get_server_load_metrics(request: Request):
413 # - /v1/audio/transcriptions
414 # - /v1/embeddings
415 # - /pooling
416 + # - /classify
417 # - /score
418 # - /v1/score
419 # - /rerank
@@ -572,6 +581,27 @@ async def create_pooling(request: PoolingRequest, raw_request: Request):
581 assert_never(generator)
582
583
584 +@router.post("/classify", dependencies=[Depends(validate_json_request)])
585 +@with_cancellation
586 +@load_aware_call
587 +async def create_classify(request: ClassificationRequest,
588 + raw_request: Request):
589 + handler = classify(raw_request)
590 + if handler is None:
591 + return base(raw_request).create_error_response(
592 + message="The model does not support Classification API")
593 +
594 + generator = await handler.create_classify(request, raw_request)
595 + if isinstance(generator, ErrorResponse):
596 + return JSONResponse(content=generator.model_dump(),
597 + status_code=generator.code)
598 +
599 + elif isinstance(generator, ClassificationResponse):
600 + return JSONResponse(content=generator.model_dump())
601 +
602 + assert_never(generator)
603 +
604 +
605 @router.post("/score", dependencies=[Depends(validate_json_request)])
606 @with_cancellation
607 @load_aware_call
@@ -1001,6 +1031,12 @@ async def init_app_state(
1031 state.openai_serving_models,
1032 request_logger=request_logger) if model_config.task in (
1033 "score", "embed", "pooling") else None
1034 + state.openai_serving_classification = ServingClassification(
1035 + engine_client,
1036 + model_config,
1037 + state.openai_serving_models,
1038 + request_logger=request_logger,
1039 + ) if model_config.task == "classify" else None
1040 state.jinaai_serving_reranking = ServingScores(
1041 engine_client,
1042 model_config,
vllm/entrypoints/openai/protocol.py
+41
@@ -1292,6 +1292,47 @@ class ScoreResponse(OpenAIBaseModel):
1292 usage: UsageInfo
1293
1294
1295 +class ClassificationRequest(OpenAIBaseModel):
1296 + model: Optional[str] = None
1297 + input: Union[list[str], str]
1298 + truncate_prompt_tokens: Optional[int] = None
1299 + user: Optional[str] = None
1300 +
1301 + # doc: begin-classification-pooling-params
1302 + additional_data: Optional[Any] = None
1303 + # doc: end-classification-pooling-params
1304 +
1305 + # doc: begin-classification-extra-params
1306 + priority: int = Field(
1307 + default=0,
1308 + description=(
1309 + "The priority of the request (lower means earlier handling; "
1310 + "default: 0). Any priority other than 0 will raise an error "
1311 + "if the served model does not use priority scheduling."),
1312 + )
1313 +
1314 + # doc: end-classification-extra-params
1315 +
1316 + def to_pooling_params(self):
1317 + return PoolingParams(additional_data=self.additional_data)
1318 +
1319 +
1320 +class ClassificationData(OpenAIBaseModel):
1321 + index: int
1322 + label: Optional[str]
1323 + probs: list[float]
1324 + num_classes: int
1325 +
1326 +
1327 +class ClassificationResponse(OpenAIBaseModel):
1328 + id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
1329 + object: str = "list"
1330 + created: int = Field(default_factory=lambda: int(time.time()))
1331 + model: str
1332 + data: list[ClassificationData]
1333 + usage: UsageInfo
1334 +
1335 +
1336 class FunctionCall(OpenAIBaseModel):
1337 name: str
1338 arguments: str
vllm/entrypoints/openai/serving_classification.py new
+159
@@ -0,0 +1,159 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from http import HTTPStatus
4 +from typing import Optional, Union, cast
5 +
6 +import numpy as np
7 +from fastapi import Request
8 +
9 +from vllm.config import ModelConfig
10 +from vllm.engine.protocol import EngineClient
11 +from vllm.entrypoints.logger import RequestLogger
12 +from vllm.entrypoints.openai.protocol import (ClassificationData,
13 + ClassificationRequest,
14 + ClassificationResponse,
15 + ErrorResponse, UsageInfo)
16 +# yapf: enable
17 +from vllm.entrypoints.openai.serving_engine import (ClassificationServeContext,
18 + OpenAIServing,
19 + ServeContext)
20 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
21 +from vllm.logger import init_logger
22 +from vllm.outputs import ClassificationOutput, PoolingRequestOutput
23 +
24 +logger = init_logger(__name__)
25 +
26 +
27 +class ClassificationMixin(OpenAIServing):
28 +
29 + async def _preprocess(
30 + self,
31 + ctx: ServeContext,
32 + ) -> Optional[ErrorResponse]:
33 + """
34 + Process classification inputs: tokenize text, resolve adapters,
35 + and prepare model-specific inputs.
36 + """
37 + ctx = cast(ClassificationServeContext, ctx)
38 + if isinstance(ctx.request.input, str) and not ctx.request.input:
39 + return self.create_error_response(
40 + "Input cannot be empty for classification",
41 + status_code=HTTPStatus.BAD_REQUEST,
42 + )
43 +
44 + if isinstance(ctx.request.input, list) and len(ctx.request.input) == 0:
45 + return None
46 +
47 + try:
48 + (
49 + ctx.lora_request,
50 + ctx.prompt_adapter_request,
51 + ) = self._maybe_get_adapters(ctx.request)
52 +
53 + ctx.tokenizer = await self.engine_client.get_tokenizer(
54 + ctx.lora_request)
55 +
56 + if ctx.prompt_adapter_request is not None:
57 + raise NotImplementedError(
58 + "Prompt adapter is not supported for classification models"
59 + )
60 +
61 + (
62 + ctx.request_prompts,
63 + ctx.engine_prompts,
64 + ) = await self._preprocess_completion(
65 + ctx.request,
66 + ctx.tokenizer,
67 + ctx.request.input,
68 + truncate_prompt_tokens=ctx.request.truncate_prompt_tokens,
69 + )
70 +
71 + return None
72 +
73 + except (ValueError, TypeError) as e:
74 + logger.exception("Error in preprocessing prompt inputs")
75 + return self.create_error_response(str(e))
76 +
77 + def _build_response(
78 + self,
79 + ctx: ServeContext,
80 + ) -> Union[ClassificationResponse, ErrorResponse]:
81 + """
82 + Convert model outputs to a formatted classification response
83 + with probabilities and labels.
84 + """
85 + ctx = cast(ClassificationServeContext, ctx)
86 + items: list[ClassificationData] = []
87 + num_prompt_tokens = 0
88 +
89 + final_res_batch_checked = cast(list[PoolingRequestOutput],
90 + ctx.final_res_batch)
91 +
92 + for idx, final_res in enumerate(final_res_batch_checked):
93 + classify_res = ClassificationOutput.from_base(final_res.outputs)
94 +
95 + probs = classify_res.probs
96 + predicted_index = int(np.argmax(probs))
97 + label = getattr(self.model_config.hf_config, "id2label",
98 + {}).get(predicted_index)
99 +
100 + item = ClassificationData(
101 + index=idx,
102 + label=label,
103 + probs=probs,
104 + num_classes=len(probs),
105 + )
106 +
107 + items.append(item)
108 + prompt_token_ids = final_res.prompt_token_ids
109 + num_prompt_tokens += len(prompt_token_ids)
110 +
111 + usage = UsageInfo(
112 + prompt_tokens=num_prompt_tokens,
113 + total_tokens=num_prompt_tokens,
114 + )
115 +
116 + return ClassificationResponse(
117 + id=ctx.request_id,
118 + created=ctx.created_time,
119 + model=ctx.model_name,
120 + data=items,
121 + usage=usage,
122 + )
123 +
124 +
125 +class ServingClassification(ClassificationMixin):
126 + request_id_prefix = "classify"
127 +
128 + def __init__(
129 + self,
130 + engine_client: EngineClient,
131 + model_config: ModelConfig,
132 + models: OpenAIServingModels,
133 + *,
134 + request_logger: Optional[RequestLogger],
135 + ) -> None:
136 + super().__init__(
137 + engine_client=engine_client,
138 + model_config=model_config,
139 + models=models,
140 + request_logger=request_logger,
141 + )
142 +
143 + async def create_classify(
144 + self,
145 + request: ClassificationRequest,
146 + raw_request: Request,
147 + ) -> Union[ClassificationResponse, ErrorResponse]:
148 + model_name = self._get_model_name(request.model)
149 + request_id = (f"{self.request_id_prefix}-"
150 + f"{self._base_request_id(raw_request)}")
151 +
152 + ctx = ClassificationServeContext(
153 + request=request,
154 + raw_request=raw_request,
155 + model_name=model_name,
156 + request_id=request_id,
157 + )
158 +
159 + return await super().handle(ctx) # type: ignore
vllm/entrypoints/openai/serving_embedding.py
+110 -149
@@ -1,14 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -import asyncio
3 import base64
5 -import time
6 -from collections.abc import AsyncGenerator
4 from typing import Final, Literal, Optional, Union, cast
5
6 import numpy as np
7 from fastapi import Request
11 -from typing_extensions import assert_never
8 +from typing_extensions import assert_never, override
9
10 from vllm.config import ModelConfig
11 from vllm.engine.protocol import EngineClient
@@ -19,13 +16,13 @@ from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
16 EmbeddingResponse,
17 EmbeddingResponseData,
18 ErrorResponse, UsageInfo)
22 -from vllm.entrypoints.openai.serving_engine import OpenAIServing
19 +from vllm.entrypoints.openai.serving_engine import (EmbeddingServeContext,
20 + OpenAIServing,
21 + ServeContext)
22 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
24 -from vllm.entrypoints.utils import _validate_truncation_size
23 from vllm.logger import init_logger
24 from vllm.outputs import (EmbeddingOutput, EmbeddingRequestOutput,
25 PoolingRequestOutput)
28 -from vllm.utils import merge_async_iterators
26
27 logger = init_logger(__name__)
28
@@ -45,180 +42,77 @@ def _get_embedding(
42 assert_never(encoding_format)
43
44
48 -class OpenAIServingEmbedding(OpenAIServing):
45 +class EmbeddingMixin(OpenAIServing):
46
50 - def __init__(
51 - self,
52 - engine_client: EngineClient,
53 - model_config: ModelConfig,
54 - models: OpenAIServingModels,
55 - *,
56 - request_logger: Optional[RequestLogger],
57 - chat_template: Optional[str],
58 - chat_template_content_format: ChatTemplateContentFormatOption,
59 - ) -> None:
60 - super().__init__(engine_client=engine_client,
61 - model_config=model_config,
62 - models=models,
63 - request_logger=request_logger)
64 -
65 - self.chat_template = chat_template
66 - self.chat_template_content_format: Final = chat_template_content_format
67 -
68 - async def create_embedding(
47 + async def _preprocess(
48 self,
70 - request: EmbeddingRequest,
71 - raw_request: Optional[Request] = None,
72 - ) -> Union[EmbeddingResponse, ErrorResponse]:
73 - """
74 - Embedding API similar to OpenAI's API.
75 -
76 - See https://platform.openai.com/docs/api-reference/embeddings/create
77 - for the API specification. This API mimics the OpenAI Embedding API.
78 - """
79 - error_check_ret = await self._check_model(request)
80 - if error_check_ret is not None:
81 - return error_check_ret
82 -
83 - encoding_format = request.encoding_format
84 -
85 - model_name = self._get_model_name(request.model)
86 - request_id = f"embd-{self._base_request_id(raw_request)}"
87 - created_time = int(time.time())
88 -
89 - truncate_prompt_tokens = request.truncate_prompt_tokens
90 -
91 - pooling_params = request.to_pooling_params()
92 -
93 - try:
94 - pooling_params.verify(self.model_config)
95 - except ValueError as e:
96 - return self.create_error_response(str(e))
97 -
49 + ctx: ServeContext,
50 + ) -> Optional[ErrorResponse]:
51 + ctx = cast(EmbeddingServeContext, ctx)
52 try:
99 - truncate_prompt_tokens = _validate_truncation_size(
100 - self.max_model_len, truncate_prompt_tokens)
53 (
102 - lora_request,
103 - prompt_adapter_request,
104 - ) = self._maybe_get_adapters(request)
54 + ctx.lora_request,
55 + ctx.prompt_adapter_request,
56 + ) = self._maybe_get_adapters(ctx.request)
57
106 - tokenizer = await self.engine_client.get_tokenizer(lora_request)
58 + tokenizer = await self.engine_client.get_tokenizer(ctx.lora_request
59 + )
60
108 - if prompt_adapter_request is not None:
61 + if ctx.prompt_adapter_request is not None:
62 raise NotImplementedError("Prompt adapter is not supported "
63 "for embedding models")
64
112 - if isinstance(request, EmbeddingChatRequest):
65 + if isinstance(ctx.request, EmbeddingChatRequest):
66 (
67 _,
115 - request_prompts,
116 - engine_prompts,
68 + ctx.request_prompts,
69 + ctx.engine_prompts,
70 ) = await self._preprocess_chat(
118 - request,
71 + ctx.request,
72 tokenizer,
120 - request.messages,
121 - chat_template=request.chat_template or self.chat_template,
122 - chat_template_content_format=self.
73 + ctx.request.messages,
74 + chat_template=ctx.request.chat_template
75 + or ctx.chat_template,
76 + chat_template_content_format=ctx.
77 chat_template_content_format,
78 # In embedding requests, we are not generating tokens,
79 # so there is no need to append extra tokens to the input
80 add_generation_prompt=False,
81 continue_final_message=False,
128 - truncate_prompt_tokens=truncate_prompt_tokens,
129 - add_special_tokens=request.add_special_tokens,
82 + truncate_prompt_tokens=ctx.truncate_prompt_tokens,
83 + add_special_tokens=ctx.request.add_special_tokens,
84 )
85 else:
132 - (request_prompts,
133 - engine_prompts) = await self._preprocess_completion(
134 - request,
86 + (ctx.request_prompts,
87 + ctx.engine_prompts) = await self._preprocess_completion(
88 + ctx.request,
89 tokenizer,
136 - request.input,
137 - truncate_prompt_tokens=truncate_prompt_tokens,
138 - add_special_tokens=request.add_special_tokens,
90 + ctx.request.input,
91 + truncate_prompt_tokens=ctx.truncate_prompt_tokens,
92 + add_special_tokens=ctx.request.add_special_tokens,
93 )
94 + return None
95 except (ValueError, TypeError) as e:
96 logger.exception("Error in preprocessing prompt inputs")
97 return self.create_error_response(str(e))
98
144 - # Schedule the request and get the result generator.
145 - generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
146 - try:
147 - for i, engine_prompt in enumerate(engine_prompts):
148 - request_id_item = f"{request_id}-{i}"
149 -
150 - self._log_inputs(request_id_item,
151 - request_prompts[i],
152 - params=pooling_params,
153 - lora_request=lora_request,
154 - prompt_adapter_request=prompt_adapter_request)
155 -
156 - trace_headers = (None if raw_request is None else await
157 - self._get_trace_headers(raw_request.headers))
158 -
159 - generator = self.engine_client.encode(
160 - engine_prompt,
161 - pooling_params,
162 - request_id_item,
163 - lora_request=lora_request,
164 - trace_headers=trace_headers,
165 - priority=request.priority,
166 - )
167 -
168 - generators.append(generator)
169 - except ValueError as e:
170 - # TODO: Use a vllm-specific Validation Error
171 - return self.create_error_response(str(e))
172 -
173 - result_generator = merge_async_iterators(*generators)
174 -
175 - num_prompts = len(engine_prompts)
176 -
177 - # Non-streaming response
178 - final_res_batch: list[Optional[PoolingRequestOutput]]
179 - final_res_batch = [None] * num_prompts
180 - try:
181 - async for i, res in result_generator:
182 - final_res_batch[i] = res
183 -
184 - assert all(final_res is not None for final_res in final_res_batch)
185 -
186 - final_res_batch_checked = cast(list[PoolingRequestOutput],
187 - final_res_batch)
188 -
189 - response = self.request_output_to_embedding_response(
190 - final_res_batch_checked,
191 - request_id,
192 - created_time,
193 - model_name,
194 - encoding_format,
195 - )
196 - except asyncio.CancelledError:
197 - return self.create_error_response("Client disconnected")
198 - except ValueError as e:
199 - # TODO: Use a vllm-specific Validation Error
200 - return self.create_error_response(str(e))
201 -
202 - return response
203 -
204 - def request_output_to_embedding_response(
99 + def _build_response(
100 self,
206 - final_res_batch: list[PoolingRequestOutput],
207 - request_id: str,
208 - created_time: int,
209 - model_name: str,
210 - encoding_format: Literal["float", "base64"],
211 - ) -> EmbeddingResponse:
101 + ctx: ServeContext,
102 + ) -> Union[EmbeddingResponse, ErrorResponse]:
103 items: list[EmbeddingResponseData] = []
104 num_prompt_tokens = 0
105
215 - for idx, final_res in enumerate(final_res_batch):
106 + final_res_batch_checked = cast(list[PoolingRequestOutput],
107 + ctx.final_res_batch)
108 +
109 + for idx, final_res in enumerate(final_res_batch_checked):
110 embedding_res = EmbeddingRequestOutput.from_base(final_res)
111
112 item = EmbeddingResponseData(
113 index=idx,
114 embedding=_get_embedding(embedding_res.outputs,
221 - encoding_format),
115 + ctx.request.encoding_format),
116 )
117 prompt_token_ids = final_res.prompt_token_ids
118
@@ -231,9 +125,76 @@ class OpenAIServingEmbedding(OpenAIServing):
125 )
126
127 return EmbeddingResponse(
234 - id=request_id,
235 - created=created_time,
236 - model=model_name,
128 + id=ctx.request_id,
129 + created=ctx.created_time,
130 + model=ctx.model_name,
131 data=items,
132 usage=usage,
133 )
134 +
135 +
136 +class OpenAIServingEmbedding(EmbeddingMixin):
137 + request_id_prefix = "embd"
138 +
139 + def __init__(
140 + self,
141 + engine_client: EngineClient,
142 + model_config: ModelConfig,
143 + models: OpenAIServingModels,
144 + *,
145 + request_logger: Optional[RequestLogger],
146 + chat_template: Optional[str],
147 + chat_template_content_format: ChatTemplateContentFormatOption,
148 + ) -> None:
149 + super().__init__(engine_client=engine_client,
150 + model_config=model_config,
151 + models=models,
152 + request_logger=request_logger)
153 +
154 + self.chat_template = chat_template
155 + self.chat_template_content_format: Final = chat_template_content_format
156 +
157 + async def create_embedding(
158 + self,
159 + request: EmbeddingRequest,
160 + raw_request: Optional[Request] = None,
161 + ) -> Union[EmbeddingResponse, ErrorResponse]:
162 + """
163 + Embedding API similar to OpenAI's API.
164 +
165 + See https://platform.openai.com/docs/api-reference/embeddings/create
166 + for the API specification. This API mimics the OpenAI Embedding API.
167 + """
168 + model_name = self._get_model_name(request.model)
169 + request_id = (f"{self.request_id_prefix}-"
170 + f"{self._base_request_id(raw_request)}")
171 +
172 + ctx = EmbeddingServeContext(
173 + request=request,
174 + raw_request=raw_request,
175 + model_name=model_name,
176 + request_id=request_id,
177 + chat_template=self.chat_template,
178 + chat_template_content_format=self.chat_template_content_format,
179 + )
180 +
181 + return await super().handle(ctx) # type: ignore
182 +
183 + @override
184 + def _validate_request(
185 + self,
186 + ctx: ServeContext[EmbeddingRequest],
187 + ) -> Optional[ErrorResponse]:
188 + if error := super()._validate_request(ctx):
189 + return error
190 +
191 + ctx.truncate_prompt_tokens = ctx.request.truncate_prompt_tokens
192 +
193 + pooling_params = ctx.request.to_pooling_params()
194 +
195 + try:
196 + pooling_params.verify(self.model_config)
197 + except ValueError as e:
198 + return self.create_error_response(str(e))
199 +
200 + return None
vllm/entrypoints/openai/serving_engine.py
+283 -13
@@ -1,13 +1,16 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import json
4 -from collections.abc import Iterable, Iterator, Mapping, Sequence
4 +import time
5 +from collections.abc import (AsyncGenerator, Iterable, Iterator, Mapping,
6 + Sequence)
7 from concurrent.futures.thread import ThreadPoolExecutor
8 from http import HTTPStatus
7 -from typing import Annotated, Any, Callable, Optional, TypedDict, Union
9 +from typing import (Annotated, Any, Callable, ClassVar, Generic, Optional,
10 + TypedDict, TypeVar, Union)
11
12 from fastapi import Request
10 -from pydantic import Field
13 +from pydantic import BaseModel, ConfigDict, Field
14 from starlette.datastructures import Headers
15
16 import vllm.envs as envs
@@ -24,15 +27,23 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
27 resolve_chat_template_content_format)
28 from vllm.entrypoints.logger import RequestLogger
29 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
30 + ChatCompletionResponse,
31 + ClassificationRequest,
32 + ClassificationResponse,
33 CompletionRequest,
34 + CompletionResponse,
35 DetokenizeRequest,
36 EmbeddingChatRequest,
37 EmbeddingCompletionRequest,
31 - ErrorResponse, RerankRequest,
32 - ScoreRequest,
38 + EmbeddingRequest,
39 + EmbeddingResponse, ErrorResponse,
40 + PoolingResponse, RerankRequest,
41 + ScoreRequest, ScoreResponse,
42 TokenizeChatRequest,
43 TokenizeCompletionRequest,
35 - TranscriptionRequest)
44 + TokenizeResponse,
45 + TranscriptionRequest,
46 + TranscriptionResponse)
47 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
48 from vllm.entrypoints.openai.tool_parsers import ToolParser
49 # yapf: enable
@@ -40,6 +51,9 @@ from vllm.inputs import TokensPrompt
51 from vllm.inputs.parse import parse_and_batch_prompt
52 from vllm.logger import init_logger
53 from vllm.lora.request import LoRARequest
54 +from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin
55 + MultiModalDataDict)
56 +from vllm.outputs import PoolingRequestOutput, RequestOutput
57 from vllm.pooling_params import PoolingParams
58 from vllm.prompt_adapter.request import PromptAdapterRequest
59 from vllm.sampling_params import BeamSearchParams, SamplingParams
@@ -47,13 +61,15 @@ from vllm.sequence import Logprob, PromptLogprobs
61 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
62 log_tracing_disabled_warning)
63 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
50 -from vllm.utils import is_list_of, make_async, random_uuid
64 +from vllm.utils import (is_list_of, make_async, merge_async_iterators,
65 + random_uuid)
66
67 logger = init_logger(__name__)
68
69 CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
70 EmbeddingCompletionRequest, RerankRequest,
56 - ScoreRequest, TokenizeCompletionRequest]
71 + ClassificationRequest, ScoreRequest,
72 + TokenizeCompletionRequest]
73
74 ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
75 TokenizeChatRequest]
@@ -61,6 +77,17 @@ ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
77 AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest,
78 TranscriptionRequest]
79
80 +AnyResponse = Union[
81 + CompletionResponse,
82 + ChatCompletionResponse,
83 + EmbeddingResponse,
84 + TranscriptionResponse,
85 + TokenizeResponse,
86 + PoolingResponse,
87 + ClassificationResponse,
88 + ScoreResponse,
89 +]
90 +
91
92 class TextTokensPrompt(TypedDict):
93 prompt: str
@@ -69,8 +96,79 @@ class TextTokensPrompt(TypedDict):
96
97 RequestPrompt = Union[list[int], str, TextTokensPrompt]
98
99 +RequestT = TypeVar("RequestT", bound=AnyRequest)
100 +
101 +
102 +class RequestProcessingMixin(BaseModel):
103 + """
104 + Mixin for request processing,
105 + handling prompt preparation and engine input.
106 + """
107 + request_prompts: Optional[Sequence[RequestPrompt]] = \
108 + Field(default_factory=list)
109 + engine_prompts: Optional[list[TokensPrompt]] = \
110 + Field(default_factory=list)
111 +
112 + model_config = ConfigDict(arbitrary_types_allowed=True)
113 +
114 +
115 +class ResponseGenerationMixin(BaseModel):
116 + """
117 + Mixin for response generation,
118 + managing result generators and final batch results.
119 + """
120 + result_generator: Optional[AsyncGenerator[tuple[int, Union[
121 + RequestOutput, PoolingRequestOutput]], None]] = None
122 + final_res_batch: list[Union[RequestOutput, PoolingRequestOutput]] = Field(
123 + default_factory=list)
124 +
125 + model_config = ConfigDict(arbitrary_types_allowed=True)
126 +
127 +
128 +class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, BaseModel,
129 + Generic[RequestT]):
130 + # Shared across all requests
131 + request: RequestT
132 + raw_request: Optional[Request] = None
133 + model_name: str
134 + request_id: str
135 + created_time: int = Field(default_factory=lambda: int(time.time()))
136 + lora_request: Optional[LoRARequest] = None
137 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
138 +
139 + # Shared across most requests
140 + tokenizer: Optional[AnyTokenizer] = None
141 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
142 +
143 + # `protected_namespaces` resolves Pydantic v2's warning
144 + # on conflict with protected namespace "model_"
145 + model_config = ConfigDict(
146 + protected_namespaces=(),
147 + arbitrary_types_allowed=True,
148 + )
149 +
150 +
151 +ClassificationServeContext = ServeContext[ClassificationRequest]
152 +
153 +
154 +class EmbeddingServeContext(ServeContext[EmbeddingRequest]):
155 + chat_template: Optional[str] = None
156 + chat_template_content_format: ChatTemplateContentFormatOption
157 +
158 +
159 +# Used to resolve the Pydantic error related to
160 +# forward reference of MultiModalDataDict in TokensPrompt
161 +RequestProcessingMixin.model_rebuild()
162 +ServeContext.model_rebuild()
163 +ClassificationServeContext.model_rebuild()
164 +EmbeddingServeContext.model_rebuild()
165 +
166
167 class OpenAIServing:
168 + request_id_prefix: ClassVar[str] = """
169 + A short string prepended to every request’s ID (e.g. "embd", "classify")
170 + so you can easily tell “this ID came from Embedding vs Classification.”
171 + """
172
173 def __init__(
174 self,
@@ -100,6 +198,167 @@ class OpenAIServing:
198 self._tokenize_prompt_input_or_inputs,
199 executor=self._tokenizer_executor)
200
201 + async def _preprocess(
202 + self,
203 + ctx: ServeContext,
204 + ) -> Optional[ErrorResponse]:
205 + """
206 + Default preprocessing hook. Subclasses may override
207 + to prepare `ctx` (classification, embedding, etc.).
208 + """
209 + return None
210 +
211 + def _build_response(
212 + self,
213 + ctx: ServeContext,
214 + ) -> Union[AnyResponse, ErrorResponse]:
215 + """
216 + Default response builder. Subclass may override this method
217 + to return the appropriate response object.
218 + """
219 + return self.create_error_response("unimplemented endpoint")
220 +
221 + async def handle(
222 + self,
223 + ctx: ServeContext,
224 + ) -> Union[AnyResponse, ErrorResponse]:
225 + generation: AsyncGenerator[Union[AnyResponse, ErrorResponse], None]
226 + generation = self._pipeline(ctx)
227 +
228 + async for response in generation:
229 + return response
230 +
231 + return self.create_error_response("No response yielded from pipeline")
232 +
233 + async def _pipeline(
234 + self,
235 + ctx: ServeContext,
236 + ) -> AsyncGenerator[Union[AnyResponse, ErrorResponse], None]:
237 + """Execute the request processing pipeline yielding responses."""
238 + if error := await self._check_model(ctx.request):
239 + yield error
240 + if error := self._validate_request(ctx):
241 + yield error
242 +
243 + preprocess_ret = await self._preprocess(ctx)
244 + if isinstance(preprocess_ret, ErrorResponse):
245 + yield preprocess_ret
246 +
247 + generators_ret = await self._prepare_generators(ctx)
248 + if isinstance(generators_ret, ErrorResponse):
249 + yield generators_ret
250 +
251 + collect_ret = await self._collect_batch(ctx)
252 + if isinstance(collect_ret, ErrorResponse):
253 + yield collect_ret
254 +
255 + yield self._build_response(ctx)
256 +
257 + def _validate_request(self, ctx: ServeContext) -> Optional[ErrorResponse]:
258 + truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens",
259 + None)
260 +
261 + if truncate_prompt_tokens is not None:
262 + if truncate_prompt_tokens <= self.max_model_len:
263 + ctx.truncate_prompt_tokens = truncate_prompt_tokens
264 + else:
265 + return self.create_error_response(
266 + "truncate_prompt_tokens value is "
267 + "greater than max_model_len."
268 + " Please, select a smaller truncation size.")
269 + return None
270 +
271 + async def _prepare_generators(
272 + self,
273 + ctx: ServeContext,
274 + ) -> Optional[ErrorResponse]:
275 + """Schedule the request and get the result generator."""
276 + generators: list[AsyncGenerator[Union[RequestOutput,
277 + PoolingRequestOutput],
278 + None]] = []
279 +
280 + try:
281 + trace_headers = (None if ctx.raw_request is None else await
282 + self._get_trace_headers(ctx.raw_request.headers))
283 +
284 + if not hasattr(ctx.request, "to_pooling_params"):
285 + return self.create_error_response(
286 + "Request type does not support pooling parameters")
287 +
288 + pooling_params = ctx.request.to_pooling_params()
289 +
290 + if ctx.engine_prompts is None:
291 + return self.create_error_response(
292 + "Engine prompts not available")
293 +
294 + for i, engine_prompt in enumerate(ctx.engine_prompts):
295 + request_id_item = f"{ctx.request_id}-{i}"
296 +
297 + if ctx.request_prompts is None:
298 + return self.create_error_response(
299 + "Request prompts not available")
300 +
301 + self._log_inputs(
302 + request_id_item,
303 + ctx.request_prompts[i],
304 + params=pooling_params,
305 + lora_request=ctx.lora_request,
306 + prompt_adapter_request=ctx.prompt_adapter_request)
307 +
308 + generator = self.engine_client.encode(
309 + engine_prompt,
310 + pooling_params,
311 + request_id_item,
312 + lora_request=ctx.lora_request,
313 + trace_headers=trace_headers,
314 + priority=getattr(ctx.request, "priority", 0),
315 + )
316 +
317 + generators.append(generator)
318 +
319 + ctx.result_generator = merge_async_iterators(*generators)
320 +
321 + return None
322 +
323 + except Exception as e:
324 + # TODO: Use a vllm-specific Validation Error
325 + return self.create_error_response(str(e))
326 +
327 + async def _collect_batch(
328 + self,
329 + ctx: ServeContext,
330 + ) -> Optional[ErrorResponse]:
331 + """Collect batch results from the result generator."""
332 + try:
333 + if ctx.engine_prompts is None:
334 + return self.create_error_response(
335 + "Engine prompts not available")
336 +
337 + num_prompts = len(ctx.engine_prompts)
338 + final_res_batch: list[Optional[Union[RequestOutput,
339 + PoolingRequestOutput]]]
340 + final_res_batch = [None] * num_prompts
341 +
342 + if ctx.result_generator is None:
343 + return self.create_error_response(
344 + "Result generator not available")
345 +
346 + async for i, res in ctx.result_generator:
347 + final_res_batch[i] = res
348 +
349 + if None in final_res_batch:
350 + return self.create_error_response(
351 + "Failed to generate results for all prompts")
352 +
353 + ctx.final_res_batch = [
354 + res for res in final_res_batch if res is not None
355 + ]
356 +
357 + return None
358 +
359 + except Exception as e:
360 + return self.create_error_response(str(e))
361 +
362 def create_error_response(
363 self,
364 message: str,
@@ -183,6 +442,12 @@ class OpenAIServing:
442
443 if truncate_prompt_tokens is None:
444 encoded = tokenizer(prompt, add_special_tokens=add_special_tokens)
445 + elif truncate_prompt_tokens < 0:
446 + # Negative means we cap at the model's max length
447 + encoded = tokenizer(prompt,
448 + add_special_tokens=add_special_tokens,
449 + truncation=True,
450 + max_length=self.max_model_len)
451 else:
452 encoded = tokenizer(prompt,
453 add_special_tokens=add_special_tokens,
@@ -204,6 +469,8 @@ class OpenAIServing:
469 ) -> TextTokensPrompt:
470 if truncate_prompt_tokens is None:
471 input_ids = prompt_ids
472 + elif truncate_prompt_tokens < 0:
473 + input_ids = prompt_ids[-self.max_model_len:]
474 else:
475 input_ids = prompt_ids[-truncate_prompt_tokens:]
476
@@ -219,13 +486,16 @@ class OpenAIServing:
486 ) -> TextTokensPrompt:
487 token_num = len(input_ids)
488
222 - # Note: EmbeddingRequest and ScoreRequest doesn't have max_tokens
489 + # Note: EmbeddingRequest, ClassificationRequest,
490 + # and ScoreRequest doesn't have max_tokens
491 if isinstance(request,
492 (EmbeddingChatRequest, EmbeddingCompletionRequest,
225 - ScoreRequest, RerankRequest)):
493 + ScoreRequest, RerankRequest, ClassificationRequest)):
494 + operation = {
495 + ScoreRequest: "score",
496 + ClassificationRequest: "classification"
497 + }.get(type(request), "embedding generation")
498
227 - operation = "score" if isinstance(request, ScoreRequest) \
228 - else "embedding generation"
499 if token_num > self.max_model_len:
500 raise ValueError(
501 f"This model's maximum context length is "
@@ -247,7 +517,7 @@ class OpenAIServing:
517 # TODO(#9845): remove max_tokens when field dropped from OpenAI API
518 max_tokens = request.max_completion_tokens or request.max_tokens
519 else:
250 - max_tokens = request.max_tokens
520 + max_tokens = getattr(request, "max_tokens", None)
521 if max_tokens is None:
522 if token_num >= self.max_model_len:
523 raise ValueError(