A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 02:30 UTC
[Chore] Remove redundant input parsing methods (#33542)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 2, 2026 at 18:50 UTC
a502831d36426ed3e0eee99ba4bb03490734381b
7 files changed
+21
-51
tests/renderers/test_completions.py
+1
-4
@@ -10,7 +10,6 @@ import pybase64
10
import pytest
11
import torch
12
13
-from vllm.inputs.data import is_embeds_prompt
13
from vllm.renderers import TokenizeParams
14
from vllm.renderers.hf import HfRenderer
15
from vllm.tokenizers.registry import tokenizer_args_from_config
@@ -320,7 +319,6 @@ class TestRenderEmbedPrompt:
319
)
320
321
assert len(results) == 1
323
- assert is_embeds_prompt(results[0])
322
assert torch.allclose(results[0]["prompt_embeds"], test_tensor)
323
324
@pytest.mark.asyncio
@@ -342,7 +340,6 @@ class TestRenderEmbedPrompt:
340
341
assert len(results) == 2
342
for i, result in enumerate(results):
345
- assert is_embeds_prompt(result)
343
assert torch.allclose(result["prompt_embeds"], test_tensors[i])
344
345
@pytest.mark.asyncio
@@ -420,7 +417,7 @@ class TestRenderEmbedPrompt:
417
418
assert len(results) == 2
419
# First should be embed prompt
423
- assert is_embeds_prompt(results[0])
420
+ assert torch.allclose(results[0]["prompt_embeds"], test_tensor)
421
# Second should be tokens prompt
422
assert "prompt_token_ids" in results[1]
423
assert results[1]["prompt_token_ids"] == [101, 102, 103]
vllm/entrypoints/openai/chat_completion/serving.py
+1
-2
@@ -68,7 +68,6 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
68
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
69
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
70
from vllm.inputs.data import EmbedsPrompt, TokensPrompt
71
-from vllm.inputs.parse import get_prompt_components
71
from vllm.logger import init_logger
72
from vllm.logprobs import Logprob
73
from vllm.outputs import CompletionOutput, RequestOutput
@@ -359,7 +358,7 @@ class OpenAIServingChat(OpenAIServing):
358
generators: list[AsyncGenerator[RequestOutput, None]] = []
359
try:
360
for i, engine_prompt in enumerate(engine_prompts):
362
- prompt_text, _, _ = get_prompt_components(engine_prompt)
361
+ prompt_text = engine_prompt.get("prompt")
362
363
# If we are creating sub requests for multiple prompts, ensure that they
364
# have unique request ids.
vllm/entrypoints/openai/completion/serving.py
+4
-13
@@ -34,8 +34,7 @@ from vllm.entrypoints.openai.engine.serving import (
34
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
35
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
36
from vllm.exceptions import VLLMValidationError
37
-from vllm.inputs.data import EmbedsPrompt, TokensPrompt, is_embeds_prompt
38
-from vllm.inputs.parse import get_prompt_components
37
+from vllm.inputs.data import EmbedsPrompt, TokensPrompt
38
from vllm.logger import init_logger
39
from vllm.logprobs import Logprob
40
from vllm.outputs import RequestOutput
@@ -161,7 +160,7 @@ class OpenAIServingCompletion(OpenAIServing):
160
generators: list[AsyncGenerator[RequestOutput, None]] = []
161
try:
162
for i, engine_prompt in enumerate(engine_prompts):
164
- prompt_text, _, _ = get_prompt_components(engine_prompt)
163
+ prompt_text = engine_prompt.get("prompt")
164
165
max_tokens = get_max_tokens(
166
max_model_len=self.max_model_len,
@@ -278,11 +277,7 @@ class OpenAIServingCompletion(OpenAIServing):
277
# with the inputs token IDs
278
if final_res.prompt is None:
279
engine_prompt = engine_prompts[i]
281
- final_res.prompt = (
282
- None
283
- if is_embeds_prompt(engine_prompt)
284
- else engine_prompt.get("prompt")
285
- )
280
+ final_res.prompt = engine_prompt.get("prompt")
281
282
final_res_batch_checked = cast(list[RequestOutput], final_res_batch)
283
@@ -352,11 +347,7 @@ class OpenAIServingCompletion(OpenAIServing):
347
prompt_text = res.prompt
348
if prompt_text is None:
349
engine_prompt = engine_prompts[prompt_idx]
355
- prompt_text = (
356
- None
357
- if is_embeds_prompt(engine_prompt)
358
- else engine_prompt.get("prompt")
359
- )
350
+ prompt_text = engine_prompt.get("prompt")
351
352
# Prompt details are excluded from later streamed outputs
353
if prompt_token_ids is not None:
vllm/entrypoints/openai/engine/serving.py
+2
-2
@@ -1116,7 +1116,7 @@ class OpenAIServing:
1116
priority: int = 0,
1117
trace_headers: Mapping[str, str] | None = None,
1118
):
1119
- prompt_text, _, _ = get_prompt_components(engine_prompt)
1119
+ prompt_text = engine_prompt.get("prompt")
1120
1121
orig_priority = priority
1122
sub_request = 0
@@ -1186,7 +1186,7 @@ class OpenAIServing:
1186
context.chat_template_content_format,
1187
)
1188
engine_prompt = engine_prompts[0]
1189
- prompt_text, _, _ = get_prompt_components(engine_prompt)
1189
+ prompt_text = engine_prompt.get("prompt")
1190
1191
sampling_params.max_tokens = get_max_tokens(
1192
self.max_model_len,
vllm/inputs/data.py
+4
-17
@@ -5,7 +5,7 @@ from dataclasses import dataclass
5
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, cast
6
7
import torch
8
-from typing_extensions import NotRequired, TypedDict, TypeIs, TypeVar
8
+from typing_extensions import NotRequired, TypedDict, TypeVar
9
10
from vllm.sampling_params import SamplingParams
11
@@ -77,6 +77,9 @@ class EmbedsPrompt(_CommonKeys):
77
prompt_embeds: torch.Tensor
78
"""The embeddings of the prompt."""
79
80
+ prompt: NotRequired[str]
81
+ """The prompt text corresponding to the token embeddings, if available."""
82
+
83
84
class DataPrompt(_CommonKeys):
85
"""Represents generic inputs handled by IO processor plugins."""
@@ -113,22 +116,6 @@ more than one prompt, i.e.
116
"""
117
118
116
-def is_tokens_prompt(prompt: SingletonPrompt) -> TypeIs[TokensPrompt]:
117
- return (
118
- isinstance(prompt, dict)
119
- and "prompt_token_ids" in prompt
120
- and "prompt_embeds" not in prompt
121
- )
122
-
123
-
124
-def is_embeds_prompt(prompt: SingletonPrompt) -> TypeIs[EmbedsPrompt]:
125
- return (
126
- isinstance(prompt, dict)
127
- and "prompt_token_ids" not in prompt
128
- and "prompt_embeds" in prompt
129
- )
130
-
131
-
119
_T1_co = TypeVar(
120
"_T1_co", bound=SingletonPrompt, default=SingletonPrompt, covariant=True
121
)
vllm/v1/engine/llm_engine.py
+3
-5
@@ -4,7 +4,7 @@
4
import time
5
from collections.abc import Callable, Mapping
6
from copy import copy
7
-from typing import Any, cast
7
+from typing import Any
8
9
import torch.nn as nn
10
from typing_extensions import TypeVar
@@ -32,6 +32,7 @@ from vllm.v1.engine.core_client import EngineCoreClient
32
from vllm.v1.engine.input_processor import InputProcessor
33
from vllm.v1.engine.output_processor import OutputProcessor
34
from vllm.v1.engine.parallel_sampling import ParentRequest
35
+from vllm.v1.engine.utils import get_prompt_text
36
from vllm.v1.executor import Executor
37
from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager
38
from vllm.v1.metrics.reader import Metric, get_metrics_snapshot
@@ -245,10 +246,7 @@ class LLMEngine:
246
trace_headers,
247
priority,
248
)
248
- if isinstance(prompt, str):
249
- prompt_text = prompt
250
- elif isinstance(prompt, Mapping):
251
- prompt_text = cast(str | None, prompt.get("prompt"))
249
+ prompt_text = get_prompt_text(prompt)
250
251
self.input_processor.assign_request_id(request)
252
vllm/v1/engine/utils.py
+6
-8
@@ -4,12 +4,12 @@
4
import contextlib
5
import os
6
import weakref
7
-from collections.abc import Callable, Iterator, Mapping
7
+from collections.abc import Callable, Iterator
8
from dataclasses import dataclass
9
from enum import Enum, auto
10
from multiprocessing import Process, connection
11
from multiprocessing.process import BaseProcess
12
-from typing import TYPE_CHECKING, Any, cast
12
+from typing import TYPE_CHECKING
13
from unittest.mock import patch
14
15
import msgspec
@@ -17,6 +17,8 @@ import zmq
17
18
from vllm import envs
19
from vllm.config import CacheConfig, ParallelConfig, VllmConfig
20
+from vllm.inputs import PromptType
21
+from vllm.inputs.parse import get_prompt_components
22
from vllm.logger import init_logger
23
from vllm.platforms import current_platform
24
from vllm.ray.ray_env import get_env_vars_to_copy
@@ -224,12 +226,8 @@ def get_device_indices(
226
return value
227
228
227
-def get_prompt_text(prompt: Any) -> str | None:
228
- if isinstance(prompt, str):
229
- return prompt
230
- if isinstance(prompt, Mapping):
231
- return cast(str | None, prompt.get("prompt"))
232
- return None
229
+def get_prompt_text(prompt: PromptType) -> str | None:
230
+ return get_prompt_components(prompt)[0]
231
232
233
class CoreEngineActorManager: