A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1] [5/N] API Server: unify `Detokenizer` and `EngineCore` input (#11545)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Robert Shaw committed
Dec 28, 2024 at 15:51 UTC
4fb8e329fd6f51d576bcf4b7e8907e0d83c4b5cf
6 files changed
+66
-77
tests/v1/engine/test_detokenizer.py
+35
-22
@@ -3,9 +3,9 @@ from typing import List
3
import pytest
4
from transformers import AutoTokenizer
5
6
-from vllm.sampling_params import RequestOutputKind
7
-from vllm.v1.engine import EngineCoreOutput
8
-from vllm.v1.engine.detokenizer import Detokenizer, DetokenizerRequest
6
+from vllm.sampling_params import RequestOutputKind, SamplingParams
7
+from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
8
+from vllm.v1.engine.detokenizer import Detokenizer
9
10
TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
11
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
@@ -71,16 +71,22 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
71
72
# Make N requests.
73
requests = [
74
- DetokenizerRequest(
75
- request_id=f"request-{idx}",
76
- prompt=prompt,
77
- prompt_token_ids=prompt_tokens,
78
- skip_special_tokens=False,
79
- spaces_between_special_tokens=False,
80
- output_kind=request_output_kind,
81
- stop=[],
82
- include_stop_str_in_output=False,
83
- ) for idx, (
74
+ EngineCoreRequest(request_id=f"request-{idx}",
75
+ prompt=prompt,
76
+ prompt_token_ids=prompt_tokens,
77
+ arrival_time=0,
78
+ mm_inputs=None,
79
+ mm_hashes=None,
80
+ mm_placeholders=None,
81
+ eos_token_id=None,
82
+ lora_request=None,
83
+ sampling_params=SamplingParams(
84
+ skip_special_tokens=False,
85
+ spaces_between_special_tokens=False,
86
+ output_kind=request_output_kind,
87
+ stop=[],
88
+ include_stop_str_in_output=False))
89
+ for idx, (
90
prompt,
91
prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
92
]
@@ -133,18 +139,25 @@ def test_stop_string(include_stop_str_in_output: bool):
139
140
# Make N requests.
141
requests = [
136
- DetokenizerRequest(
142
+ EngineCoreRequest(
143
request_id=f"request-{idx}",
144
prompt=prompt,
145
prompt_token_ids=prompt_tokens,
140
- skip_special_tokens=False,
141
- spaces_between_special_tokens=False,
142
- output_kind=RequestOutputKind.DELTA,
143
- stop=STOP_STRINGS,
144
- include_stop_str_in_output=include_stop_str_in_output,
145
- ) for idx, (
146
- prompt,
147
- prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
146
+ arrival_time=0,
147
+ mm_inputs=None,
148
+ mm_hashes=None,
149
+ mm_placeholders=None,
150
+ eos_token_id=None,
151
+ lora_request=None,
152
+ sampling_params=SamplingParams(
153
+ skip_special_tokens=False,
154
+ spaces_between_special_tokens=False,
155
+ output_kind=RequestOutputKind.DELTA,
156
+ stop=STOP_STRINGS,
157
+ include_stop_str_in_output=include_stop_str_in_output,
158
+ )) for idx, (
159
+ prompt,
160
+ prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
161
]
162
163
# Add requests to the detokenizer.
vllm/v1/engine/__init__.py
+1
-15
@@ -6,21 +6,7 @@ import msgspec
6
7
from vllm.lora.request import LoRARequest
8
from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
9
-from vllm.sampling_params import RequestOutputKind, SamplingParams
10
-
11
-
12
-@dataclass
13
-class DetokenizerRequest:
14
-
15
- request_id: str
16
- prompt: Optional[str]
17
- prompt_token_ids: List[int]
18
- skip_special_tokens: bool
19
- spaces_between_special_tokens: bool
20
- output_kind: RequestOutputKind
21
-
22
- stop: List[str]
23
- include_stop_str_in_output: bool
9
+from vllm.sampling_params import SamplingParams
10
11
12
@dataclass
vllm/v1/engine/async_llm.py
+8
-6
@@ -158,16 +158,18 @@ class AsyncLLM(EngineClient):
158
raise ValueError(f"Request id {request_id} already running.")
159
self.rid_to_queue[request_id] = asyncio.Queue()
160
161
- # 2) Convert input --> DetokenizerRequest / EngineCoreRequest.
162
- detokenizer_req, engine_core_req = self.processor.process_inputs(
163
- request_id, prompt, params, arrival_time, lora_request,
164
- trace_headers, prompt_adapter_request, priority)
161
+ # 2) Convert Input --> Request.
162
+ request = self.processor.process_inputs(request_id, prompt, params,
163
+ arrival_time, lora_request,
164
+ trace_headers,
165
+ prompt_adapter_request,
166
+ priority)
167
168
# 3) Add the request to Detokenizer (this process).
167
- self.detokenizer.add_request(detokenizer_req)
169
+ self.detokenizer.add_request(request)
170
171
# 4) Add the EngineCoreRequest to EngineCore (separate process).
170
- await self.engine_core.add_request_async(engine_core_req)
172
+ await self.engine_core.add_request_async(request)
173
174
if self.log_requests:
175
logger.info("Added request %s.", request_id)
vllm/v1/engine/detokenizer.py
+11
-10
@@ -8,7 +8,7 @@ from vllm.sampling_params import RequestOutputKind
8
from vllm.transformers_utils.detokenizer_utils import (
9
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
10
from vllm.transformers_utils.tokenizer import get_tokenizer
11
-from vllm.v1.engine import DetokenizerRequest, EngineCoreOutput
11
+from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
12
13
logger = init_logger(__name__)
14
@@ -55,19 +55,19 @@ class IncrementalDetokenizer:
55
def from_new_request(
56
cls,
57
tokenizer: AnyTokenizer,
58
- request: DetokenizerRequest,
58
+ request: EngineCoreRequest,
59
) -> "IncrementalDetokenizer":
60
61
tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
62
tokenizer=tokenizer,
63
prompt_ids=request.prompt_token_ids,
64
- skip_special_tokens=request.skip_special_tokens,
64
+ skip_special_tokens=request.sampling_params.skip_special_tokens,
65
)
66
67
- stops = request.stop
67
+ stops = request.sampling_params.stop
68
# Number of chars to hold back when stop strings are to be excluded
69
# from streamed output.
70
- if stops and not request.include_stop_str_in_output:
70
+ if stops and not request.sampling_params.include_stop_str_in_output:
71
stop_buffer_length = max(len(s) for s in stops) - 1
72
else:
73
stop_buffer_length = 0
@@ -79,13 +79,14 @@ class IncrementalDetokenizer:
79
# NOTE(Nick): could we take ownership of it though?
80
token_ids=request.prompt_token_ids.copy(),
81
stop=stops,
82
- include_stop_str_in_output=request.include_stop_str_in_output,
82
+ include_stop_str_in_output=request.sampling_params.
83
+ include_stop_str_in_output,
84
prefix_offset=prefix_offset,
85
read_offset=read_offset,
85
- skip_special_tokens=request.skip_special_tokens,
86
- spaces_between_special_tokens=request.
86
+ skip_special_tokens=request.sampling_params.skip_special_tokens,
87
+ spaces_between_special_tokens=request.sampling_params.
88
spaces_between_special_tokens,
88
- output_kind=request.output_kind,
89
+ output_kind=request.sampling_params.output_kind,
90
request_id=request.request_id,
91
prompt=request.prompt,
92
prompt_token_ids=request.prompt_token_ids,
@@ -227,7 +228,7 @@ class Detokenizer:
228
229
def add_request(
230
self,
230
- request: DetokenizerRequest,
231
+ request: EngineCoreRequest,
232
):
233
"""Add new request to the Detokenizer."""
234
vllm/v1/engine/llm_engine.py
+7
-5
@@ -152,15 +152,17 @@ class LLMEngine:
152
) -> None:
153
154
# 1) Process raw inputs into the request.
155
- detokenizer_req, engine_core_req = self.processor.process_inputs(
156
- request_id, prompt, params, arrival_time, lora_request,
157
- trace_headers, prompt_adapter_request, priority)
155
+ request = self.processor.process_inputs(request_id, prompt, params,
156
+ arrival_time, lora_request,
157
+ trace_headers,
158
+ prompt_adapter_request,
159
+ priority)
160
161
# 2) Add the request to Detokenizer.
160
- self.detokenizer.add_request(detokenizer_req)
162
+ self.detokenizer.add_request(request)
163
164
# 3) Add the request to EngineCore.
163
- self.engine_core.add_request(engine_core_req)
165
+ self.engine_core.add_request(request)
166
167
def step(self) -> List[RequestOutput]:
168
vllm/v1/engine/processor.py
+4
-19
@@ -1,5 +1,5 @@
1
import time
2
-from typing import Mapping, Optional, Tuple, Union
2
+from typing import Mapping, Optional, Union
3
4
from vllm.config import CacheConfig, LoRAConfig, ModelConfig
5
from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
@@ -13,7 +13,7 @@ from vllm.pooling_params import PoolingParams
13
from vllm.prompt_adapter.request import PromptAdapterRequest
14
from vllm.sampling_params import SamplingParams
15
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16
-from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
16
+from vllm.v1.engine import EngineCoreRequest
17
from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
18
19
@@ -62,7 +62,7 @@ class Processor:
62
trace_headers: Optional[Mapping[str, str]] = None,
63
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
64
priority: int = 0,
65
- ) -> Tuple[DetokenizerRequest, EngineCoreRequest]:
65
+ ) -> EngineCoreRequest:
66
67
# TODO(woosuk): Support pooling models.
68
# TODO(woosuk): Check max_logprobs
@@ -123,20 +123,7 @@ class Processor:
123
decoder_inputs.multi_modal_data, mm_hashes,
124
decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
125
126
- # Make Request for Detokenizer.
127
- detokenizer_request = DetokenizerRequest(
128
- request_id,
129
- decoder_inputs.prompt,
130
- decoder_inputs.prompt_token_ids,
131
- sampling_params.skip_special_tokens,
132
- sampling_params.spaces_between_special_tokens,
133
- sampling_params.output_kind,
134
- sampling_params.stop,
135
- sampling_params.include_stop_str_in_output,
136
- )
137
-
138
- # Make Request for EngineCore.
139
- engine_core_request = EngineCoreRequest(
126
+ return EngineCoreRequest(
127
request_id,
128
decoder_inputs.prompt,
129
decoder_inputs.prompt_token_ids,
@@ -149,8 +136,6 @@ class Processor:
136
lora_request,
137
)
138
152
- return detokenizer_request, engine_core_request
153
-
139
def _validate_model_inputs(self, inputs: ProcessorInputs):
140
if is_encoder_decoder_inputs(inputs):
141
# For encoder-decoder multimodal models, the max_prompt_len