@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4fb8e329fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1] [5/N] API Server: unify `Detokenizer` and `EngineCore` input (#11545)

Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>

Robert Shaw committed Dec 28, 2024 at 15:51 UTC 4fb8e329fd6f51d576bcf4b7e8907e0d83c4b5cf
6 files changed +66 -77
tests/v1/engine/test_detokenizer.py
+35 -22
@@ -3,9 +3,9 @@ from typing import List
3 import pytest
4 from transformers import AutoTokenizer
5
6 -from vllm.sampling_params import RequestOutputKind
7 -from vllm.v1.engine import EngineCoreOutput
8 -from vllm.v1.engine.detokenizer import Detokenizer, DetokenizerRequest
6 +from vllm.sampling_params import RequestOutputKind, SamplingParams
7 +from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
8 +from vllm.v1.engine.detokenizer import Detokenizer
9
10 TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
11 tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
@@ -71,16 +71,22 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
71
72 # Make N requests.
73 requests = [
74 - DetokenizerRequest(
75 - request_id=f"request-{idx}",
76 - prompt=prompt,
77 - prompt_token_ids=prompt_tokens,
78 - skip_special_tokens=False,
79 - spaces_between_special_tokens=False,
80 - output_kind=request_output_kind,
81 - stop=[],
82 - include_stop_str_in_output=False,
83 - ) for idx, (
74 + EngineCoreRequest(request_id=f"request-{idx}",
75 + prompt=prompt,
76 + prompt_token_ids=prompt_tokens,
77 + arrival_time=0,
78 + mm_inputs=None,
79 + mm_hashes=None,
80 + mm_placeholders=None,
81 + eos_token_id=None,
82 + lora_request=None,
83 + sampling_params=SamplingParams(
84 + skip_special_tokens=False,
85 + spaces_between_special_tokens=False,
86 + output_kind=request_output_kind,
87 + stop=[],
88 + include_stop_str_in_output=False))
89 + for idx, (
90 prompt,
91 prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
92 ]
@@ -133,18 +139,25 @@ def test_stop_string(include_stop_str_in_output: bool):
139
140 # Make N requests.
141 requests = [
136 - DetokenizerRequest(
142 + EngineCoreRequest(
143 request_id=f"request-{idx}",
144 prompt=prompt,
145 prompt_token_ids=prompt_tokens,
140 - skip_special_tokens=False,
141 - spaces_between_special_tokens=False,
142 - output_kind=RequestOutputKind.DELTA,
143 - stop=STOP_STRINGS,
144 - include_stop_str_in_output=include_stop_str_in_output,
145 - ) for idx, (
146 - prompt,
147 - prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
146 + arrival_time=0,
147 + mm_inputs=None,
148 + mm_hashes=None,
149 + mm_placeholders=None,
150 + eos_token_id=None,
151 + lora_request=None,
152 + sampling_params=SamplingParams(
153 + skip_special_tokens=False,
154 + spaces_between_special_tokens=False,
155 + output_kind=RequestOutputKind.DELTA,
156 + stop=STOP_STRINGS,
157 + include_stop_str_in_output=include_stop_str_in_output,
158 + )) for idx, (
159 + prompt,
160 + prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
161 ]
162
163 # Add requests to the detokenizer.
vllm/v1/engine/__init__.py
+1 -15
@@ -6,21 +6,7 @@ import msgspec
6
7 from vllm.lora.request import LoRARequest
8 from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
9 -from vllm.sampling_params import RequestOutputKind, SamplingParams
10 -
11 -
12 -@dataclass
13 -class DetokenizerRequest:
14 -
15 - request_id: str
16 - prompt: Optional[str]
17 - prompt_token_ids: List[int]
18 - skip_special_tokens: bool
19 - spaces_between_special_tokens: bool
20 - output_kind: RequestOutputKind
21 -
22 - stop: List[str]
23 - include_stop_str_in_output: bool
9 +from vllm.sampling_params import SamplingParams
10
11
12 @dataclass
vllm/v1/engine/async_llm.py
+8 -6
@@ -158,16 +158,18 @@ class AsyncLLM(EngineClient):
158 raise ValueError(f"Request id {request_id} already running.")
159 self.rid_to_queue[request_id] = asyncio.Queue()
160
161 - # 2) Convert input --> DetokenizerRequest / EngineCoreRequest.
162 - detokenizer_req, engine_core_req = self.processor.process_inputs(
163 - request_id, prompt, params, arrival_time, lora_request,
164 - trace_headers, prompt_adapter_request, priority)
161 + # 2) Convert Input --> Request.
162 + request = self.processor.process_inputs(request_id, prompt, params,
163 + arrival_time, lora_request,
164 + trace_headers,
165 + prompt_adapter_request,
166 + priority)
167
168 # 3) Add the request to Detokenizer (this process).
167 - self.detokenizer.add_request(detokenizer_req)
169 + self.detokenizer.add_request(request)
170
171 # 4) Add the EngineCoreRequest to EngineCore (separate process).
170 - await self.engine_core.add_request_async(engine_core_req)
172 + await self.engine_core.add_request_async(request)
173
174 if self.log_requests:
175 logger.info("Added request %s.", request_id)
vllm/v1/engine/detokenizer.py
+11 -10
@@ -8,7 +8,7 @@ from vllm.sampling_params import RequestOutputKind
8 from vllm.transformers_utils.detokenizer_utils import (
9 AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
10 from vllm.transformers_utils.tokenizer import get_tokenizer
11 -from vllm.v1.engine import DetokenizerRequest, EngineCoreOutput
11 +from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
12
13 logger = init_logger(__name__)
14
@@ -55,19 +55,19 @@ class IncrementalDetokenizer:
55 def from_new_request(
56 cls,
57 tokenizer: AnyTokenizer,
58 - request: DetokenizerRequest,
58 + request: EngineCoreRequest,
59 ) -> "IncrementalDetokenizer":
60
61 tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens(
62 tokenizer=tokenizer,
63 prompt_ids=request.prompt_token_ids,
64 - skip_special_tokens=request.skip_special_tokens,
64 + skip_special_tokens=request.sampling_params.skip_special_tokens,
65 )
66
67 - stops = request.stop
67 + stops = request.sampling_params.stop
68 # Number of chars to hold back when stop strings are to be excluded
69 # from streamed output.
70 - if stops and not request.include_stop_str_in_output:
70 + if stops and not request.sampling_params.include_stop_str_in_output:
71 stop_buffer_length = max(len(s) for s in stops) - 1
72 else:
73 stop_buffer_length = 0
@@ -79,13 +79,14 @@ class IncrementalDetokenizer:
79 # NOTE(Nick): could we take ownership of it though?
80 token_ids=request.prompt_token_ids.copy(),
81 stop=stops,
82 - include_stop_str_in_output=request.include_stop_str_in_output,
82 + include_stop_str_in_output=request.sampling_params.
83 + include_stop_str_in_output,
84 prefix_offset=prefix_offset,
85 read_offset=read_offset,
85 - skip_special_tokens=request.skip_special_tokens,
86 - spaces_between_special_tokens=request.
86 + skip_special_tokens=request.sampling_params.skip_special_tokens,
87 + spaces_between_special_tokens=request.sampling_params.
88 spaces_between_special_tokens,
88 - output_kind=request.output_kind,
89 + output_kind=request.sampling_params.output_kind,
90 request_id=request.request_id,
91 prompt=request.prompt,
92 prompt_token_ids=request.prompt_token_ids,
@@ -227,7 +228,7 @@ class Detokenizer:
228
229 def add_request(
230 self,
230 - request: DetokenizerRequest,
231 + request: EngineCoreRequest,
232 ):
233 """Add new request to the Detokenizer."""
234
vllm/v1/engine/llm_engine.py
+7 -5
@@ -152,15 +152,17 @@ class LLMEngine:
152 ) -> None:
153
154 # 1) Process raw inputs into the request.
155 - detokenizer_req, engine_core_req = self.processor.process_inputs(
156 - request_id, prompt, params, arrival_time, lora_request,
157 - trace_headers, prompt_adapter_request, priority)
155 + request = self.processor.process_inputs(request_id, prompt, params,
156 + arrival_time, lora_request,
157 + trace_headers,
158 + prompt_adapter_request,
159 + priority)
160
161 # 2) Add the request to Detokenizer.
160 - self.detokenizer.add_request(detokenizer_req)
162 + self.detokenizer.add_request(request)
163
164 # 3) Add the request to EngineCore.
163 - self.engine_core.add_request(engine_core_req)
165 + self.engine_core.add_request(request)
166
167 def step(self) -> List[RequestOutput]:
168
vllm/v1/engine/processor.py
+4 -19
@@ -1,5 +1,5 @@
1 import time
2 -from typing import Mapping, Optional, Tuple, Union
2 +from typing import Mapping, Optional, Union
3
4 from vllm.config import CacheConfig, LoRAConfig, ModelConfig
5 from vllm.inputs import (INPUT_REGISTRY, InputRegistry, ProcessorInputs,
@@ -13,7 +13,7 @@ from vllm.pooling_params import PoolingParams
13 from vllm.prompt_adapter.request import PromptAdapterRequest
14 from vllm.sampling_params import SamplingParams
15 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16 -from vllm.v1.engine import DetokenizerRequest, EngineCoreRequest
16 +from vllm.v1.engine import EngineCoreRequest
17 from vllm.v1.engine.mm_input_mapper import MMHasher, MMInputMapperClient
18
19
@@ -62,7 +62,7 @@ class Processor:
62 trace_headers: Optional[Mapping[str, str]] = None,
63 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
64 priority: int = 0,
65 - ) -> Tuple[DetokenizerRequest, EngineCoreRequest]:
65 + ) -> EngineCoreRequest:
66
67 # TODO(woosuk): Support pooling models.
68 # TODO(woosuk): Check max_logprobs
@@ -123,20 +123,7 @@ class Processor:
123 decoder_inputs.multi_modal_data, mm_hashes,
124 decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
125
126 - # Make Request for Detokenizer.
127 - detokenizer_request = DetokenizerRequest(
128 - request_id,
129 - decoder_inputs.prompt,
130 - decoder_inputs.prompt_token_ids,
131 - sampling_params.skip_special_tokens,
132 - sampling_params.spaces_between_special_tokens,
133 - sampling_params.output_kind,
134 - sampling_params.stop,
135 - sampling_params.include_stop_str_in_output,
136 - )
137 -
138 - # Make Request for EngineCore.
139 - engine_core_request = EngineCoreRequest(
126 + return EngineCoreRequest(
127 request_id,
128 decoder_inputs.prompt,
129 decoder_inputs.prompt_token_ids,
@@ -149,8 +136,6 @@ class Processor:
136 lora_request,
137 )
138
152 - return detokenizer_request, engine_core_request
153 -
139 def _validate_model_inputs(self, inputs: ProcessorInputs):
140 if is_encoder_decoder_inputs(inputs):
141 # For encoder-decoder multimodal models, the max_prompt_len