A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Frontend] Refactor prompt processing (#4028)
Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Jul 23, 2024 at 01:13 UTC
739b61a348afa5da297a80ff15f4e39d6e524b53
24 files changed
+698
-390
benchmarks/benchmark_latency.py
+2
-2
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
13
from vllm.engine.arg_utils import EngineArgs
14
-from vllm.inputs import PromptStrictInputs
14
+from vllm.inputs import PromptInputs
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61
dummy_prompt_token_ids = np.random.randint(10000,
62
size=(args.batch_size,
63
args.input_len))
64
- dummy_inputs: List[PromptStrictInputs] = [{
64
+ dummy_inputs: List[PromptInputs] = [{
65
"prompt_token_ids": batch
66
} for batch in dummy_prompt_token_ids.tolist()]
67
docs/source/dev/multimodal/multimodal_index.rst
+1
-1
@@ -8,7 +8,7 @@ Multi-Modality
8
vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10
Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11
-via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptStrictInputs`.
11
+via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
12
13
Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14
by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1
-1
@@ -1,7 +1,7 @@
1
LLM Inputs
2
==========
3
4
-.. autodata:: vllm.inputs.PromptStrictInputs
4
+.. autodata:: vllm.inputs.PromptInputs
5
6
.. autoclass:: vllm.inputs.TextPrompt
7
:show-inheritance:
docs/source/models/vlm.rst
+1
-1
@@ -30,7 +30,7 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
30
internally for each model.
31
32
33
-To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
33
+To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
34
35
* ``prompt``: The prompt should follow the format that is documented on HuggingFace.
36
* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/engine/output_processor/test_stop_checker.py
+2
-2
@@ -35,8 +35,8 @@ def sequence_with_eos(text: str, eos_token: str,
35
@pytest.mark.parametrize(["text_wo_eos", "eos_token", "eos_token_id"], [
36
("This text ends with EOS token", "</s>", 2),
37
])
38
-@pytest.mark.parametrize("ignore_eos", [True, False, None])
39
-@pytest.mark.parametrize("include_stop_str_in_output", [True, False, None])
38
+@pytest.mark.parametrize("ignore_eos", [True, False])
39
+@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
40
@pytest.mark.skip_global_cleanup
41
def test_stop_on_eos_token(text_wo_eos: str, eos_token: str, eos_token_id: int,
42
ignore_eos: bool, include_stop_str_in_output: bool):
tests/entrypoints/openai/test_serving_chat.py
+4
-1
@@ -32,7 +32,10 @@ async def _async_serving_chat_init():
32
model_config,
33
served_model_names=[MODEL_NAME],
34
response_role="assistant",
35
- chat_template=CHAT_TEMPLATE)
35
+ chat_template=CHAT_TEMPLATE,
36
+ lora_modules=None,
37
+ prompt_adapters=None,
38
+ request_logger=None)
39
return serving_completion
40
41
vllm/__init__.py
+2
-2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
-from vllm.inputs import PromptStrictInputs, TextPrompt, TokensPrompt
8
+from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
9
from vllm.model_executor.models import ModelRegistry
10
from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11
EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19
"__version__",
20
"LLM",
21
"ModelRegistry",
22
- "PromptStrictInputs",
22
+ "PromptInputs",
23
"TextPrompt",
24
"TokensPrompt",
25
"SamplingParams",
vllm/engine/arg_utils.py
-7
@@ -827,7 +827,6 @@ class AsyncEngineArgs(EngineArgs):
827
"""Arguments for asynchronous vLLM engine."""
828
engine_use_ray: bool = False
829
disable_log_requests: bool = False
830
- max_log_len: Optional[int] = None
830
831
@staticmethod
832
def add_cli_args(parser: FlexibleArgumentParser,
@@ -841,12 +840,6 @@ class AsyncEngineArgs(EngineArgs):
840
parser.add_argument('--disable-log-requests',
841
action='store_true',
842
help='Disable logging requests.')
844
- parser.add_argument('--max-log-len',
845
- type=int,
846
- default=None,
847
- help='Max number of prompt characters or prompt '
848
- 'ID numbers being printed in log.'
849
- '\n\nDefault: Unlimited')
843
return parser
844
845
vllm/engine/async_llm_engine.py
+22
-41
@@ -1,8 +1,8 @@
1
import asyncio
2
import time
3
from functools import partial
4
-from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
5
- Set, Tuple, Type, Union)
4
+from typing import (AsyncIterator, Callable, Dict, Iterable, List, Mapping,
5
+ Optional, Set, Tuple, Type, Union)
6
7
from transformers import PreTrainedTokenizer
8
@@ -151,7 +151,10 @@ class RequestTracker:
151
logger.info("Finished request %s.", request_id)
152
self.abort_request(request_id)
153
154
- def add_request(self, request_id: str,
154
+ def add_request(self,
155
+ request_id: str,
156
+ *,
157
+ verbose: bool = False,
158
**engine_add_request_kwargs) -> AsyncStream:
159
"""Add a request to be sent to the engine on the next background
160
loop iteration."""
@@ -166,6 +169,9 @@ class RequestTracker:
169
170
self.new_requests_event.set()
171
172
+ if verbose:
173
+ logger.info("Added request %s.", request_id)
174
+
175
return stream
176
177
def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
@@ -299,14 +305,14 @@ class _AsyncLLMEngine(LLMEngine):
305
return self.input_processor(llm_inputs)
306
307
async def add_request_async(
302
- self,
303
- request_id: str,
304
- inputs: PromptInputs,
305
- params: Union[SamplingParams, PoolingParams],
306
- arrival_time: Optional[float] = None,
307
- lora_request: Optional[LoRARequest] = None,
308
- trace_headers: Optional[Dict[str, str]] = None,
309
- prompt_adapter_request: Optional[PromptAdapterRequest] = None
308
+ self,
309
+ request_id: str,
310
+ inputs: PromptInputs,
311
+ params: Union[SamplingParams, PoolingParams],
312
+ arrival_time: Optional[float] = None,
313
+ lora_request: Optional[LoRARequest] = None,
314
+ trace_headers: Optional[Mapping[str, str]] = None,
315
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
316
) -> None:
317
if lora_request is not None and not self.lora_config:
318
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -353,8 +359,6 @@ class AsyncLLMEngine:
359
async frontend will be executed in a separate process as the
360
model workers.
361
log_requests: Whether to log the requests.
356
- max_log_len: Maximum number of prompt characters or prompt ID numbers
357
- being printed in log.
362
start_engine_loop: If True, the background task to run the engine
363
will be automatically started in the generate call.
364
*args: Arguments for :class:`LLMEngine`.
@@ -368,13 +372,11 @@ class AsyncLLMEngine:
372
engine_use_ray: bool,
373
*args,
374
log_requests: bool = True,
371
- max_log_len: Optional[int] = None,
375
start_engine_loop: bool = True,
376
**kwargs) -> None:
377
self.worker_use_ray = worker_use_ray
378
self.engine_use_ray = engine_use_ray
379
self.log_requests = log_requests
377
- self.max_log_len = max_log_len
380
self.engine = self._init_engine(*args, **kwargs)
381
382
self.background_loop: Optional[asyncio.Future] = None
@@ -468,7 +470,6 @@ class AsyncLLMEngine:
470
executor_class=executor_class,
471
log_requests=not engine_args.disable_log_requests,
472
log_stats=not engine_args.disable_log_stats,
471
- max_log_len=engine_args.max_log_len,
473
start_engine_loop=start_engine_loop,
474
usage_context=usage_context,
475
stat_loggers=stat_loggers,
@@ -667,30 +668,9 @@ class AsyncLLMEngine:
668
params: Union[SamplingParams, PoolingParams],
669
arrival_time: Optional[float] = None,
670
lora_request: Optional[LoRARequest] = None,
670
- trace_headers: Optional[Dict[str, str]] = None,
671
+ trace_headers: Optional[Mapping[str, str]] = None,
672
prompt_adapter_request: Optional[PromptAdapterRequest] = None
673
) -> AsyncStream:
673
- if self.log_requests:
674
- if isinstance(inputs, str):
675
- shortened_prompt = inputs
676
- shortened_token_ids = None
677
- else:
678
- shortened_prompt = inputs.get("prompt")
679
- shortened_token_ids = inputs.get("prompt_token_ids")
680
-
681
- max_log_len = self.max_log_len
682
- if max_log_len is not None:
683
- if shortened_prompt is not None:
684
- shortened_prompt = shortened_prompt[:max_log_len]
685
- if shortened_token_ids is not None:
686
- shortened_token_ids = shortened_token_ids[:max_log_len]
687
-
688
- logger.info(
689
- "Received request %s: prompt: %r, "
690
- "params: %s, prompt_token_ids: %s, "
691
- "lora_request: %s.", request_id, shortened_prompt, params,
692
- shortened_token_ids, lora_request)
693
-
674
if not self.is_running:
675
if self.start_engine_loop:
676
self.start_background_loop()
@@ -706,6 +686,7 @@ class AsyncLLMEngine:
686
687
stream = self._request_tracker.add_request(
688
request_id,
689
+ verbose=self.log_requests,
690
inputs=inputs,
691
params=params,
692
arrival_time=arrival_time,
@@ -721,7 +702,7 @@ class AsyncLLMEngine:
702
sampling_params: SamplingParams,
703
request_id: str,
704
lora_request: Optional[LoRARequest] = None,
724
- trace_headers: Optional[Dict[str, str]] = None,
705
+ trace_headers: Optional[Mapping[str, str]] = None,
706
prompt_adapter_request: Optional[PromptAdapterRequest] = None
707
) -> AsyncIterator[RequestOutput]:
708
"""Generate outputs for a request.
@@ -804,7 +785,7 @@ class AsyncLLMEngine:
785
pooling_params: PoolingParams,
786
request_id: str,
787
lora_request: Optional[LoRARequest] = None,
807
- trace_headers: Optional[Dict[str, str]] = None,
788
+ trace_headers: Optional[Mapping[str, str]] = None,
789
) -> AsyncIterator[EmbeddingRequestOutput]:
790
"""Generate outputs for a request from an embedding model.
791
@@ -882,7 +863,7 @@ class AsyncLLMEngine:
863
params: Union[SamplingParams, PoolingParams],
864
*,
865
lora_request: Optional[LoRARequest] = None,
885
- trace_headers: Optional[Dict[str, str]] = None,
866
+ trace_headers: Optional[Mapping[str, str]] = None,
867
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
868
) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
869
"""Common logic to process requests with SamplingParams or
vllm/engine/llm_engine.py
+5
-4
@@ -1,6 +1,7 @@
1
import time
2
from contextlib import contextmanager
3
-from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List, Optional
3
+from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
4
+ Mapping, Optional)
5
from typing import Sequence as GenericSequence
6
from typing import Set, Type, TypeVar, Union
7
@@ -522,7 +523,7 @@ class LLMEngine:
523
arrival_time: float,
524
lora_request: Optional[LoRARequest],
525
prompt_adapter_request: Optional[PromptAdapterRequest],
525
- trace_headers: Optional[Dict[str, str]] = None,
526
+ trace_headers: Optional[Mapping[str, str]] = None,
527
) -> None:
528
# Create the sequences.
529
block_size = self.cache_config.block_size
@@ -603,7 +604,7 @@ class LLMEngine:
604
params: Union[SamplingParams, PoolingParams],
605
arrival_time: Optional[float] = None,
606
lora_request: Optional[LoRARequest] = None,
606
- trace_headers: Optional[Dict[str, str]] = None,
607
+ trace_headers: Optional[Mapping[str, str]] = None,
608
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
609
) -> None:
610
"""Add a request to the engine's request pool.
@@ -677,7 +678,7 @@ class LLMEngine:
678
sampling_params: SamplingParams,
679
arrival_time: float,
680
lora_request: Optional[LoRARequest],
680
- trace_headers: Optional[Dict[str, str]] = None,
681
+ trace_headers: Optional[Mapping[str, str]] = None,
682
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
683
) -> SequenceGroup:
684
"""Creates a SequenceGroup with SamplingParams."""
vllm/entrypoints/llm.py
+13
-24
@@ -6,8 +6,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.llm_engine import LLMEngine
9
-from vllm.inputs import (PromptInputs, PromptStrictInputs, TextPrompt,
10
- TextTokensPrompt, TokensPrompt,
9
+from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
10
parse_and_batch_prompt)
11
from vllm.logger import init_logger
12
from vllm.lora.request import LoRARequest
@@ -238,7 +237,7 @@ class LLM:
237
@overload
238
def generate(
239
self,
241
- inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
240
+ inputs: Union[PromptInputs, Sequence[PromptInputs]],
241
/, # We may enable `inputs` keyword after removing the old API
242
*,
243
sampling_params: Optional[Union[SamplingParams,
@@ -255,7 +254,7 @@ class LLM:
254
"instead.")
255
def generate(
256
self,
258
- prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
257
+ prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
258
Optional[Union[str, List[str]]]] = None,
259
sampling_params: Optional[Union[SamplingParams,
260
Sequence[SamplingParams]]] = None,
@@ -302,9 +301,7 @@ class LLM:
301
prompt_token_ids=prompt_token_ids,
302
)
303
else:
305
- inputs = cast(
306
- Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
307
- prompts)
304
+ inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
305
306
if sampling_params is None:
307
# Use default sampling params.
@@ -383,7 +380,7 @@ class LLM:
380
@overload
381
def encode(
382
self,
386
- inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
383
+ inputs: Union[PromptInputs, Sequence[PromptInputs]],
384
/, # We may enable `inputs` keyword after removing the old API
385
*,
386
pooling_params: Optional[Union[PoolingParams,
@@ -400,7 +397,7 @@ class LLM:
397
"instead.")
398
def encode(
399
self,
403
- prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
400
+ prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
401
Optional[Union[str, List[str]]]] = None,
402
pooling_params: Optional[Union[PoolingParams,
403
Sequence[PoolingParams]]] = None,
@@ -417,7 +414,7 @@ class LLM:
414
415
Args:
416
inputs: The inputs to the LLM. You may pass a sequence of inputs for
420
- batch inference. See :class:`~vllm.inputs.PromptStrictInputs`
417
+ batch inference. See :class:`~vllm.inputs.PromptInputs`
418
for more details about the format of each input.
419
pooling_params: The pooling parameters for pooling. If None, we
420
use the default pooling parameters.
@@ -446,9 +443,7 @@ class LLM:
443
prompt_token_ids=prompt_token_ids,
444
)
445
else:
449
- inputs = cast(
450
- Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
451
- prompts)
446
+ inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
447
448
if pooling_params is None:
449
# Use default pooling params.
@@ -496,17 +491,11 @@ class LLM:
491
inputs: List[PromptInputs] = []
492
for i in range(num_requests):
493
if prompts is not None:
499
- if prompt_token_ids is not None:
500
- item = TextTokensPrompt(
501
- prompt=prompts[i],
502
- prompt_token_ids=prompt_token_ids[i])
503
- else:
504
- item = TextPrompt(prompt=prompts[i])
494
+ item = TextPrompt(prompt=prompts[i])
495
+ elif prompt_token_ids is not None:
496
+ item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
497
else:
506
- if prompt_token_ids is not None:
507
- item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
508
- else:
509
- raise AssertionError
498
+ raise AssertionError
499
500
inputs.append(item)
501
@@ -514,7 +503,7 @@ class LLM:
503
504
def _validate_and_add_requests(
505
self,
517
- inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
506
+ inputs: Union[PromptInputs, Sequence[PromptInputs]],
507
params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
508
Sequence[PoolingParams]],
509
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
vllm/entrypoints/logger.py
new
+41
@@ -0,0 +1,41 @@
1
+from typing import List, Optional, Union
2
+
3
+from vllm.logger import init_logger
4
+from vllm.lora.request import LoRARequest
5
+from vllm.pooling_params import PoolingParams
6
+from vllm.prompt_adapter.request import PromptAdapterRequest
7
+from vllm.sampling_params import SamplingParams
8
+
9
+logger = init_logger(__name__)
10
+
11
+
12
+class RequestLogger:
13
+
14
+ def __init__(self, *, max_log_len: Optional[int]) -> None:
15
+ super().__init__()
16
+
17
+ self.max_log_len = max_log_len
18
+
19
+ def log_inputs(
20
+ self,
21
+ request_id: str,
22
+ prompt: Optional[str],
23
+ prompt_token_ids: Optional[List[int]],
24
+ params: Optional[Union[SamplingParams, PoolingParams]],
25
+ lora_request: Optional[LoRARequest],
26
+ prompt_adapter_request: Optional[PromptAdapterRequest],
27
+ ) -> None:
28
+ max_log_len = self.max_log_len
29
+ if max_log_len is not None:
30
+ if prompt is not None:
31
+ prompt = prompt[:max_log_len]
32
+
33
+ if prompt_token_ids is not None:
34
+ prompt_token_ids = prompt_token_ids[:max_log_len]
35
+
36
+ logger.info(
37
+ "Received request %s: prompt: %r, "
38
+ "params: %s, prompt_token_ids: %s, "
39
+ "lora_request: %s, prompt_adapter_request: %s.", request_id,
40
+ prompt, params, prompt_token_ids, lora_request,
41
+ prompt_adapter_request)
vllm/entrypoints/openai/api_server.py
+36
-11
@@ -18,6 +18,7 @@ from starlette.routing import Mount
18
import vllm.envs as envs
19
from vllm.engine.arg_utils import AsyncEngineArgs
20
from vllm.engine.async_llm_engine import AsyncLLMEngine
21
+from vllm.entrypoints.logger import RequestLogger
22
from vllm.entrypoints.openai.cli_args import make_arg_parser
23
# yapf conflicts with isort for this block
24
# yapf: disable
@@ -244,24 +245,48 @@ def run_server(args, llm_engine=None):
245
# When using single vLLM without engine_use_ray
246
model_config = asyncio.run(engine.get_model_config())
247
248
+ if args.disable_log_requests:
249
+ request_logger = None
250
+ else:
251
+ request_logger = RequestLogger(max_log_len=args.max_log_len)
252
+
253
global openai_serving_chat
254
global openai_serving_completion
255
global openai_serving_embedding
256
global openai_serving_tokenization
257
252
- openai_serving_chat = OpenAIServingChat(engine, model_config,
253
- served_model_names,
254
- args.response_role,
255
- args.lora_modules,
256
- args.chat_template)
258
+ openai_serving_chat = OpenAIServingChat(
259
+ engine,
260
+ model_config,
261
+ served_model_names,
262
+ args.response_role,
263
+ lora_modules=args.lora_modules,
264
+ prompt_adapters=args.prompt_adapters,
265
+ request_logger=request_logger,
266
+ chat_template=args.chat_template,
267
+ )
268
openai_serving_completion = OpenAIServingCompletion(
258
- engine, model_config, served_model_names, args.lora_modules,
259
- args.prompt_adapters)
260
- openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
261
- served_model_names)
269
+ engine,
270
+ model_config,
271
+ served_model_names,
272
+ lora_modules=args.lora_modules,
273
+ prompt_adapters=args.prompt_adapters,
274
+ request_logger=request_logger,
275
+ )
276
+ openai_serving_embedding = OpenAIServingEmbedding(
277
+ engine,
278
+ model_config,
279
+ served_model_names,
280
+ request_logger=request_logger,
281
+ )
282
openai_serving_tokenization = OpenAIServingTokenization(
263
- engine, model_config, served_model_names, args.lora_modules,
264
- args.chat_template)
283
+ engine,
284
+ model_config,
285
+ served_model_names,
286
+ lora_modules=args.lora_modules,
287
+ request_logger=request_logger,
288
+ chat_template=args.chat_template,
289
+ )
290
app.root_path = args.root_path
291
292
logger.info("Available routes are:")
vllm/entrypoints/openai/cli_args.py
+8
@@ -130,6 +130,14 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
130
"using app.add_middleware(). ")
131
132
parser = AsyncEngineArgs.add_cli_args(parser)
133
+
134
+ parser.add_argument('--max-log-len',
135
+ type=int,
136
+ default=None,
137
+ help='Max number of prompt characters or prompt '
138
+ 'ID numbers being printed in log.'
139
+ '\n\nDefault: Unlimited')
140
+
141
return parser
142
143
vllm/entrypoints/openai/protocol.py
+58
-50
@@ -121,40 +121,42 @@ class ChatCompletionRequest(OpenAIBaseModel):
121
122
# doc: begin-chat-completion-sampling-params
123
best_of: Optional[int] = None
124
- use_beam_search: Optional[bool] = False
125
- top_k: Optional[int] = -1
126
- min_p: Optional[float] = 0.0
127
- repetition_penalty: Optional[float] = 1.0
128
- length_penalty: Optional[float] = 1.0
129
- early_stopping: Optional[bool] = False
130
- ignore_eos: Optional[bool] = False
131
- min_tokens: Optional[int] = 0
124
+ use_beam_search: bool = False
125
+ top_k: int = -1
126
+ min_p: float = 0.0
127
+ repetition_penalty: float = 1.0
128
+ length_penalty: float = 1.0
129
+ early_stopping: bool = False
130
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
133
- skip_special_tokens: Optional[bool] = True
134
- spaces_between_special_tokens: Optional[bool] = True
131
+ include_stop_str_in_output: bool = False
132
+ ignore_eos: bool = False
133
+ min_tokens: int = 0
134
+ skip_special_tokens: bool = True
135
+ spaces_between_special_tokens: bool = True
136
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
137
# doc: end-chat-completion-sampling-params
138
139
# doc: begin-chat-completion-extra-params
138
- echo: Optional[bool] = Field(
140
+ echo: bool = Field(
141
default=False,
142
description=(
143
"If true, the new message will be prepended with the last message "
144
"if they belong to the same role."),
145
)
144
- add_generation_prompt: Optional[bool] = Field(
146
+ add_generation_prompt: bool = Field(
147
default=True,
148
description=
149
("If true, the generation prompt will be added to the chat template. "
150
"This is a parameter used by chat template in tokenizer config of the "
151
"model."),
152
)
151
- add_special_tokens: Optional[bool] = Field(
153
+ add_special_tokens: bool = Field(
154
default=False,
155
description=(
156
"If true, special tokens (e.g. BOS) will be added to the prompt "
157
"on top of what is added by the chat template. "
158
"For most models, the chat template takes care of adding the "
157
- "special tokens so this should be set to False (as is the "
159
+ "special tokens so this should be set to false (as is the "
160
"default)."),
161
)
162
documents: Optional[List[Dict[str, str]]] = Field(
@@ -178,12 +180,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
180
description=("Additional kwargs to pass to the template renderer. "
181
"Will be accessible by the chat template."),
182
)
181
- include_stop_str_in_output: Optional[bool] = Field(
182
- default=False,
183
- description=(
184
- "Whether to include the stop string in the output. "
185
- "This is only applied when the stop or stop_token_ids is set."),
186
- )
183
guided_json: Optional[Union[str, dict, BaseModel]] = Field(
184
default=None,
185
description=("If specified, the output will follow the JSON schema."),
@@ -244,22 +240,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
240
241
return SamplingParams(
242
n=self.n,
243
+ best_of=self.best_of,
244
presence_penalty=self.presence_penalty,
245
frequency_penalty=self.frequency_penalty,
246
repetition_penalty=self.repetition_penalty,
247
temperature=self.temperature,
248
top_p=self.top_p,
249
+ top_k=self.top_k,
250
min_p=self.min_p,
251
seed=self.seed,
252
stop=self.stop,
253
stop_token_ids=self.stop_token_ids,
256
- max_tokens=self.max_tokens,
257
- min_tokens=self.min_tokens,
254
logprobs=self.top_logprobs if self.logprobs else None,
255
prompt_logprobs=self.top_logprobs if self.echo else None,
260
- best_of=self.best_of,
261
- top_k=self.top_k,
256
ignore_eos=self.ignore_eos,
257
+ max_tokens=self.max_tokens,
258
+ min_tokens=self.min_tokens,
259
use_beam_search=self.use_beam_search,
260
early_stopping=self.early_stopping,
261
skip_special_tokens=self.skip_special_tokens,
@@ -267,6 +263,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
263
include_stop_str_in_output=self.include_stop_str_in_output,
264
length_penalty=self.length_penalty,
265
logits_processors=logits_processors,
266
+ truncate_prompt_tokens=self.truncate_prompt_tokens,
267
)
268
269
@model_validator(mode='before')
@@ -348,26 +345,27 @@ class CompletionRequest(OpenAIBaseModel):
345
user: Optional[str] = None
346
347
# doc: begin-completion-sampling-params
351
- use_beam_search: Optional[bool] = False
352
- top_k: Optional[int] = -1
353
- min_p: Optional[float] = 0.0
354
- repetition_penalty: Optional[float] = 1.0
355
- length_penalty: Optional[float] = 1.0
356
- early_stopping: Optional[bool] = False
348
+ use_beam_search: bool = False
349
+ top_k: int = -1
350
+ min_p: float = 0.0
351
+ repetition_penalty: float = 1.0
352
+ length_penalty: float = 1.0
353
+ early_stopping: bool = False
354
stop_token_ids: Optional[List[int]] = Field(default_factory=list)
358
- ignore_eos: Optional[bool] = False
359
- min_tokens: Optional[int] = 0
360
- skip_special_tokens: Optional[bool] = True
361
- spaces_between_special_tokens: Optional[bool] = True
355
+ include_stop_str_in_output: bool = False
356
+ ignore_eos: bool = False
357
+ min_tokens: int = 0
358
+ skip_special_tokens: bool = True
359
+ spaces_between_special_tokens: bool = True
360
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
361
# doc: end-completion-sampling-params
362
363
# doc: begin-completion-extra-params
366
- include_stop_str_in_output: Optional[bool] = Field(
367
- default=False,
364
+ add_special_tokens: bool = Field(
365
+ default=True,
366
description=(
369
- "Whether to include the stop string in the output. "
370
- "This is only applied when the stop or stop_token_ids is set."),
367
+ "If true (the default), special tokens (e.g. BOS) will be added to "
368
+ "the prompt."),
369
)
370
response_format: Optional[ResponseFormat] = Field(
371
default=None,
@@ -447,15 +445,15 @@ class CompletionRequest(OpenAIBaseModel):
445
seed=self.seed,
446
stop=self.stop,
447
stop_token_ids=self.stop_token_ids,
448
+ logprobs=self.logprobs,
449
ignore_eos=self.ignore_eos,
450
max_tokens=self.max_tokens if not echo_without_generation else 1,
451
min_tokens=self.min_tokens,
453
- logprobs=self.logprobs,
452
use_beam_search=self.use_beam_search,
453
early_stopping=self.early_stopping,
454
prompt_logprobs=self.logprobs if self.echo else None,
455
skip_special_tokens=self.skip_special_tokens,
458
- spaces_between_special_tokens=(self.spaces_between_special_tokens),
456
+ spaces_between_special_tokens=self.spaces_between_special_tokens,
457
include_stop_str_in_output=self.include_stop_str_in_output,
458
length_penalty=self.length_penalty,
459
logits_processors=logits_processors,
@@ -489,11 +487,11 @@ class CompletionRequest(OpenAIBaseModel):
487
def validate_stream_options(cls, data):
488
if data.get("stream_options") and not data.get("stream"):
489
raise ValueError(
492
- "Stream options can only be defined when stream is True.")
490
+ "Stream options can only be defined when stream is true.")
491
return data
492
493
496
-class EmbeddingRequest(BaseModel):
494
+class EmbeddingRequest(OpenAIBaseModel):
495
# Ordered by official OpenAI API documentation
496
# https://platform.openai.com/docs/api-reference/embeddings
497
model: str
@@ -565,13 +563,13 @@ class CompletionStreamResponse(OpenAIBaseModel):
563
usage: Optional[UsageInfo] = Field(default=None)
564
565
568
-class EmbeddingResponseData(BaseModel):
566
+class EmbeddingResponseData(OpenAIBaseModel):
567
index: int
568
object: str = "embedding"
569
embedding: Union[List[float], str]
570
571
574
-class EmbeddingResponse(BaseModel):
572
+class EmbeddingResponse(OpenAIBaseModel):
573
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
574
object: str = "list"
575
created: int = Field(default_factory=lambda: int(time.time()))
@@ -670,8 +668,8 @@ class BatchRequestInput(OpenAIBaseModel):
668
# /v1/chat/completions is supported.
669
url: str
670
673
- # The parameteters of the request.
674
- body: Union[ChatCompletionRequest, ]
671
+ # The parameters of the request.
672
+ body: ChatCompletionRequest
673
674
675
class BatchResponseData(OpenAIBaseModel):
@@ -703,12 +701,22 @@ class BatchRequestOutput(OpenAIBaseModel):
701
error: Optional[Any]
702
703
706
-class TokenizeRequest(OpenAIBaseModel):
704
+class TokenizeCompletionRequest(OpenAIBaseModel):
705
+ model: str
706
+ prompt: str
707
+
708
+ add_special_tokens: bool = Field(default=True)
709
+
710
+
711
+class TokenizeChatRequest(OpenAIBaseModel):
712
+ model: str
713
+ messages: List[ChatCompletionMessageParam]
714
+
715
add_generation_prompt: bool = Field(default=True)
716
add_special_tokens: bool = Field(default=False)
709
- prompt: Optional[str] = Field(default=None)
710
- messages: Optional[List[ChatCompletionMessageParam]] = Field(default=None)
711
- model: str
717
+
718
+
719
+TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
720
721
722
class TokenizeResponse(OpenAIBaseModel):
vllm/entrypoints/openai/run_batch.py
+19
-1
@@ -6,6 +6,7 @@ import aiohttp
6
7
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
8
from vllm.engine.async_llm_engine import AsyncLLMEngine
9
+from vllm.entrypoints.logger import RequestLogger
10
from vllm.entrypoints.openai.protocol import (BatchRequestInput,
11
BatchRequestOutput,
12
BatchResponseData,
@@ -44,9 +45,17 @@ def parse_args():
45
type=nullable_str,
46
default="assistant",
47
help="The role name to return if "
47
- "`request.add_generation_prompt=true`.")
48
+ "`request.add_generation_prompt=True`.")
49
50
parser = AsyncEngineArgs.add_cli_args(parser)
51
+
52
+ parser.add_argument('--max-log-len',
53
+ type=int,
54
+ default=None,
55
+ help='Max number of prompt characters or prompt '
56
+ 'ID numbers being printed in log.'
57
+ '\n\nDefault: Unlimited')
58
+
59
return parser.parse_args()
60
61
@@ -114,11 +123,20 @@ async def main(args):
123
# When using single vLLM without engine_use_ray
124
model_config = await engine.get_model_config()
125
126
+ if args.disable_log_requests:
127
+ request_logger = None
128
+ else:
129
+ request_logger = RequestLogger(max_log_len=args.max_log_len)
130
+
131
openai_serving_chat = OpenAIServingChat(
132
engine,
133
model_config,
134
served_model_names,
135
args.response_role,
136
+ lora_modules=None,
137
+ prompt_adapters=None,
138
+ request_logger=request_logger,
139
+ chat_template=None,
140
)
141
142
# Submit all requests in the file to the engine "concurrently".
vllm/entrypoints/openai/serving_chat.py
+72
-47
@@ -12,6 +12,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.entrypoints.chat_utils import (ConversationMessage,
13
load_chat_template,
14
parse_chat_message_content)
15
+from vllm.entrypoints.logger import RequestLogger
16
from vllm.entrypoints.openai.protocol import (
17
ChatCompletionLogProb, ChatCompletionLogProbs,
18
ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
@@ -20,7 +21,8 @@ from vllm.entrypoints.openai.protocol import (
21
ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
22
FunctionCall, ToolCall, UsageInfo)
23
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
23
- OpenAIServing)
24
+ OpenAIServing,
25
+ PromptAdapterPath)
26
from vllm.inputs import PromptInputs
27
from vllm.logger import init_logger
28
from vllm.model_executor.guided_decoding import (
@@ -37,17 +39,24 @@ logger = init_logger(__name__)
39
40
class OpenAIServingChat(OpenAIServing):
41
40
- def __init__(self,
41
- engine: AsyncLLMEngine,
42
- model_config: ModelConfig,
43
- served_model_names: List[str],
44
- response_role: str,
45
- lora_modules: Optional[List[LoRAModulePath]] = None,
46
- chat_template: Optional[str] = None):
42
+ def __init__(
43
+ self,
44
+ engine: AsyncLLMEngine,
45
+ model_config: ModelConfig,
46
+ served_model_names: List[str],
47
+ response_role: str,
48
+ *,
49
+ lora_modules: Optional[List[LoRAModulePath]],
50
+ prompt_adapters: Optional[List[PromptAdapterPath]],
51
+ request_logger: Optional[RequestLogger],
52
+ chat_template: Optional[str],
53
+ ):
54
super().__init__(engine=engine,
55
model_config=model_config,
56
served_model_names=served_model_names,
50
- lora_modules=lora_modules)
57
+ lora_modules=lora_modules,
58
+ prompt_adapters=prompt_adapters,
59
+ request_logger=request_logger)
60
61
self.response_role = response_role
62
@@ -74,7 +83,12 @@ class OpenAIServingChat(OpenAIServing):
83
return error_check_ret
84
85
try:
77
- _, lora_request = self._maybe_get_adapter(request)
86
+ (
87
+ lora_request,
88
+ prompt_adapter_request,
89
+ ) = self._maybe_get_adapters(request)
90
+
91
+ model_config = self.model_config
92
tokenizer = await self.engine.get_tokenizer(lora_request)
93
94
conversation: List[ConversationMessage] = []
@@ -82,7 +96,7 @@ class OpenAIServingChat(OpenAIServing):
96
97
for msg in request.messages:
98
chat_parsed_result = parse_chat_message_content(
85
- msg, self.model_config, tokenizer)
99
+ msg, model_config, tokenizer)
100
101
conversation.extend(chat_parsed_result.messages)
102
mm_futures.extend(chat_parsed_result.mm_futures)
@@ -116,14 +130,8 @@ class OpenAIServingChat(OpenAIServing):
130
logger.error("Error in loading multi-modal data: %s", e)
131
return self.create_error_response(str(e))
132
119
- request_id = f"cmpl-{random_uuid()}"
133
+ request_id = f"chat-{random_uuid()}"
134
try:
121
- # Tokenize/detokenize depending on prompt format (string/token list)
122
- prompt_ids, prompt_text = await self._validate_prompt_and_tokenize(
123
- request,
124
- tokenizer,
125
- prompt=prompt,
126
- add_special_tokens=request.add_special_tokens)
135
sampling_params = request.to_sampling_params()
136
decoding_config = await self.engine.get_decoding_config()
137
guided_decoding_backend = request.guided_decoding_backend \
@@ -137,31 +145,47 @@ class OpenAIServingChat(OpenAIServing):
145
sampling_params.logits_processors = []
146
sampling_params.logits_processors.append(
147
guided_decode_logits_processor)
148
+
149
+ prompt_inputs = self._tokenize_prompt_input(
150
+ request,
151
+ tokenizer,
152
+ prompt,
153
+ truncate_prompt_tokens=sampling_params.truncate_prompt_tokens,
154
+ add_special_tokens=request.add_special_tokens,
155
+ )
156
+
157
+ self._log_inputs(request_id,
158
+ prompt_inputs,
159
+ params=sampling_params,
160
+ lora_request=lora_request,
161
+ prompt_adapter_request=prompt_adapter_request)
162
+
163
+ engine_inputs: PromptInputs = {
164
+ "prompt_token_ids": prompt_inputs["prompt_token_ids"],
165
+ }
166
+ if mm_data is not None:
167
+ engine_inputs["multi_modal_data"] = mm_data
168
+
169
+ is_tracing_enabled = await self.engine.is_tracing_enabled()
170
+ trace_headers = None
171
+ if is_tracing_enabled and raw_request:
172
+ trace_headers = extract_trace_headers(raw_request.headers)
173
+ if (not is_tracing_enabled and raw_request
174
+ and contains_trace_headers(raw_request.headers)):
175
+ log_tracing_disabled_warning()
176
+
177
+ result_generator = self.engine.generate(
178
+ engine_inputs,
179
+ sampling_params,
180
+ request_id,
181
+ lora_request=lora_request,
182
+ trace_headers=trace_headers,
183
+ prompt_adapter_request=prompt_adapter_request,
184
+ )
185
except ValueError as e:
186
+ # TODO: Use a vllm-specific Validation Error
187
return self.create_error_response(str(e))
188
143
- inputs: PromptInputs = {
144
- "prompt": prompt_text,
145
- "prompt_token_ids": prompt_ids,
146
- }
147
- if mm_data:
148
- inputs["multi_modal_data"] = mm_data
149
-
150
- is_tracing_enabled = await self.engine.is_tracing_enabled()
151
- trace_headers = None
152
- if is_tracing_enabled and raw_request:
153
- trace_headers = extract_trace_headers(raw_request.headers)
154
- if not is_tracing_enabled and raw_request and contains_trace_headers(
155
- raw_request.headers):
156
- log_tracing_disabled_warning()
157
-
158
- result_generator = self.engine.generate(
159
- inputs,
160
- sampling_params,
161
- request_id,
162
- lora_request,
163
- trace_headers=trace_headers,
164
- )
189
# Streaming response
190
if request.stream:
191
return self.chat_completion_stream_generator(
@@ -195,10 +219,11 @@ class OpenAIServingChat(OpenAIServing):
219
first_iteration = True
220
221
# Send response for each token for each request.n (index)
198
- assert request.n is not None
199
- previous_texts = [""] * request.n
200
- previous_num_tokens = [0] * request.n
201
- finish_reason_sent = [False] * request.n
222
+ num_choices = 1 if request.n is None else request.n
223
+ previous_texts = [""] * num_choices
224
+ previous_num_tokens = [0] * num_choices
225
+ finish_reason_sent = [False] * num_choices
226
+
227
try:
228
async for res in result_generator:
229
# We need to do it here, because if there are exceptions in
@@ -208,7 +233,7 @@ class OpenAIServingChat(OpenAIServing):
233
# Send first response for each request.n (index) with
234
# the role
235
role = self.get_chat_request_role(request)
211
- for i in range(request.n):
236
+ for i in range(num_choices):
237
choice_data = ChatCompletionResponseStreamChoice(
238
index=i,
239
delta=DeltaMessage(role=role),
@@ -236,19 +261,19 @@ class OpenAIServingChat(OpenAIServing):
261
last_msg_content = conversation[-1]["content"]
262
263
if last_msg_content:
239
- for i in range(request.n):
264
+ for i in range(num_choices):
265
choice_data = (
266
ChatCompletionResponseStreamChoice(
267
index=i,
268
delta=DeltaMessage(
269
content=last_msg_content),
270
+ logprobs=None,
271
finish_reason=None))
272
chunk = ChatCompletionStreamResponse(
273
id=request_id,
274
object=chunk_object_type,
275
created=created_time,
276
choices=[choice_data],
251
- logprobs=None,
277
model=model_name)
278
if (request.stream_options and
279
request.stream_options.include_usage):
vllm/entrypoints/openai/serving_completion.py
+61
-54
@@ -2,13 +2,14 @@ import time
2
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
3
Optional)
4
from typing import Sequence as GenericSequence
5
-from typing import Tuple
5
+from typing import Tuple, cast
6
7
from fastapi import Request
8
from transformers import PreTrainedTokenizer
9
10
from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
+from vllm.entrypoints.logger import RequestLogger
13
# yapf conflicts with isort for this block
14
# yapf: disable
15
from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
@@ -39,40 +40,24 @@ TypeCreateLogProbsFn = Callable[
40
[TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs]
41
42
42
-def parse_prompt_format(prompt) -> Tuple[bool, list]:
43
- # get the prompt, openai supports the following
44
- # "a string, array of strings, array of tokens, or array of token arrays."
45
- prompt_is_tokens = False
46
- prompts = [prompt] # case 1: a string
47
- if isinstance(prompt, list):
48
- if len(prompt) == 0:
49
- raise ValueError("please provide at least one prompt")
50
- elif isinstance(prompt[0], str):
51
- prompt_is_tokens = False
52
- prompts = prompt # case 2: array of strings
53
- elif isinstance(prompt[0], int):
54
- prompt_is_tokens = True
55
- prompts = [prompt] # case 3: array of tokens
56
- elif isinstance(prompt[0], list) and isinstance(prompt[0][0], int):
57
- prompt_is_tokens = True
58
- prompts = prompt # case 4: array of token arrays
59
- else:
60
- raise ValueError("prompt must be a string, array of strings, "
61
- "array of tokens, or array of token arrays")
62
- return prompt_is_tokens, prompts
63
-
64
-
43
class OpenAIServingCompletion(OpenAIServing):
44
67
- def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
68
- served_model_names: List[str],
69
- lora_modules: Optional[List[LoRAModulePath]],
70
- prompt_adapters: Optional[List[PromptAdapterPath]]):
45
+ def __init__(
46
+ self,
47
+ engine: AsyncLLMEngine,
48
+ model_config: ModelConfig,
49
+ served_model_names: List[str],
50
+ *,
51
+ lora_modules: Optional[List[LoRAModulePath]],
52
+ prompt_adapters: Optional[List[PromptAdapterPath]],
53
+ request_logger: Optional[RequestLogger],
54
+ ):
55
super().__init__(engine=engine,
56
model_config=model_config,
57
served_model_names=served_model_names,
58
lora_modules=lora_modules,
75
- prompt_adapters=prompt_adapters)
59
+ prompt_adapters=prompt_adapters,
60
+ request_logger=request_logger)
61
62
async def create_completion(self, request: CompletionRequest,
63
raw_request: Request):
@@ -101,12 +86,11 @@ class OpenAIServingCompletion(OpenAIServing):
86
# Schedule the request and get the result generator.
87
generators: List[AsyncIterator[RequestOutput]] = []
88
try:
104
- adapter_type, adapter_request = self._maybe_get_adapter(request)
105
- lora_request, prompt_adapter_request = None, None
106
- if adapter_type == 'LoRA':
107
- lora_request, prompt_adapter_request = adapter_request, None
108
- elif adapter_type == 'PromptAdapter':
109
- lora_request, prompt_adapter_request = None, adapter_request
89
+ (
90
+ lora_request,
91
+ prompt_adapter_request,
92
+ ) = self._maybe_get_adapters(request)
93
+
94
tokenizer = await self.engine.get_tokenizer(lora_request)
95
96
sampling_params = request.to_sampling_params()
@@ -122,17 +106,25 @@ class OpenAIServingCompletion(OpenAIServing):
106
sampling_params.logits_processors = []
107
sampling_params.logits_processors.append(
108
guided_decode_logit_processor)
125
- prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
109
127
- for i, prompt in enumerate(prompts):
128
- prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
129
- prompt_formats = await self._validate_prompt_and_tokenize(
110
+ prompts = list(
111
+ self._tokenize_prompt_input_or_inputs(
112
request,
113
tokenizer,
114
+ request.prompt,
115
truncate_prompt_tokens=sampling_params.
116
truncate_prompt_tokens,
134
- **{prompt_arg: prompt})
135
- prompt_ids, prompt_text = prompt_formats
117
+ add_special_tokens=request.add_special_tokens,
118
+ ))
119
+
120
+ for i, prompt_inputs in enumerate(prompts):
121
+ request_id_item = f"{request_id}-{i}"
122
+
123
+ self._log_inputs(request_id_item,
124
+ prompt_inputs,
125
+ params=sampling_params,
126
+ lora_request=lora_request,
127
+ prompt_adapter_request=prompt_adapter_request)
128
129
is_tracing_enabled = await self.engine.is_tracing_enabled()
130
trace_headers = None
@@ -143,12 +135,9 @@ class OpenAIServingCompletion(OpenAIServing):
135
log_tracing_disabled_warning()
136
137
generator = self.engine.generate(
146
- {
147
- "prompt": prompt_text,
148
- "prompt_token_ids": prompt_ids
149
- },
138
+ {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
139
sampling_params,
151
- f"{request_id}-{i}",
140
+ request_id_item,
141
lora_request=lora_request,
142
prompt_adapter_request=prompt_adapter_request,
143
trace_headers=trace_headers,
@@ -189,9 +178,27 @@ class OpenAIServingCompletion(OpenAIServing):
178
await self.engine.abort(f"{request_id}-{i}")
179
return self.create_error_response("Client disconnected")
180
final_res_batch[i] = res
181
+
182
+ for i, final_res in enumerate(final_res_batch):
183
+ assert final_res is not None
184
+
185
+ # The output should contain the input text
186
+ # We did not pass it into vLLM engine to avoid being redundant
187
+ # with the inputs token IDs
188
+ if final_res.prompt is None:
189
+ final_res.prompt = prompts[i]["prompt"]
190
+
191
+ final_res_batch_checked = cast(List[RequestOutput],
192
+ final_res_batch)
193
+
194
response = self.request_output_to_completion_response(
193
- final_res_batch, request, request_id, created_time, model_name,
194
- tokenizer)
195
+ final_res_batch_checked,
196
+ request,
197
+ request_id,
198
+ created_time,
199
+ model_name,
200
+ tokenizer,
201
+ )
202
except ValueError as e:
203
# TODO: Use a vllm-specific Validation Error
204
return self.create_error_response(str(e))
@@ -220,10 +227,10 @@ class OpenAIServingCompletion(OpenAIServing):
227
num_prompts: int,
228
tokenizer: PreTrainedTokenizer,
229
) -> AsyncGenerator[str, None]:
223
- assert request.n is not None
224
- previous_texts = [""] * request.n * num_prompts
225
- previous_num_tokens = [0] * request.n * num_prompts
226
- has_echoed = [False] * request.n * num_prompts
230
+ num_choices = 1 if request.n is None else request.n
231
+ previous_texts = [""] * num_choices * num_prompts
232
+ previous_num_tokens = [0] * num_choices * num_prompts
233
+ has_echoed = [False] * num_choices * num_prompts
234
235
try:
236
async for prompt_idx, res in result_generator:
@@ -234,7 +241,7 @@ class OpenAIServingCompletion(OpenAIServing):
241
raise StopAsyncIteration()
242
243
for output in res.outputs:
237
- i = output.index + prompt_idx * request.n
244
+ i = output.index + prompt_idx * num_choices
245
# TODO(simon): optimize the performance by avoiding full
246
# text O(n^2) sending.
247
@@ -343,8 +350,8 @@ class OpenAIServingCompletion(OpenAIServing):
350
choices: List[CompletionResponseChoice] = []
351
num_prompt_tokens = 0
352
num_generated_tokens = 0
353
+
354
for final_res in final_res_batch:
347
- assert final_res is not None
355
prompt_token_ids = final_res.prompt_token_ids
356
prompt_logprobs = final_res.prompt_logprobs
357
prompt_text = final_res.prompt
vllm/entrypoints/openai/serving_embedding.py
+55
-23
@@ -1,16 +1,16 @@
1
import base64
2
import time
3
-from typing import AsyncIterator, List, Optional, Tuple
3
+from typing import AsyncIterator, List, Optional, Tuple, cast
4
5
import numpy as np
6
from fastapi import Request
7
8
from vllm.config import ModelConfig
9
from vllm.engine.async_llm_engine import AsyncLLMEngine
10
+from vllm.entrypoints.logger import RequestLogger
11
from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
12
EmbeddingResponse,
13
EmbeddingResponseData, UsageInfo)
13
-from vllm.entrypoints.openai.serving_completion import parse_prompt_format
14
from vllm.entrypoints.openai.serving_engine import OpenAIServing
15
from vllm.logger import init_logger
16
from vllm.outputs import EmbeddingRequestOutput
@@ -28,11 +28,11 @@ def request_output_to_embedding_response(
28
data: List[EmbeddingResponseData] = []
29
num_prompt_tokens = 0
30
for idx, final_res in enumerate(final_res_batch):
31
- assert final_res is not None
31
prompt_token_ids = final_res.prompt_token_ids
32
embedding = final_res.outputs.embedding
33
if encoding_format == "base64":
35
- embedding = base64.b64encode(np.array(embedding))
34
+ embedding_bytes = np.array(embedding).tobytes()
35
+ embedding = base64.b64encode(embedding_bytes).decode("utf-8")
36
embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
37
data.append(embedding_data)
38
@@ -54,12 +54,20 @@ def request_output_to_embedding_response(
54
55
class OpenAIServingEmbedding(OpenAIServing):
56
57
- def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
58
- served_model_names: List[str]):
57
+ def __init__(
58
+ self,
59
+ engine: AsyncLLMEngine,
60
+ model_config: ModelConfig,
61
+ served_model_names: List[str],
62
+ *,
63
+ request_logger: Optional[RequestLogger],
64
+ ):
65
super().__init__(engine=engine,
66
model_config=model_config,
67
served_model_names=served_model_names,
62
- lora_modules=None)
68
+ lora_modules=None,
69
+ prompt_adapters=None,
70
+ request_logger=request_logger)
71
self._check_embedding_mode(model_config.embedding_mode)
72
73
async def create_embedding(self, request: EmbeddingRequest,
@@ -80,29 +88,47 @@ class OpenAIServingEmbedding(OpenAIServing):
88
"dimensions is currently not supported")
89
90
model_name = request.model
83
- request_id = f"cmpl-{random_uuid()}"
91
+ request_id = f"embd-{random_uuid()}"
92
created_time = int(time.monotonic())
93
94
# Schedule the request and get the result generator.
87
- generators = []
95
+ generators: List[AsyncIterator[EmbeddingRequestOutput]] = []
96
try:
89
- prompt_is_tokens, prompts = parse_prompt_format(request.input)
97
+ (
98
+ lora_request,
99
+ prompt_adapter_request,
100
+ ) = self._maybe_get_adapters(request)
101
+
102
+ tokenizer = await self.engine.get_tokenizer(lora_request)
103
+
104
pooling_params = request.to_pooling_params()
105
92
- tokenizer = await self.engine.get_tokenizer()
93
- for i, prompt in enumerate(prompts):
94
- prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
95
- prompt_formats = await self._validate_prompt_and_tokenize(
96
- request, tokenizer, **{prompt_arg: prompt})
97
- prompt_ids, prompt_text = prompt_formats
106
+ prompts = list(
107
+ self._tokenize_prompt_input_or_inputs(
108
+ request,
109
+ tokenizer,
110
+ request.input,
111
+ ))
112
+
113
+ for i, prompt_inputs in enumerate(prompts):
114
+ request_id_item = f"{request_id}-{i}"
115
+
116
+ self._log_inputs(request_id_item,
117
+ prompt_inputs,
118
+ params=pooling_params,
119
+ lora_request=lora_request,
120
+ prompt_adapter_request=prompt_adapter_request)
121
+
122
+ if prompt_adapter_request is not None:
123
+ raise NotImplementedError(
124
+ "Prompt adapter is not supported "
125
+ "for embedding models")
126
127
generator = self.engine.encode(
100
- {
101
- "prompt": prompt_text,
102
- "prompt_token_ids": prompt_ids
103
- },
128
+ {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
129
pooling_params,
105
- f"{request_id}-{i}",
130
+ request_id_item,
131
+ lora_request=lora_request,
132
)
133
134
generators.append(generator)
@@ -121,11 +147,17 @@ class OpenAIServingEmbedding(OpenAIServing):
147
if await raw_request.is_disconnected():
148
# Abort the request if the client disconnects.
149
await self.engine.abort(f"{request_id}-{i}")
124
- # TODO: Use a vllm-specific Validation Error
150
return self.create_error_response("Client disconnected")
151
final_res_batch[i] = res
152
+
153
+ for final_res in final_res_batch:
154
+ assert final_res is not None
155
+
156
+ final_res_batch_checked = cast(List[EmbeddingRequestOutput],
157
+ final_res_batch)
158
+
159
response = request_output_to_embedding_response(
128
- final_res_batch, request_id, created_time, model_name,
160
+ final_res_batch_checked, request_id, created_time, model_name,
161
encoding_format)
162
except ValueError as e:
163
# TODO: Use a vllm-specific Validation Error
vllm/entrypoints/openai/serving_engine.py
+209
-62
@@ -2,23 +2,33 @@ import json
2
import pathlib
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
-from typing import Any, Dict, List, Optional, Tuple, Union
5
+from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union
6
7
from pydantic import Field
8
-from transformers import PreTrainedTokenizer
8
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
9
from typing_extensions import Annotated
10
11
from vllm.config import ModelConfig
12
from vllm.engine.async_llm_engine import AsyncLLMEngine
13
+from vllm.entrypoints.logger import RequestLogger
14
+# yapf conflicts with isort for this block
15
+# yapf: disable
16
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
17
CompletionRequest,
18
DetokenizeRequest,
19
EmbeddingRequest, ErrorResponse,
20
ModelCard, ModelList,
18
- ModelPermission, TokenizeRequest)
21
+ ModelPermission,
22
+ TokenizeChatRequest,
23
+ TokenizeCompletionRequest,
24
+ TokenizeRequest)
25
+# yapf: enable
26
+from vllm.inputs import parse_and_batch_prompt
27
from vllm.logger import init_logger
28
from vllm.lora.request import LoRARequest
29
+from vllm.pooling_params import PoolingParams
30
from vllm.prompt_adapter.request import PromptAdapterRequest
31
+from vllm.sampling_params import SamplingParams
32
from vllm.sequence import Logprob
33
34
logger = init_logger(__name__)
@@ -36,6 +46,17 @@ class LoRAModulePath:
46
local_path: str
47
48
49
+AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
50
+ EmbeddingRequest, TokenizeRequest]
51
+
52
+AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
53
+
54
+
55
+class TextTokensPrompt(TypedDict):
56
+ prompt: str
57
+ prompt_token_ids: List[int]
58
+
59
+
60
class OpenAIServing:
61
62
def __init__(
@@ -43,8 +64,10 @@ class OpenAIServing:
64
engine: AsyncLLMEngine,
65
model_config: ModelConfig,
66
served_model_names: List[str],
67
+ *,
68
lora_modules: Optional[List[LoRAModulePath]],
47
- prompt_adapters: Optional[List[PromptAdapterPath]] = None,
69
+ prompt_adapters: Optional[List[PromptAdapterPath]],
70
+ request_logger: Optional[RequestLogger],
71
):
72
super().__init__()
73
@@ -78,6 +101,8 @@ class OpenAIServing:
101
prompt_adapter_local_path=prompt_adapter.local_path,
102
prompt_adapter_num_virtual_tokens=num_virtual_tokens))
103
104
+ self.request_logger = request_logger
105
+
106
async def show_available_models(self) -> ModelList:
107
"""Show available models. Right now we only have one model."""
108
model_cards = [
@@ -126,9 +151,8 @@ class OpenAIServing:
151
return json_str
152
153
async def _check_model(
129
- self, request: Union[ChatCompletionRequest, CompletionRequest,
130
- DetokenizeRequest, EmbeddingRequest,
131
- TokenizeRequest]
154
+ self,
155
+ request: AnyRequest,
156
) -> Optional[ErrorResponse]:
157
if request.model in self.served_model_names:
158
return None
@@ -144,64 +168,65 @@ class OpenAIServing:
168
err_type="NotFoundError",
169
status_code=HTTPStatus.NOT_FOUND)
170
147
- def _maybe_get_adapter(
148
- self, request: Union[CompletionRequest, ChatCompletionRequest,
149
- EmbeddingRequest, TokenizeRequest,
150
- DetokenizeRequest]
151
- ) -> Tuple[Optional[str], Optional[Union[LoRARequest,
152
- PromptAdapterRequest]]]:
171
+ def _maybe_get_adapters(
172
+ self, request: AnyRequest
173
+ ) -> Union[Tuple[None, None], Tuple[LoRARequest, None], Tuple[
174
+ None, PromptAdapterRequest]]:
175
if request.model in self.served_model_names:
176
return None, None
177
for lora in self.lora_requests:
178
if request.model == lora.lora_name:
157
- return 'LoRA', lora
179
+ return lora, None
180
for prompt_adapter in self.prompt_adapter_requests:
181
if request.model == prompt_adapter.prompt_adapter_name:
160
- return 'PromptAdapter', prompt_adapter
182
+ return None, prompt_adapter
183
# if _check_model has been called earlier, this will be unreachable
184
raise ValueError(f"The model `{request.model}` does not exist.")
185
164
- async def _validate_prompt_and_tokenize(
165
- self,
166
- request: Union[ChatCompletionRequest, CompletionRequest,
167
- DetokenizeRequest, EmbeddingRequest,
168
- TokenizeRequest],
169
- tokenizer: "PreTrainedTokenizer",
170
- prompt: Optional[str] = None,
171
- prompt_ids: Optional[List[int]] = None,
172
- truncate_prompt_tokens: Optional[Annotated[int,
173
- Field(ge=1)]] = None,
174
- add_special_tokens: Optional[bool] = True
175
- ) -> Tuple[List[int], str]:
176
- if not (prompt or prompt_ids):
177
- raise ValueError("Either prompt or prompt_ids should be provided.")
178
- if prompt and prompt_ids:
179
- raise ValueError(
180
- "Only one of prompt or prompt_ids should be provided.")
181
-
182
- if prompt_ids is None:
183
- # When using OpenAIServingChat for chat completions, for
184
- # most models the special tokens (e.g., BOS) have already
185
- # been added by the chat template. Therefore, we do not
186
- # need to add them again.
187
- # Set add_special_tokens to False (by default) to avoid
188
- # adding the BOS tokens again.
189
- tokenizer_kwargs: Dict[str, Any] = {
190
- "add_special_tokens": add_special_tokens
191
- }
192
- if truncate_prompt_tokens is not None:
193
- tokenizer_kwargs.update({
194
- "truncation": True,
195
- "max_length": truncate_prompt_tokens,
196
- })
197
- input_ids = tokenizer(prompt, **tokenizer_kwargs).input_ids
198
- elif truncate_prompt_tokens is not None:
199
- input_ids = prompt_ids[-truncate_prompt_tokens:]
186
+ def _normalize_prompt_text_to_input(
187
+ self,
188
+ request: AnyRequest,
189
+ tokenizer: AnyTokenizer,
190
+ prompt: str,
191
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
192
+ add_special_tokens: bool,
193
+ ) -> TextTokensPrompt:
194
+ if truncate_prompt_tokens is None:
195
+ encoded = tokenizer(prompt, add_special_tokens=add_special_tokens)
196
else:
197
+ encoded = tokenizer(prompt,
198
+ add_special_tokens=add_special_tokens,
199
+ truncation=True,
200
+ max_length=truncate_prompt_tokens)
201
+
202
+ input_ids = encoded.input_ids
203
+
204
+ input_text = prompt
205
+
206
+ return self._validate_input(request, input_ids, input_text)
207
+
208
+ def _normalize_prompt_tokens_to_input(
209
+ self,
210
+ request: AnyRequest,
211
+ tokenizer: AnyTokenizer,
212
+ prompt_ids: List[int],
213
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
214
+ ) -> TextTokensPrompt:
215
+ if truncate_prompt_tokens is None:
216
input_ids = prompt_ids
217
+ else:
218
+ input_ids = prompt_ids[-truncate_prompt_tokens:]
219
+
220
+ input_text = tokenizer.decode(input_ids)
221
203
- input_text = prompt if prompt is not None else tokenizer.decode(
204
- input_ids)
222
+ return self._validate_input(request, input_ids, input_text)
223
+
224
+ def _validate_input(
225
+ self,
226
+ request: AnyRequest,
227
+ input_ids: List[int],
228
+ input_text: str,
229
+ ) -> TextTokensPrompt:
230
token_num = len(input_ids)
231
232
# Note: EmbeddingRequest doesn't have max_tokens
@@ -211,13 +236,16 @@ class OpenAIServing:
236
f"This model's maximum context length is "
237
f"{self.max_model_len} tokens. However, you requested "
238
f"{token_num} tokens in the input for embedding "
214
- f"generation. Please reduce the length of the input.", )
215
- return input_ids, input_text
239
+ f"generation. Please reduce the length of the input.")
240
+ return TextTokensPrompt(prompt=input_text,
241
+ prompt_token_ids=input_ids)
242
243
# Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
244
# and does not require model context length validation
219
- if isinstance(request, (TokenizeRequest, DetokenizeRequest)):
220
- return input_ids, input_text
245
+ if isinstance(request, (TokenizeCompletionRequest, TokenizeChatRequest,
246
+ DetokenizeRequest)):
247
+ return TextTokensPrompt(prompt=input_text,
248
+ prompt_token_ids=input_ids)
249
250
if request.max_tokens is None:
251
if token_num >= self.max_model_len:
@@ -225,7 +253,7 @@ class OpenAIServing:
253
f"This model's maximum context length is "
254
f"{self.max_model_len} tokens. However, you requested "
255
f"{token_num} tokens in the messages, "
228
- f"Please reduce the length of the messages.", )
256
+ f"Please reduce the length of the messages.")
257
request.max_tokens = self.max_model_len - token_num
258
259
if token_num + request.max_tokens > self.max_model_len:
@@ -235,13 +263,132 @@ class OpenAIServing:
263
f"{request.max_tokens + token_num} tokens "
264
f"({token_num} in the messages, "
265
f"{request.max_tokens} in the completion). "
238
- f"Please reduce the length of the messages or completion.", )
266
+ f"Please reduce the length of the messages or completion.")
267
+
268
+ return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
269
+
270
+ def _tokenize_prompt_input(
271
+ self,
272
+ request: AnyRequest,
273
+ tokenizer: AnyTokenizer,
274
+ prompt_input: Union[str, List[int]],
275
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
276
+ add_special_tokens: bool = True,
277
+ ) -> TextTokensPrompt:
278
+ """
279
+ A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs`
280
+ that assumes single input.
281
+ """
282
+ return next(
283
+ self._tokenize_prompt_inputs(
284
+ request,
285
+ tokenizer,
286
+ [prompt_input],
287
+ truncate_prompt_tokens=truncate_prompt_tokens,
288
+ add_special_tokens=add_special_tokens,
289
+ ))
290
+
291
+ def _tokenize_prompt_inputs(
292
+ self,
293
+ request: AnyRequest,
294
+ tokenizer: AnyTokenizer,
295
+ prompt_inputs: Iterable[Union[str, List[int]]],
296
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
297
+ add_special_tokens: bool = True,
298
+ ) -> Iterator[TextTokensPrompt]:
299
+ """
300
+ A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs`
301
+ that assumes multiple inputs.
302
+ """
303
+ for text in prompt_inputs:
304
+ if isinstance(text, str):
305
+ yield self._normalize_prompt_text_to_input(
306
+ request,
307
+ tokenizer,
308
+ prompt=text,
309
+ truncate_prompt_tokens=truncate_prompt_tokens,
310
+ add_special_tokens=add_special_tokens,
311
+ )
312
+ else:
313
+ yield self._normalize_prompt_tokens_to_input(
314
+ request,
315
+ tokenizer,
316
+ prompt_ids=text,
317
+ truncate_prompt_tokens=truncate_prompt_tokens,
318
+ )
319
+
320
+ def _tokenize_prompt_input_or_inputs(
321
+ self,
322
+ request: AnyRequest,
323
+ tokenizer: AnyTokenizer,
324
+ input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
325
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
326
+ add_special_tokens: bool = True,
327
+ ) -> Iterator[TextTokensPrompt]:
328
+ """
329
+ Tokenize/detokenize depending on the input format.
330
+
331
+ According to `OpenAI API <https://platform.openai.com/docs/api-reference/embeddings/create>`_
332
+ , each input can be a string or array of tokens. Note that each request
333
+ can pass one or more inputs.
334
+ """
335
+ for prompt_input in parse_and_batch_prompt(input_or_inputs):
336
+ # Although our type checking is based on mypy,
337
+ # VSCode Pyright extension should still work properly
338
+ # "is True" is required for Pyright to perform type narrowing
339
+ # See: https://github.com/microsoft/pyright/issues/7672
340
+ if prompt_input["is_tokens"] is False:
341
+ yield self._normalize_prompt_text_to_input(
342
+ request,
343
+ tokenizer,
344
+ prompt=prompt_input["content"],
345
+ truncate_prompt_tokens=truncate_prompt_tokens,
346
+ add_special_tokens=add_special_tokens,
347
+ )
348
+ else:
349
+ yield self._normalize_prompt_tokens_to_input(
350
+ request,
351
+ tokenizer,
352
+ prompt_ids=prompt_input["content"],
353
+ truncate_prompt_tokens=truncate_prompt_tokens,
354
+ )
355
+
356
+ def _log_inputs(
357
+ self,
358
+ request_id: str,
359
+ inputs: Union[str, List[int], TextTokensPrompt],
360
+ params: Optional[Union[SamplingParams, PoolingParams]],
361
+ lora_request: Optional[LoRARequest],
362
+ prompt_adapter_request: Optional[PromptAdapterRequest],
363
+ ) -> None:
364
+ if self.request_logger is None:
365
+ return
366
+
367
+ if isinstance(inputs, str):
368
+ prompt = inputs
369
+ prompt_token_ids = None
370
+ elif isinstance(inputs, list):
371
+ prompt = None
372
+ prompt_token_ids = inputs
373
else:
240
- return input_ids, input_text
374
+ prompt = inputs["prompt"]
375
+ prompt_token_ids = inputs["prompt_token_ids"]
376
+
377
+ self.request_logger.log_inputs(
378
+ request_id,
379
+ prompt,
380
+ prompt_token_ids,
381
+ params=params,
382
+ lora_request=lora_request,
383
+ prompt_adapter_request=prompt_adapter_request,
384
+ )
385
386
@staticmethod
243
- def _get_decoded_token(logprob: Logprob, token_id: int,
244
- tokenizer: PreTrainedTokenizer) -> str:
387
+ def _get_decoded_token(
388
+ logprob: Logprob,
389
+ token_id: int,
390
+ tokenizer: AnyTokenizer,
391
+ ) -> str:
392
if logprob.decoded_token is not None:
393
return logprob.decoded_token
394
return tokenizer.decode(token_id)
vllm/entrypoints/openai/serving_tokenization.py
+79
-27
@@ -1,83 +1,135 @@
1
-from typing import List, Optional
1
+from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
from vllm.engine.async_llm_engine import AsyncLLMEngine
5
+# yapf conflicts with isort for this block
6
+# yapf: disable
7
from vllm.entrypoints.chat_utils import (ConversationMessage,
8
load_chat_template,
9
parse_chat_message_content)
10
+from vllm.entrypoints.logger import RequestLogger
11
from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
12
DetokenizeResponse,
13
+ ErrorResponse,
14
+ TokenizeChatRequest,
15
TokenizeRequest,
16
TokenizeResponse)
17
+# yapf: enable
18
from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
19
OpenAIServing)
20
+from vllm.utils import random_uuid
21
22
23
class OpenAIServingTokenization(OpenAIServing):
24
18
- def __init__(self,
19
- engine: AsyncLLMEngine,
20
- model_config: ModelConfig,
21
- served_model_names: List[str],
22
- lora_modules: Optional[List[LoRAModulePath]] = None,
23
- chat_template: Optional[str] = None):
25
+ def __init__(
26
+ self,
27
+ engine: AsyncLLMEngine,
28
+ model_config: ModelConfig,
29
+ served_model_names: List[str],
30
+ *,
31
+ lora_modules: Optional[List[LoRAModulePath]],
32
+ request_logger: Optional[RequestLogger],
33
+ chat_template: Optional[str],
34
+ ):
35
super().__init__(engine=engine,
36
model_config=model_config,
37
served_model_names=served_model_names,
27
- lora_modules=lora_modules)
38
+ lora_modules=lora_modules,
39
+ prompt_adapters=None,
40
+ request_logger=request_logger)
41
42
# If this is None we use the tokenizer's default chat template
43
self.chat_template = load_chat_template(chat_template)
44
32
- async def create_tokenize(self,
33
- request: TokenizeRequest) -> TokenizeResponse:
45
+ async def create_tokenize(
46
+ self,
47
+ request: TokenizeRequest,
48
+ ) -> Union[TokenizeResponse, ErrorResponse]:
49
error_check_ret = await self._check_model(request)
50
if error_check_ret is not None:
51
return error_check_ret
52
38
- if not (request.prompt or request.messages):
39
- return self.create_error_response(
40
- "Either `prompt` or `messages` should be provided.")
53
+ request_id = f"tokn-{random_uuid()}"
54
42
- if (request.prompt and request.messages):
43
- return self.create_error_response(
44
- "Only one of `prompt` or `messages` should be provided.")
55
+ (
56
+ lora_request,
57
+ prompt_adapter_request,
58
+ ) = self._maybe_get_adapters(request)
59
46
- _, lora_request = self._maybe_get_adapter(request)
60
tokenizer = await self.engine.get_tokenizer(lora_request)
48
- if request.messages:
61
+
62
+ if isinstance(request, TokenizeChatRequest):
63
+ model_config = self.model_config
64
+
65
conversation: List[ConversationMessage] = []
66
67
for message in request.messages:
52
- result = parse_chat_message_content(message, self.model_config,
68
+ result = parse_chat_message_content(message, model_config,
69
tokenizer)
70
conversation.extend(result.messages)
71
56
- request.prompt = tokenizer.apply_chat_template(
72
+ prompt = tokenizer.apply_chat_template(
73
add_generation_prompt=request.add_generation_prompt,
74
conversation=conversation,
75
tokenize=False,
76
chat_template=self.chat_template)
77
+ assert isinstance(prompt, str)
78
+ else:
79
+ prompt = request.prompt
80
+
81
+ self._log_inputs(request_id,
82
+ prompt,
83
+ params=None,
84
+ lora_request=lora_request,
85
+ prompt_adapter_request=prompt_adapter_request)
86
62
- (input_ids, input_text) = await self._validate_prompt_and_tokenize(
87
+ # Silently ignore prompt adapter since it does not affect tokenization
88
+
89
+ prompt_input = self._tokenize_prompt_input(
90
request,
91
tokenizer,
65
- prompt=request.prompt,
66
- add_special_tokens=request.add_special_tokens)
92
+ prompt,
93
+ add_special_tokens=request.add_special_tokens,
94
+ )
95
+ input_ids = prompt_input["prompt_token_ids"]
96
97
return TokenizeResponse(tokens=input_ids,
98
count=len(input_ids),
99
max_model_len=self.max_model_len)
100
101
async def create_detokenize(
73
- self, request: DetokenizeRequest) -> DetokenizeResponse:
102
+ self,
103
+ request: DetokenizeRequest,
104
+ ) -> Union[DetokenizeResponse, ErrorResponse]:
105
error_check_ret = await self._check_model(request)
106
if error_check_ret is not None:
107
return error_check_ret
108
78
- _, lora_request = self._maybe_get_adapter(request)
109
+ request_id = f"tokn-{random_uuid()}"
110
+
111
+ (
112
+ lora_request,
113
+ prompt_adapter_request,
114
+ ) = self._maybe_get_adapters(request)
115
+
116
tokenizer = await self.engine.get_tokenizer(lora_request)
80
- (input_ids, input_text) = await self._validate_prompt_and_tokenize(
81
- request, tokenizer, prompt_ids=request.tokens)
117
+
118
+ self._log_inputs(request_id,
119
+ request.tokens,
120
+ params=None,
121
+ lora_request=lora_request,
122
+ prompt_adapter_request=prompt_adapter_request)
123
+
124
+ if prompt_adapter_request is not None:
125
+ raise NotImplementedError("Prompt adapter is not supported "
126
+ "for tokenization")
127
+
128
+ prompt_input = self._tokenize_prompt_input(
129
+ request,
130
+ tokenizer,
131
+ request.tokens,
132
+ )
133
+ input_text = prompt_input["prompt"]
134
135
return DetokenizeResponse(prompt=input_text)
vllm/inputs/__init__.py
+3
-4
@@ -1,6 +1,5 @@
1
from .data import (LLMInputs, ParsedText, ParsedTokens, PromptInputs,
2
- PromptStrictInputs, TextPrompt, TextTokensPrompt,
3
- TokensPrompt, parse_and_batch_prompt)
2
+ TextPrompt, TokensPrompt, parse_and_batch_prompt)
3
from .registry import InputContext, InputRegistry
4
5
INPUT_REGISTRY = InputRegistry()
@@ -14,6 +13,6 @@ See also:
13
14
__all__ = [
15
"ParsedText", "ParsedTokens", "parse_and_batch_prompt", "TextPrompt",
17
- "TokensPrompt", "TextTokensPrompt", "PromptStrictInputs", "PromptInputs",
18
- "LLMInputs", "INPUT_REGISTRY", "InputContext", "InputRegistry"
16
+ "TokensPrompt", "PromptInputs", "LLMInputs", "INPUT_REGISTRY",
17
+ "InputContext", "InputRegistry"
18
]
vllm/inputs/data.py
+1
-23
@@ -92,25 +92,7 @@ class TokensPrompt(TypedDict):
92
"""
93
94
95
-class TextTokensPrompt(TypedDict):
96
- """It is assumed that :attr:`prompt` is consistent with
97
- :attr:`prompt_token_ids`. This is currently used in
98
- :class:`AsyncLLMEngine` for logging both the text and token IDs."""
99
-
100
- prompt: str
101
- """The prompt text."""
102
-
103
- prompt_token_ids: List[int]
104
- """The token IDs of the prompt."""
105
-
106
- multi_modal_data: NotRequired["MultiModalDataDict"]
107
- """
108
- Optional multi-modal data to pass to the model,
109
- if the model supports it.
110
- """
111
-
112
-
113
-PromptStrictInputs = Union[str, TextPrompt, TokensPrompt]
95
+PromptInputs = Union[str, TextPrompt, TokensPrompt]
96
"""
97
The inputs to the LLM, which can take one of the following forms:
98
@@ -118,10 +100,6 @@ The inputs to the LLM, which can take one of the following forms:
100
- A tokenized prompt (:class:`TokensPrompt`)
101
"""
102
121
-PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
122
-"""Same as :const:`PromptStrictInputs` but additionally accepts
123
-:class:`TextTokensPrompt`."""
124
-
103
104
class LLMInputs(TypedDict):
105
"""
vllm/sequence.py
+3
-2
@@ -5,7 +5,8 @@ import math
5
from abc import ABC, abstractmethod
6
from collections import defaultdict
7
from dataclasses import dataclass, field
8
-from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
8
+from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
9
+ Union)
10
11
import torch
12
@@ -438,7 +439,7 @@ class SequenceGroup:
439
embeddings: Optional[List[float]] = None,
440
pooling_params: Optional[PoolingParams] = None,
441
encoder_seq: Optional[Sequence] = None,
441
- trace_headers: Optional[Dict[str, str]] = None,
442
+ trace_headers: Optional[Mapping[str, str]] = None,
443
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
444
) -> None:
445
self.request_id = request_id