@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 739b61a348
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Frontend] Refactor prompt processing (#4028)

Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Jul 23, 2024 at 01:13 UTC 739b61a348afa5da297a80ff15f4e39d6e524b53
24 files changed +698 -390
benchmarks/benchmark_latency.py
+2 -2
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import EngineArgs
14 -from vllm.inputs import PromptStrictInputs
14 +from vllm.inputs import PromptInputs
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61 dummy_prompt_token_ids = np.random.randint(10000,
62 size=(args.batch_size,
63 args.input_len))
64 - dummy_inputs: List[PromptStrictInputs] = [{
64 + dummy_inputs: List[PromptInputs] = [{
65 "prompt_token_ids": batch
66 } for batch in dummy_prompt_token_ids.tolist()]
67
docs/source/dev/multimodal/multimodal_index.rst
+1 -1
@@ -8,7 +8,7 @@ Multi-Modality
8 vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10 Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11 -via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptStrictInputs`.
11 +via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
12
13 Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1 -1
@@ -1,7 +1,7 @@
1 LLM Inputs
2 ==========
3
4 -.. autodata:: vllm.inputs.PromptStrictInputs
4 +.. autodata:: vllm.inputs.PromptInputs
5
6 .. autoclass:: vllm.inputs.TextPrompt
7 :show-inheritance:
docs/source/models/vlm.rst
+1 -1
@@ -30,7 +30,7 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
30 internally for each model.
31
32
33 -To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
33 +To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
34
35 * ``prompt``: The prompt should follow the format that is documented on HuggingFace.
36 * ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/engine/output_processor/test_stop_checker.py
+2 -2
@@ -35,8 +35,8 @@ def sequence_with_eos(text: str, eos_token: str,
35 @pytest.mark.parametrize(["text_wo_eos", "eos_token", "eos_token_id"], [
36 ("This text ends with EOS token", "</s>", 2),
37 ])
38 -@pytest.mark.parametrize("ignore_eos", [True, False, None])
39 -@pytest.mark.parametrize("include_stop_str_in_output", [True, False, None])
38 +@pytest.mark.parametrize("ignore_eos", [True, False])
39 +@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
40 @pytest.mark.skip_global_cleanup
41 def test_stop_on_eos_token(text_wo_eos: str, eos_token: str, eos_token_id: int,
42 ignore_eos: bool, include_stop_str_in_output: bool):
tests/entrypoints/openai/test_serving_chat.py
+4 -1
@@ -32,7 +32,10 @@ async def _async_serving_chat_init():
32 model_config,
33 served_model_names=[MODEL_NAME],
34 response_role="assistant",
35 - chat_template=CHAT_TEMPLATE)
35 + chat_template=CHAT_TEMPLATE,
36 + lora_modules=None,
37 + prompt_adapters=None,
38 + request_logger=None)
39 return serving_completion
40
41
vllm/__init__.py
+2 -2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 -from vllm.inputs import PromptStrictInputs, TextPrompt, TokensPrompt
8 +from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19 "__version__",
20 "LLM",
21 "ModelRegistry",
22 - "PromptStrictInputs",
22 + "PromptInputs",
23 "TextPrompt",
24 "TokensPrompt",
25 "SamplingParams",
vllm/engine/arg_utils.py
-7
@@ -827,7 +827,6 @@ class AsyncEngineArgs(EngineArgs):
827 """Arguments for asynchronous vLLM engine."""
828 engine_use_ray: bool = False
829 disable_log_requests: bool = False
830 - max_log_len: Optional[int] = None
830
831 @staticmethod
832 def add_cli_args(parser: FlexibleArgumentParser,
@@ -841,12 +840,6 @@ class AsyncEngineArgs(EngineArgs):
840 parser.add_argument('--disable-log-requests',
841 action='store_true',
842 help='Disable logging requests.')
844 - parser.add_argument('--max-log-len',
845 - type=int,
846 - default=None,
847 - help='Max number of prompt characters or prompt '
848 - 'ID numbers being printed in log.'
849 - '\n\nDefault: Unlimited')
843 return parser
844
845
vllm/engine/async_llm_engine.py
+22 -41
@@ -1,8 +1,8 @@
1 import asyncio
2 import time
3 from functools import partial
4 -from typing import (AsyncIterator, Callable, Dict, Iterable, List, Optional,
5 - Set, Tuple, Type, Union)
4 +from typing import (AsyncIterator, Callable, Dict, Iterable, List, Mapping,
5 + Optional, Set, Tuple, Type, Union)
6
7 from transformers import PreTrainedTokenizer
8
@@ -151,7 +151,10 @@ class RequestTracker:
151 logger.info("Finished request %s.", request_id)
152 self.abort_request(request_id)
153
154 - def add_request(self, request_id: str,
154 + def add_request(self,
155 + request_id: str,
156 + *,
157 + verbose: bool = False,
158 **engine_add_request_kwargs) -> AsyncStream:
159 """Add a request to be sent to the engine on the next background
160 loop iteration."""
@@ -166,6 +169,9 @@ class RequestTracker:
169
170 self.new_requests_event.set()
171
172 + if verbose:
173 + logger.info("Added request %s.", request_id)
174 +
175 return stream
176
177 def abort_request(self, request_id: str, *, verbose: bool = False) -> None:
@@ -299,14 +305,14 @@ class _AsyncLLMEngine(LLMEngine):
305 return self.input_processor(llm_inputs)
306
307 async def add_request_async(
302 - self,
303 - request_id: str,
304 - inputs: PromptInputs,
305 - params: Union[SamplingParams, PoolingParams],
306 - arrival_time: Optional[float] = None,
307 - lora_request: Optional[LoRARequest] = None,
308 - trace_headers: Optional[Dict[str, str]] = None,
309 - prompt_adapter_request: Optional[PromptAdapterRequest] = None
308 + self,
309 + request_id: str,
310 + inputs: PromptInputs,
311 + params: Union[SamplingParams, PoolingParams],
312 + arrival_time: Optional[float] = None,
313 + lora_request: Optional[LoRARequest] = None,
314 + trace_headers: Optional[Mapping[str, str]] = None,
315 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
316 ) -> None:
317 if lora_request is not None and not self.lora_config:
318 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
@@ -353,8 +359,6 @@ class AsyncLLMEngine:
359 async frontend will be executed in a separate process as the
360 model workers.
361 log_requests: Whether to log the requests.
356 - max_log_len: Maximum number of prompt characters or prompt ID numbers
357 - being printed in log.
362 start_engine_loop: If True, the background task to run the engine
363 will be automatically started in the generate call.
364 *args: Arguments for :class:`LLMEngine`.
@@ -368,13 +372,11 @@ class AsyncLLMEngine:
372 engine_use_ray: bool,
373 *args,
374 log_requests: bool = True,
371 - max_log_len: Optional[int] = None,
375 start_engine_loop: bool = True,
376 **kwargs) -> None:
377 self.worker_use_ray = worker_use_ray
378 self.engine_use_ray = engine_use_ray
379 self.log_requests = log_requests
377 - self.max_log_len = max_log_len
380 self.engine = self._init_engine(*args, **kwargs)
381
382 self.background_loop: Optional[asyncio.Future] = None
@@ -468,7 +470,6 @@ class AsyncLLMEngine:
470 executor_class=executor_class,
471 log_requests=not engine_args.disable_log_requests,
472 log_stats=not engine_args.disable_log_stats,
471 - max_log_len=engine_args.max_log_len,
473 start_engine_loop=start_engine_loop,
474 usage_context=usage_context,
475 stat_loggers=stat_loggers,
@@ -667,30 +668,9 @@ class AsyncLLMEngine:
668 params: Union[SamplingParams, PoolingParams],
669 arrival_time: Optional[float] = None,
670 lora_request: Optional[LoRARequest] = None,
670 - trace_headers: Optional[Dict[str, str]] = None,
671 + trace_headers: Optional[Mapping[str, str]] = None,
672 prompt_adapter_request: Optional[PromptAdapterRequest] = None
673 ) -> AsyncStream:
673 - if self.log_requests:
674 - if isinstance(inputs, str):
675 - shortened_prompt = inputs
676 - shortened_token_ids = None
677 - else:
678 - shortened_prompt = inputs.get("prompt")
679 - shortened_token_ids = inputs.get("prompt_token_ids")
680 -
681 - max_log_len = self.max_log_len
682 - if max_log_len is not None:
683 - if shortened_prompt is not None:
684 - shortened_prompt = shortened_prompt[:max_log_len]
685 - if shortened_token_ids is not None:
686 - shortened_token_ids = shortened_token_ids[:max_log_len]
687 -
688 - logger.info(
689 - "Received request %s: prompt: %r, "
690 - "params: %s, prompt_token_ids: %s, "
691 - "lora_request: %s.", request_id, shortened_prompt, params,
692 - shortened_token_ids, lora_request)
693 -
674 if not self.is_running:
675 if self.start_engine_loop:
676 self.start_background_loop()
@@ -706,6 +686,7 @@ class AsyncLLMEngine:
686
687 stream = self._request_tracker.add_request(
688 request_id,
689 + verbose=self.log_requests,
690 inputs=inputs,
691 params=params,
692 arrival_time=arrival_time,
@@ -721,7 +702,7 @@ class AsyncLLMEngine:
702 sampling_params: SamplingParams,
703 request_id: str,
704 lora_request: Optional[LoRARequest] = None,
724 - trace_headers: Optional[Dict[str, str]] = None,
705 + trace_headers: Optional[Mapping[str, str]] = None,
706 prompt_adapter_request: Optional[PromptAdapterRequest] = None
707 ) -> AsyncIterator[RequestOutput]:
708 """Generate outputs for a request.
@@ -804,7 +785,7 @@ class AsyncLLMEngine:
785 pooling_params: PoolingParams,
786 request_id: str,
787 lora_request: Optional[LoRARequest] = None,
807 - trace_headers: Optional[Dict[str, str]] = None,
788 + trace_headers: Optional[Mapping[str, str]] = None,
789 ) -> AsyncIterator[EmbeddingRequestOutput]:
790 """Generate outputs for a request from an embedding model.
791
@@ -882,7 +863,7 @@ class AsyncLLMEngine:
863 params: Union[SamplingParams, PoolingParams],
864 *,
865 lora_request: Optional[LoRARequest] = None,
885 - trace_headers: Optional[Dict[str, str]] = None,
866 + trace_headers: Optional[Mapping[str, str]] = None,
867 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
868 ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
869 """Common logic to process requests with SamplingParams or
vllm/engine/llm_engine.py
+5 -4
@@ -1,6 +1,7 @@
1 import time
2 from contextlib import contextmanager
3 -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List, Optional
3 +from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
4 + Mapping, Optional)
5 from typing import Sequence as GenericSequence
6 from typing import Set, Type, TypeVar, Union
7
@@ -522,7 +523,7 @@ class LLMEngine:
523 arrival_time: float,
524 lora_request: Optional[LoRARequest],
525 prompt_adapter_request: Optional[PromptAdapterRequest],
525 - trace_headers: Optional[Dict[str, str]] = None,
526 + trace_headers: Optional[Mapping[str, str]] = None,
527 ) -> None:
528 # Create the sequences.
529 block_size = self.cache_config.block_size
@@ -603,7 +604,7 @@ class LLMEngine:
604 params: Union[SamplingParams, PoolingParams],
605 arrival_time: Optional[float] = None,
606 lora_request: Optional[LoRARequest] = None,
606 - trace_headers: Optional[Dict[str, str]] = None,
607 + trace_headers: Optional[Mapping[str, str]] = None,
608 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
609 ) -> None:
610 """Add a request to the engine's request pool.
@@ -677,7 +678,7 @@ class LLMEngine:
678 sampling_params: SamplingParams,
679 arrival_time: float,
680 lora_request: Optional[LoRARequest],
680 - trace_headers: Optional[Dict[str, str]] = None,
681 + trace_headers: Optional[Mapping[str, str]] = None,
682 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
683 ) -> SequenceGroup:
684 """Creates a SequenceGroup with SamplingParams."""
vllm/entrypoints/llm.py
+13 -24
@@ -6,8 +6,7 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 -from vllm.inputs import (PromptInputs, PromptStrictInputs, TextPrompt,
10 - TextTokensPrompt, TokensPrompt,
9 +from vllm.inputs import (PromptInputs, TextPrompt, TokensPrompt,
10 parse_and_batch_prompt)
11 from vllm.logger import init_logger
12 from vllm.lora.request import LoRARequest
@@ -238,7 +237,7 @@ class LLM:
237 @overload
238 def generate(
239 self,
241 - inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
240 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
241 /, # We may enable `inputs` keyword after removing the old API
242 *,
243 sampling_params: Optional[Union[SamplingParams,
@@ -255,7 +254,7 @@ class LLM:
254 "instead.")
255 def generate(
256 self,
258 - prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
257 + prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
258 Optional[Union[str, List[str]]]] = None,
259 sampling_params: Optional[Union[SamplingParams,
260 Sequence[SamplingParams]]] = None,
@@ -302,9 +301,7 @@ class LLM:
301 prompt_token_ids=prompt_token_ids,
302 )
303 else:
305 - inputs = cast(
306 - Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
307 - prompts)
304 + inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
305
306 if sampling_params is None:
307 # Use default sampling params.
@@ -383,7 +380,7 @@ class LLM:
380 @overload
381 def encode(
382 self,
386 - inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
383 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
384 /, # We may enable `inputs` keyword after removing the old API
385 *,
386 pooling_params: Optional[Union[PoolingParams,
@@ -400,7 +397,7 @@ class LLM:
397 "instead.")
398 def encode(
399 self,
403 - prompts: Union[Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
400 + prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
401 Optional[Union[str, List[str]]]] = None,
402 pooling_params: Optional[Union[PoolingParams,
403 Sequence[PoolingParams]]] = None,
@@ -417,7 +414,7 @@ class LLM:
414
415 Args:
416 inputs: The inputs to the LLM. You may pass a sequence of inputs for
420 - batch inference. See :class:`~vllm.inputs.PromptStrictInputs`
417 + batch inference. See :class:`~vllm.inputs.PromptInputs`
418 for more details about the format of each input.
419 pooling_params: The pooling parameters for pooling. If None, we
420 use the default pooling parameters.
@@ -446,9 +443,7 @@ class LLM:
443 prompt_token_ids=prompt_token_ids,
444 )
445 else:
449 - inputs = cast(
450 - Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
451 - prompts)
446 + inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
447
448 if pooling_params is None:
449 # Use default pooling params.
@@ -496,17 +491,11 @@ class LLM:
491 inputs: List[PromptInputs] = []
492 for i in range(num_requests):
493 if prompts is not None:
499 - if prompt_token_ids is not None:
500 - item = TextTokensPrompt(
501 - prompt=prompts[i],
502 - prompt_token_ids=prompt_token_ids[i])
503 - else:
504 - item = TextPrompt(prompt=prompts[i])
494 + item = TextPrompt(prompt=prompts[i])
495 + elif prompt_token_ids is not None:
496 + item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
497 else:
506 - if prompt_token_ids is not None:
507 - item = TokensPrompt(prompt_token_ids=prompt_token_ids[i])
508 - else:
509 - raise AssertionError
498 + raise AssertionError
499
500 inputs.append(item)
501
@@ -514,7 +503,7 @@ class LLM:
503
504 def _validate_and_add_requests(
505 self,
517 - inputs: Union[PromptStrictInputs, Sequence[PromptStrictInputs]],
506 + inputs: Union[PromptInputs, Sequence[PromptInputs]],
507 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
508 Sequence[PoolingParams]],
509 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
vllm/entrypoints/logger.py new
+41
@@ -0,0 +1,41 @@
1 +from typing import List, Optional, Union
2 +
3 +from vllm.logger import init_logger
4 +from vllm.lora.request import LoRARequest
5 +from vllm.pooling_params import PoolingParams
6 +from vllm.prompt_adapter.request import PromptAdapterRequest
7 +from vllm.sampling_params import SamplingParams
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +class RequestLogger:
13 +
14 + def __init__(self, *, max_log_len: Optional[int]) -> None:
15 + super().__init__()
16 +
17 + self.max_log_len = max_log_len
18 +
19 + def log_inputs(
20 + self,
21 + request_id: str,
22 + prompt: Optional[str],
23 + prompt_token_ids: Optional[List[int]],
24 + params: Optional[Union[SamplingParams, PoolingParams]],
25 + lora_request: Optional[LoRARequest],
26 + prompt_adapter_request: Optional[PromptAdapterRequest],
27 + ) -> None:
28 + max_log_len = self.max_log_len
29 + if max_log_len is not None:
30 + if prompt is not None:
31 + prompt = prompt[:max_log_len]
32 +
33 + if prompt_token_ids is not None:
34 + prompt_token_ids = prompt_token_ids[:max_log_len]
35 +
36 + logger.info(
37 + "Received request %s: prompt: %r, "
38 + "params: %s, prompt_token_ids: %s, "
39 + "lora_request: %s, prompt_adapter_request: %s.", request_id,
40 + prompt, params, prompt_token_ids, lora_request,
41 + prompt_adapter_request)
vllm/entrypoints/openai/api_server.py
+36 -11
@@ -18,6 +18,7 @@ from starlette.routing import Mount
18 import vllm.envs as envs
19 from vllm.engine.arg_utils import AsyncEngineArgs
20 from vllm.engine.async_llm_engine import AsyncLLMEngine
21 +from vllm.entrypoints.logger import RequestLogger
22 from vllm.entrypoints.openai.cli_args import make_arg_parser
23 # yapf conflicts with isort for this block
24 # yapf: disable
@@ -244,24 +245,48 @@ def run_server(args, llm_engine=None):
245 # When using single vLLM without engine_use_ray
246 model_config = asyncio.run(engine.get_model_config())
247
248 + if args.disable_log_requests:
249 + request_logger = None
250 + else:
251 + request_logger = RequestLogger(max_log_len=args.max_log_len)
252 +
253 global openai_serving_chat
254 global openai_serving_completion
255 global openai_serving_embedding
256 global openai_serving_tokenization
257
252 - openai_serving_chat = OpenAIServingChat(engine, model_config,
253 - served_model_names,
254 - args.response_role,
255 - args.lora_modules,
256 - args.chat_template)
258 + openai_serving_chat = OpenAIServingChat(
259 + engine,
260 + model_config,
261 + served_model_names,
262 + args.response_role,
263 + lora_modules=args.lora_modules,
264 + prompt_adapters=args.prompt_adapters,
265 + request_logger=request_logger,
266 + chat_template=args.chat_template,
267 + )
268 openai_serving_completion = OpenAIServingCompletion(
258 - engine, model_config, served_model_names, args.lora_modules,
259 - args.prompt_adapters)
260 - openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
261 - served_model_names)
269 + engine,
270 + model_config,
271 + served_model_names,
272 + lora_modules=args.lora_modules,
273 + prompt_adapters=args.prompt_adapters,
274 + request_logger=request_logger,
275 + )
276 + openai_serving_embedding = OpenAIServingEmbedding(
277 + engine,
278 + model_config,
279 + served_model_names,
280 + request_logger=request_logger,
281 + )
282 openai_serving_tokenization = OpenAIServingTokenization(
263 - engine, model_config, served_model_names, args.lora_modules,
264 - args.chat_template)
283 + engine,
284 + model_config,
285 + served_model_names,
286 + lora_modules=args.lora_modules,
287 + request_logger=request_logger,
288 + chat_template=args.chat_template,
289 + )
290 app.root_path = args.root_path
291
292 logger.info("Available routes are:")
vllm/entrypoints/openai/cli_args.py
+8
@@ -130,6 +130,14 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
130 "using app.add_middleware(). ")
131
132 parser = AsyncEngineArgs.add_cli_args(parser)
133 +
134 + parser.add_argument('--max-log-len',
135 + type=int,
136 + default=None,
137 + help='Max number of prompt characters or prompt '
138 + 'ID numbers being printed in log.'
139 + '\n\nDefault: Unlimited')
140 +
141 return parser
142
143
vllm/entrypoints/openai/protocol.py
+58 -50
@@ -121,40 +121,42 @@ class ChatCompletionRequest(OpenAIBaseModel):
121
122 # doc: begin-chat-completion-sampling-params
123 best_of: Optional[int] = None
124 - use_beam_search: Optional[bool] = False
125 - top_k: Optional[int] = -1
126 - min_p: Optional[float] = 0.0
127 - repetition_penalty: Optional[float] = 1.0
128 - length_penalty: Optional[float] = 1.0
129 - early_stopping: Optional[bool] = False
130 - ignore_eos: Optional[bool] = False
131 - min_tokens: Optional[int] = 0
124 + use_beam_search: bool = False
125 + top_k: int = -1
126 + min_p: float = 0.0
127 + repetition_penalty: float = 1.0
128 + length_penalty: float = 1.0
129 + early_stopping: bool = False
130 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
133 - skip_special_tokens: Optional[bool] = True
134 - spaces_between_special_tokens: Optional[bool] = True
131 + include_stop_str_in_output: bool = False
132 + ignore_eos: bool = False
133 + min_tokens: int = 0
134 + skip_special_tokens: bool = True
135 + spaces_between_special_tokens: bool = True
136 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
137 # doc: end-chat-completion-sampling-params
138
139 # doc: begin-chat-completion-extra-params
138 - echo: Optional[bool] = Field(
140 + echo: bool = Field(
141 default=False,
142 description=(
143 "If true, the new message will be prepended with the last message "
144 "if they belong to the same role."),
145 )
144 - add_generation_prompt: Optional[bool] = Field(
146 + add_generation_prompt: bool = Field(
147 default=True,
148 description=
149 ("If true, the generation prompt will be added to the chat template. "
150 "This is a parameter used by chat template in tokenizer config of the "
151 "model."),
152 )
151 - add_special_tokens: Optional[bool] = Field(
153 + add_special_tokens: bool = Field(
154 default=False,
155 description=(
156 "If true, special tokens (e.g. BOS) will be added to the prompt "
157 "on top of what is added by the chat template. "
158 "For most models, the chat template takes care of adding the "
157 - "special tokens so this should be set to False (as is the "
159 + "special tokens so this should be set to false (as is the "
160 "default)."),
161 )
162 documents: Optional[List[Dict[str, str]]] = Field(
@@ -178,12 +180,6 @@ class ChatCompletionRequest(OpenAIBaseModel):
180 description=("Additional kwargs to pass to the template renderer. "
181 "Will be accessible by the chat template."),
182 )
181 - include_stop_str_in_output: Optional[bool] = Field(
182 - default=False,
183 - description=(
184 - "Whether to include the stop string in the output. "
185 - "This is only applied when the stop or stop_token_ids is set."),
186 - )
183 guided_json: Optional[Union[str, dict, BaseModel]] = Field(
184 default=None,
185 description=("If specified, the output will follow the JSON schema."),
@@ -244,22 +240,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
240
241 return SamplingParams(
242 n=self.n,
243 + best_of=self.best_of,
244 presence_penalty=self.presence_penalty,
245 frequency_penalty=self.frequency_penalty,
246 repetition_penalty=self.repetition_penalty,
247 temperature=self.temperature,
248 top_p=self.top_p,
249 + top_k=self.top_k,
250 min_p=self.min_p,
251 seed=self.seed,
252 stop=self.stop,
253 stop_token_ids=self.stop_token_ids,
256 - max_tokens=self.max_tokens,
257 - min_tokens=self.min_tokens,
254 logprobs=self.top_logprobs if self.logprobs else None,
255 prompt_logprobs=self.top_logprobs if self.echo else None,
260 - best_of=self.best_of,
261 - top_k=self.top_k,
256 ignore_eos=self.ignore_eos,
257 + max_tokens=self.max_tokens,
258 + min_tokens=self.min_tokens,
259 use_beam_search=self.use_beam_search,
260 early_stopping=self.early_stopping,
261 skip_special_tokens=self.skip_special_tokens,
@@ -267,6 +263,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
263 include_stop_str_in_output=self.include_stop_str_in_output,
264 length_penalty=self.length_penalty,
265 logits_processors=logits_processors,
266 + truncate_prompt_tokens=self.truncate_prompt_tokens,
267 )
268
269 @model_validator(mode='before')
@@ -348,26 +345,27 @@ class CompletionRequest(OpenAIBaseModel):
345 user: Optional[str] = None
346
347 # doc: begin-completion-sampling-params
351 - use_beam_search: Optional[bool] = False
352 - top_k: Optional[int] = -1
353 - min_p: Optional[float] = 0.0
354 - repetition_penalty: Optional[float] = 1.0
355 - length_penalty: Optional[float] = 1.0
356 - early_stopping: Optional[bool] = False
348 + use_beam_search: bool = False
349 + top_k: int = -1
350 + min_p: float = 0.0
351 + repetition_penalty: float = 1.0
352 + length_penalty: float = 1.0
353 + early_stopping: bool = False
354 stop_token_ids: Optional[List[int]] = Field(default_factory=list)
358 - ignore_eos: Optional[bool] = False
359 - min_tokens: Optional[int] = 0
360 - skip_special_tokens: Optional[bool] = True
361 - spaces_between_special_tokens: Optional[bool] = True
355 + include_stop_str_in_output: bool = False
356 + ignore_eos: bool = False
357 + min_tokens: int = 0
358 + skip_special_tokens: bool = True
359 + spaces_between_special_tokens: bool = True
360 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
361 # doc: end-completion-sampling-params
362
363 # doc: begin-completion-extra-params
366 - include_stop_str_in_output: Optional[bool] = Field(
367 - default=False,
364 + add_special_tokens: bool = Field(
365 + default=True,
366 description=(
369 - "Whether to include the stop string in the output. "
370 - "This is only applied when the stop or stop_token_ids is set."),
367 + "If true (the default), special tokens (e.g. BOS) will be added to "
368 + "the prompt."),
369 )
370 response_format: Optional[ResponseFormat] = Field(
371 default=None,
@@ -447,15 +445,15 @@ class CompletionRequest(OpenAIBaseModel):
445 seed=self.seed,
446 stop=self.stop,
447 stop_token_ids=self.stop_token_ids,
448 + logprobs=self.logprobs,
449 ignore_eos=self.ignore_eos,
450 max_tokens=self.max_tokens if not echo_without_generation else 1,
451 min_tokens=self.min_tokens,
453 - logprobs=self.logprobs,
452 use_beam_search=self.use_beam_search,
453 early_stopping=self.early_stopping,
454 prompt_logprobs=self.logprobs if self.echo else None,
455 skip_special_tokens=self.skip_special_tokens,
458 - spaces_between_special_tokens=(self.spaces_between_special_tokens),
456 + spaces_between_special_tokens=self.spaces_between_special_tokens,
457 include_stop_str_in_output=self.include_stop_str_in_output,
458 length_penalty=self.length_penalty,
459 logits_processors=logits_processors,
@@ -489,11 +487,11 @@ class CompletionRequest(OpenAIBaseModel):
487 def validate_stream_options(cls, data):
488 if data.get("stream_options") and not data.get("stream"):
489 raise ValueError(
492 - "Stream options can only be defined when stream is True.")
490 + "Stream options can only be defined when stream is true.")
491 return data
492
493
496 -class EmbeddingRequest(BaseModel):
494 +class EmbeddingRequest(OpenAIBaseModel):
495 # Ordered by official OpenAI API documentation
496 # https://platform.openai.com/docs/api-reference/embeddings
497 model: str
@@ -565,13 +563,13 @@ class CompletionStreamResponse(OpenAIBaseModel):
563 usage: Optional[UsageInfo] = Field(default=None)
564
565
568 -class EmbeddingResponseData(BaseModel):
566 +class EmbeddingResponseData(OpenAIBaseModel):
567 index: int
568 object: str = "embedding"
569 embedding: Union[List[float], str]
570
571
574 -class EmbeddingResponse(BaseModel):
572 +class EmbeddingResponse(OpenAIBaseModel):
573 id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
574 object: str = "list"
575 created: int = Field(default_factory=lambda: int(time.time()))
@@ -670,8 +668,8 @@ class BatchRequestInput(OpenAIBaseModel):
668 # /v1/chat/completions is supported.
669 url: str
670
673 - # The parameteters of the request.
674 - body: Union[ChatCompletionRequest, ]
671 + # The parameters of the request.
672 + body: ChatCompletionRequest
673
674
675 class BatchResponseData(OpenAIBaseModel):
@@ -703,12 +701,22 @@ class BatchRequestOutput(OpenAIBaseModel):
701 error: Optional[Any]
702
703
706 -class TokenizeRequest(OpenAIBaseModel):
704 +class TokenizeCompletionRequest(OpenAIBaseModel):
705 + model: str
706 + prompt: str
707 +
708 + add_special_tokens: bool = Field(default=True)
709 +
710 +
711 +class TokenizeChatRequest(OpenAIBaseModel):
712 + model: str
713 + messages: List[ChatCompletionMessageParam]
714 +
715 add_generation_prompt: bool = Field(default=True)
716 add_special_tokens: bool = Field(default=False)
709 - prompt: Optional[str] = Field(default=None)
710 - messages: Optional[List[ChatCompletionMessageParam]] = Field(default=None)
711 - model: str
717 +
718 +
719 +TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
720
721
722 class TokenizeResponse(OpenAIBaseModel):
vllm/entrypoints/openai/run_batch.py
+19 -1
@@ -6,6 +6,7 @@ import aiohttp
6
7 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
8 from vllm.engine.async_llm_engine import AsyncLLMEngine
9 +from vllm.entrypoints.logger import RequestLogger
10 from vllm.entrypoints.openai.protocol import (BatchRequestInput,
11 BatchRequestOutput,
12 BatchResponseData,
@@ -44,9 +45,17 @@ def parse_args():
45 type=nullable_str,
46 default="assistant",
47 help="The role name to return if "
47 - "`request.add_generation_prompt=true`.")
48 + "`request.add_generation_prompt=True`.")
49
50 parser = AsyncEngineArgs.add_cli_args(parser)
51 +
52 + parser.add_argument('--max-log-len',
53 + type=int,
54 + default=None,
55 + help='Max number of prompt characters or prompt '
56 + 'ID numbers being printed in log.'
57 + '\n\nDefault: Unlimited')
58 +
59 return parser.parse_args()
60
61
@@ -114,11 +123,20 @@ async def main(args):
123 # When using single vLLM without engine_use_ray
124 model_config = await engine.get_model_config()
125
126 + if args.disable_log_requests:
127 + request_logger = None
128 + else:
129 + request_logger = RequestLogger(max_log_len=args.max_log_len)
130 +
131 openai_serving_chat = OpenAIServingChat(
132 engine,
133 model_config,
134 served_model_names,
135 args.response_role,
136 + lora_modules=None,
137 + prompt_adapters=None,
138 + request_logger=request_logger,
139 + chat_template=None,
140 )
141
142 # Submit all requests in the file to the engine "concurrently".
vllm/entrypoints/openai/serving_chat.py
+72 -47
@@ -12,6 +12,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.entrypoints.chat_utils import (ConversationMessage,
13 load_chat_template,
14 parse_chat_message_content)
15 +from vllm.entrypoints.logger import RequestLogger
16 from vllm.entrypoints.openai.protocol import (
17 ChatCompletionLogProb, ChatCompletionLogProbs,
18 ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
@@ -20,7 +21,8 @@ from vllm.entrypoints.openai.protocol import (
21 ChatCompletionStreamResponse, ChatMessage, DeltaMessage, ErrorResponse,
22 FunctionCall, ToolCall, UsageInfo)
23 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
23 - OpenAIServing)
24 + OpenAIServing,
25 + PromptAdapterPath)
26 from vllm.inputs import PromptInputs
27 from vllm.logger import init_logger
28 from vllm.model_executor.guided_decoding import (
@@ -37,17 +39,24 @@ logger = init_logger(__name__)
39
40 class OpenAIServingChat(OpenAIServing):
41
40 - def __init__(self,
41 - engine: AsyncLLMEngine,
42 - model_config: ModelConfig,
43 - served_model_names: List[str],
44 - response_role: str,
45 - lora_modules: Optional[List[LoRAModulePath]] = None,
46 - chat_template: Optional[str] = None):
42 + def __init__(
43 + self,
44 + engine: AsyncLLMEngine,
45 + model_config: ModelConfig,
46 + served_model_names: List[str],
47 + response_role: str,
48 + *,
49 + lora_modules: Optional[List[LoRAModulePath]],
50 + prompt_adapters: Optional[List[PromptAdapterPath]],
51 + request_logger: Optional[RequestLogger],
52 + chat_template: Optional[str],
53 + ):
54 super().__init__(engine=engine,
55 model_config=model_config,
56 served_model_names=served_model_names,
50 - lora_modules=lora_modules)
57 + lora_modules=lora_modules,
58 + prompt_adapters=prompt_adapters,
59 + request_logger=request_logger)
60
61 self.response_role = response_role
62
@@ -74,7 +83,12 @@ class OpenAIServingChat(OpenAIServing):
83 return error_check_ret
84
85 try:
77 - _, lora_request = self._maybe_get_adapter(request)
86 + (
87 + lora_request,
88 + prompt_adapter_request,
89 + ) = self._maybe_get_adapters(request)
90 +
91 + model_config = self.model_config
92 tokenizer = await self.engine.get_tokenizer(lora_request)
93
94 conversation: List[ConversationMessage] = []
@@ -82,7 +96,7 @@ class OpenAIServingChat(OpenAIServing):
96
97 for msg in request.messages:
98 chat_parsed_result = parse_chat_message_content(
85 - msg, self.model_config, tokenizer)
99 + msg, model_config, tokenizer)
100
101 conversation.extend(chat_parsed_result.messages)
102 mm_futures.extend(chat_parsed_result.mm_futures)
@@ -116,14 +130,8 @@ class OpenAIServingChat(OpenAIServing):
130 logger.error("Error in loading multi-modal data: %s", e)
131 return self.create_error_response(str(e))
132
119 - request_id = f"cmpl-{random_uuid()}"
133 + request_id = f"chat-{random_uuid()}"
134 try:
121 - # Tokenize/detokenize depending on prompt format (string/token list)
122 - prompt_ids, prompt_text = await self._validate_prompt_and_tokenize(
123 - request,
124 - tokenizer,
125 - prompt=prompt,
126 - add_special_tokens=request.add_special_tokens)
135 sampling_params = request.to_sampling_params()
136 decoding_config = await self.engine.get_decoding_config()
137 guided_decoding_backend = request.guided_decoding_backend \
@@ -137,31 +145,47 @@ class OpenAIServingChat(OpenAIServing):
145 sampling_params.logits_processors = []
146 sampling_params.logits_processors.append(
147 guided_decode_logits_processor)
148 +
149 + prompt_inputs = self._tokenize_prompt_input(
150 + request,
151 + tokenizer,
152 + prompt,
153 + truncate_prompt_tokens=sampling_params.truncate_prompt_tokens,
154 + add_special_tokens=request.add_special_tokens,
155 + )
156 +
157 + self._log_inputs(request_id,
158 + prompt_inputs,
159 + params=sampling_params,
160 + lora_request=lora_request,
161 + prompt_adapter_request=prompt_adapter_request)
162 +
163 + engine_inputs: PromptInputs = {
164 + "prompt_token_ids": prompt_inputs["prompt_token_ids"],
165 + }
166 + if mm_data is not None:
167 + engine_inputs["multi_modal_data"] = mm_data
168 +
169 + is_tracing_enabled = await self.engine.is_tracing_enabled()
170 + trace_headers = None
171 + if is_tracing_enabled and raw_request:
172 + trace_headers = extract_trace_headers(raw_request.headers)
173 + if (not is_tracing_enabled and raw_request
174 + and contains_trace_headers(raw_request.headers)):
175 + log_tracing_disabled_warning()
176 +
177 + result_generator = self.engine.generate(
178 + engine_inputs,
179 + sampling_params,
180 + request_id,
181 + lora_request=lora_request,
182 + trace_headers=trace_headers,
183 + prompt_adapter_request=prompt_adapter_request,
184 + )
185 except ValueError as e:
186 + # TODO: Use a vllm-specific Validation Error
187 return self.create_error_response(str(e))
188
143 - inputs: PromptInputs = {
144 - "prompt": prompt_text,
145 - "prompt_token_ids": prompt_ids,
146 - }
147 - if mm_data:
148 - inputs["multi_modal_data"] = mm_data
149 -
150 - is_tracing_enabled = await self.engine.is_tracing_enabled()
151 - trace_headers = None
152 - if is_tracing_enabled and raw_request:
153 - trace_headers = extract_trace_headers(raw_request.headers)
154 - if not is_tracing_enabled and raw_request and contains_trace_headers(
155 - raw_request.headers):
156 - log_tracing_disabled_warning()
157 -
158 - result_generator = self.engine.generate(
159 - inputs,
160 - sampling_params,
161 - request_id,
162 - lora_request,
163 - trace_headers=trace_headers,
164 - )
189 # Streaming response
190 if request.stream:
191 return self.chat_completion_stream_generator(
@@ -195,10 +219,11 @@ class OpenAIServingChat(OpenAIServing):
219 first_iteration = True
220
221 # Send response for each token for each request.n (index)
198 - assert request.n is not None
199 - previous_texts = [""] * request.n
200 - previous_num_tokens = [0] * request.n
201 - finish_reason_sent = [False] * request.n
222 + num_choices = 1 if request.n is None else request.n
223 + previous_texts = [""] * num_choices
224 + previous_num_tokens = [0] * num_choices
225 + finish_reason_sent = [False] * num_choices
226 +
227 try:
228 async for res in result_generator:
229 # We need to do it here, because if there are exceptions in
@@ -208,7 +233,7 @@ class OpenAIServingChat(OpenAIServing):
233 # Send first response for each request.n (index) with
234 # the role
235 role = self.get_chat_request_role(request)
211 - for i in range(request.n):
236 + for i in range(num_choices):
237 choice_data = ChatCompletionResponseStreamChoice(
238 index=i,
239 delta=DeltaMessage(role=role),
@@ -236,19 +261,19 @@ class OpenAIServingChat(OpenAIServing):
261 last_msg_content = conversation[-1]["content"]
262
263 if last_msg_content:
239 - for i in range(request.n):
264 + for i in range(num_choices):
265 choice_data = (
266 ChatCompletionResponseStreamChoice(
267 index=i,
268 delta=DeltaMessage(
269 content=last_msg_content),
270 + logprobs=None,
271 finish_reason=None))
272 chunk = ChatCompletionStreamResponse(
273 id=request_id,
274 object=chunk_object_type,
275 created=created_time,
276 choices=[choice_data],
251 - logprobs=None,
277 model=model_name)
278 if (request.stream_options and
279 request.stream_options.include_usage):
vllm/entrypoints/openai/serving_completion.py
+61 -54
@@ -2,13 +2,14 @@ import time
2 from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
3 Optional)
4 from typing import Sequence as GenericSequence
5 -from typing import Tuple
5 +from typing import Tuple, cast
6
7 from fastapi import Request
8 from transformers import PreTrainedTokenizer
9
10 from vllm.config import ModelConfig
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 +from vllm.entrypoints.logger import RequestLogger
13 # yapf conflicts with isort for this block
14 # yapf: disable
15 from vllm.entrypoints.openai.protocol import (CompletionLogProbs,
@@ -39,40 +40,24 @@ TypeCreateLogProbsFn = Callable[
40 [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs]
41
42
42 -def parse_prompt_format(prompt) -> Tuple[bool, list]:
43 - # get the prompt, openai supports the following
44 - # "a string, array of strings, array of tokens, or array of token arrays."
45 - prompt_is_tokens = False
46 - prompts = [prompt] # case 1: a string
47 - if isinstance(prompt, list):
48 - if len(prompt) == 0:
49 - raise ValueError("please provide at least one prompt")
50 - elif isinstance(prompt[0], str):
51 - prompt_is_tokens = False
52 - prompts = prompt # case 2: array of strings
53 - elif isinstance(prompt[0], int):
54 - prompt_is_tokens = True
55 - prompts = [prompt] # case 3: array of tokens
56 - elif isinstance(prompt[0], list) and isinstance(prompt[0][0], int):
57 - prompt_is_tokens = True
58 - prompts = prompt # case 4: array of token arrays
59 - else:
60 - raise ValueError("prompt must be a string, array of strings, "
61 - "array of tokens, or array of token arrays")
62 - return prompt_is_tokens, prompts
63 -
64 -
43 class OpenAIServingCompletion(OpenAIServing):
44
67 - def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
68 - served_model_names: List[str],
69 - lora_modules: Optional[List[LoRAModulePath]],
70 - prompt_adapters: Optional[List[PromptAdapterPath]]):
45 + def __init__(
46 + self,
47 + engine: AsyncLLMEngine,
48 + model_config: ModelConfig,
49 + served_model_names: List[str],
50 + *,
51 + lora_modules: Optional[List[LoRAModulePath]],
52 + prompt_adapters: Optional[List[PromptAdapterPath]],
53 + request_logger: Optional[RequestLogger],
54 + ):
55 super().__init__(engine=engine,
56 model_config=model_config,
57 served_model_names=served_model_names,
58 lora_modules=lora_modules,
75 - prompt_adapters=prompt_adapters)
59 + prompt_adapters=prompt_adapters,
60 + request_logger=request_logger)
61
62 async def create_completion(self, request: CompletionRequest,
63 raw_request: Request):
@@ -101,12 +86,11 @@ class OpenAIServingCompletion(OpenAIServing):
86 # Schedule the request and get the result generator.
87 generators: List[AsyncIterator[RequestOutput]] = []
88 try:
104 - adapter_type, adapter_request = self._maybe_get_adapter(request)
105 - lora_request, prompt_adapter_request = None, None
106 - if adapter_type == 'LoRA':
107 - lora_request, prompt_adapter_request = adapter_request, None
108 - elif adapter_type == 'PromptAdapter':
109 - lora_request, prompt_adapter_request = None, adapter_request
89 + (
90 + lora_request,
91 + prompt_adapter_request,
92 + ) = self._maybe_get_adapters(request)
93 +
94 tokenizer = await self.engine.get_tokenizer(lora_request)
95
96 sampling_params = request.to_sampling_params()
@@ -122,17 +106,25 @@ class OpenAIServingCompletion(OpenAIServing):
106 sampling_params.logits_processors = []
107 sampling_params.logits_processors.append(
108 guided_decode_logit_processor)
125 - prompt_is_tokens, prompts = parse_prompt_format(request.prompt)
109
127 - for i, prompt in enumerate(prompts):
128 - prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
129 - prompt_formats = await self._validate_prompt_and_tokenize(
110 + prompts = list(
111 + self._tokenize_prompt_input_or_inputs(
112 request,
113 tokenizer,
114 + request.prompt,
115 truncate_prompt_tokens=sampling_params.
116 truncate_prompt_tokens,
134 - **{prompt_arg: prompt})
135 - prompt_ids, prompt_text = prompt_formats
117 + add_special_tokens=request.add_special_tokens,
118 + ))
119 +
120 + for i, prompt_inputs in enumerate(prompts):
121 + request_id_item = f"{request_id}-{i}"
122 +
123 + self._log_inputs(request_id_item,
124 + prompt_inputs,
125 + params=sampling_params,
126 + lora_request=lora_request,
127 + prompt_adapter_request=prompt_adapter_request)
128
129 is_tracing_enabled = await self.engine.is_tracing_enabled()
130 trace_headers = None
@@ -143,12 +135,9 @@ class OpenAIServingCompletion(OpenAIServing):
135 log_tracing_disabled_warning()
136
137 generator = self.engine.generate(
146 - {
147 - "prompt": prompt_text,
148 - "prompt_token_ids": prompt_ids
149 - },
138 + {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
139 sampling_params,
151 - f"{request_id}-{i}",
140 + request_id_item,
141 lora_request=lora_request,
142 prompt_adapter_request=prompt_adapter_request,
143 trace_headers=trace_headers,
@@ -189,9 +178,27 @@ class OpenAIServingCompletion(OpenAIServing):
178 await self.engine.abort(f"{request_id}-{i}")
179 return self.create_error_response("Client disconnected")
180 final_res_batch[i] = res
181 +
182 + for i, final_res in enumerate(final_res_batch):
183 + assert final_res is not None
184 +
185 + # The output should contain the input text
186 + # We did not pass it into vLLM engine to avoid being redundant
187 + # with the inputs token IDs
188 + if final_res.prompt is None:
189 + final_res.prompt = prompts[i]["prompt"]
190 +
191 + final_res_batch_checked = cast(List[RequestOutput],
192 + final_res_batch)
193 +
194 response = self.request_output_to_completion_response(
193 - final_res_batch, request, request_id, created_time, model_name,
194 - tokenizer)
195 + final_res_batch_checked,
196 + request,
197 + request_id,
198 + created_time,
199 + model_name,
200 + tokenizer,
201 + )
202 except ValueError as e:
203 # TODO: Use a vllm-specific Validation Error
204 return self.create_error_response(str(e))
@@ -220,10 +227,10 @@ class OpenAIServingCompletion(OpenAIServing):
227 num_prompts: int,
228 tokenizer: PreTrainedTokenizer,
229 ) -> AsyncGenerator[str, None]:
223 - assert request.n is not None
224 - previous_texts = [""] * request.n * num_prompts
225 - previous_num_tokens = [0] * request.n * num_prompts
226 - has_echoed = [False] * request.n * num_prompts
230 + num_choices = 1 if request.n is None else request.n
231 + previous_texts = [""] * num_choices * num_prompts
232 + previous_num_tokens = [0] * num_choices * num_prompts
233 + has_echoed = [False] * num_choices * num_prompts
234
235 try:
236 async for prompt_idx, res in result_generator:
@@ -234,7 +241,7 @@ class OpenAIServingCompletion(OpenAIServing):
241 raise StopAsyncIteration()
242
243 for output in res.outputs:
237 - i = output.index + prompt_idx * request.n
244 + i = output.index + prompt_idx * num_choices
245 # TODO(simon): optimize the performance by avoiding full
246 # text O(n^2) sending.
247
@@ -343,8 +350,8 @@ class OpenAIServingCompletion(OpenAIServing):
350 choices: List[CompletionResponseChoice] = []
351 num_prompt_tokens = 0
352 num_generated_tokens = 0
353 +
354 for final_res in final_res_batch:
347 - assert final_res is not None
355 prompt_token_ids = final_res.prompt_token_ids
356 prompt_logprobs = final_res.prompt_logprobs
357 prompt_text = final_res.prompt
vllm/entrypoints/openai/serving_embedding.py
+55 -23
@@ -1,16 +1,16 @@
1 import base64
2 import time
3 -from typing import AsyncIterator, List, Optional, Tuple
3 +from typing import AsyncIterator, List, Optional, Tuple, cast
4
5 import numpy as np
6 from fastapi import Request
7
8 from vllm.config import ModelConfig
9 from vllm.engine.async_llm_engine import AsyncLLMEngine
10 +from vllm.entrypoints.logger import RequestLogger
11 from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
12 EmbeddingResponse,
13 EmbeddingResponseData, UsageInfo)
13 -from vllm.entrypoints.openai.serving_completion import parse_prompt_format
14 from vllm.entrypoints.openai.serving_engine import OpenAIServing
15 from vllm.logger import init_logger
16 from vllm.outputs import EmbeddingRequestOutput
@@ -28,11 +28,11 @@ def request_output_to_embedding_response(
28 data: List[EmbeddingResponseData] = []
29 num_prompt_tokens = 0
30 for idx, final_res in enumerate(final_res_batch):
31 - assert final_res is not None
31 prompt_token_ids = final_res.prompt_token_ids
32 embedding = final_res.outputs.embedding
33 if encoding_format == "base64":
35 - embedding = base64.b64encode(np.array(embedding))
34 + embedding_bytes = np.array(embedding).tobytes()
35 + embedding = base64.b64encode(embedding_bytes).decode("utf-8")
36 embedding_data = EmbeddingResponseData(index=idx, embedding=embedding)
37 data.append(embedding_data)
38
@@ -54,12 +54,20 @@ def request_output_to_embedding_response(
54
55 class OpenAIServingEmbedding(OpenAIServing):
56
57 - def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
58 - served_model_names: List[str]):
57 + def __init__(
58 + self,
59 + engine: AsyncLLMEngine,
60 + model_config: ModelConfig,
61 + served_model_names: List[str],
62 + *,
63 + request_logger: Optional[RequestLogger],
64 + ):
65 super().__init__(engine=engine,
66 model_config=model_config,
67 served_model_names=served_model_names,
62 - lora_modules=None)
68 + lora_modules=None,
69 + prompt_adapters=None,
70 + request_logger=request_logger)
71 self._check_embedding_mode(model_config.embedding_mode)
72
73 async def create_embedding(self, request: EmbeddingRequest,
@@ -80,29 +88,47 @@ class OpenAIServingEmbedding(OpenAIServing):
88 "dimensions is currently not supported")
89
90 model_name = request.model
83 - request_id = f"cmpl-{random_uuid()}"
91 + request_id = f"embd-{random_uuid()}"
92 created_time = int(time.monotonic())
93
94 # Schedule the request and get the result generator.
87 - generators = []
95 + generators: List[AsyncIterator[EmbeddingRequestOutput]] = []
96 try:
89 - prompt_is_tokens, prompts = parse_prompt_format(request.input)
97 + (
98 + lora_request,
99 + prompt_adapter_request,
100 + ) = self._maybe_get_adapters(request)
101 +
102 + tokenizer = await self.engine.get_tokenizer(lora_request)
103 +
104 pooling_params = request.to_pooling_params()
105
92 - tokenizer = await self.engine.get_tokenizer()
93 - for i, prompt in enumerate(prompts):
94 - prompt_arg = "prompt_ids" if prompt_is_tokens else "prompt"
95 - prompt_formats = await self._validate_prompt_and_tokenize(
96 - request, tokenizer, **{prompt_arg: prompt})
97 - prompt_ids, prompt_text = prompt_formats
106 + prompts = list(
107 + self._tokenize_prompt_input_or_inputs(
108 + request,
109 + tokenizer,
110 + request.input,
111 + ))
112 +
113 + for i, prompt_inputs in enumerate(prompts):
114 + request_id_item = f"{request_id}-{i}"
115 +
116 + self._log_inputs(request_id_item,
117 + prompt_inputs,
118 + params=pooling_params,
119 + lora_request=lora_request,
120 + prompt_adapter_request=prompt_adapter_request)
121 +
122 + if prompt_adapter_request is not None:
123 + raise NotImplementedError(
124 + "Prompt adapter is not supported "
125 + "for embedding models")
126
127 generator = self.engine.encode(
100 - {
101 - "prompt": prompt_text,
102 - "prompt_token_ids": prompt_ids
103 - },
128 + {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
129 pooling_params,
105 - f"{request_id}-{i}",
130 + request_id_item,
131 + lora_request=lora_request,
132 )
133
134 generators.append(generator)
@@ -121,11 +147,17 @@ class OpenAIServingEmbedding(OpenAIServing):
147 if await raw_request.is_disconnected():
148 # Abort the request if the client disconnects.
149 await self.engine.abort(f"{request_id}-{i}")
124 - # TODO: Use a vllm-specific Validation Error
150 return self.create_error_response("Client disconnected")
151 final_res_batch[i] = res
152 +
153 + for final_res in final_res_batch:
154 + assert final_res is not None
155 +
156 + final_res_batch_checked = cast(List[EmbeddingRequestOutput],
157 + final_res_batch)
158 +
159 response = request_output_to_embedding_response(
128 - final_res_batch, request_id, created_time, model_name,
160 + final_res_batch_checked, request_id, created_time, model_name,
161 encoding_format)
162 except ValueError as e:
163 # TODO: Use a vllm-specific Validation Error
vllm/entrypoints/openai/serving_engine.py
+209 -62
@@ -2,23 +2,33 @@ import json
2 import pathlib
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 -from typing import Any, Dict, List, Optional, Tuple, Union
5 +from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union
6
7 from pydantic import Field
8 -from transformers import PreTrainedTokenizer
8 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
9 from typing_extensions import Annotated
10
11 from vllm.config import ModelConfig
12 from vllm.engine.async_llm_engine import AsyncLLMEngine
13 +from vllm.entrypoints.logger import RequestLogger
14 +# yapf conflicts with isort for this block
15 +# yapf: disable
16 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
17 CompletionRequest,
18 DetokenizeRequest,
19 EmbeddingRequest, ErrorResponse,
20 ModelCard, ModelList,
18 - ModelPermission, TokenizeRequest)
21 + ModelPermission,
22 + TokenizeChatRequest,
23 + TokenizeCompletionRequest,
24 + TokenizeRequest)
25 +# yapf: enable
26 +from vllm.inputs import parse_and_batch_prompt
27 from vllm.logger import init_logger
28 from vllm.lora.request import LoRARequest
29 +from vllm.pooling_params import PoolingParams
30 from vllm.prompt_adapter.request import PromptAdapterRequest
31 +from vllm.sampling_params import SamplingParams
32 from vllm.sequence import Logprob
33
34 logger = init_logger(__name__)
@@ -36,6 +46,17 @@ class LoRAModulePath:
46 local_path: str
47
48
49 +AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
50 + EmbeddingRequest, TokenizeRequest]
51 +
52 +AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
53 +
54 +
55 +class TextTokensPrompt(TypedDict):
56 + prompt: str
57 + prompt_token_ids: List[int]
58 +
59 +
60 class OpenAIServing:
61
62 def __init__(
@@ -43,8 +64,10 @@ class OpenAIServing:
64 engine: AsyncLLMEngine,
65 model_config: ModelConfig,
66 served_model_names: List[str],
67 + *,
68 lora_modules: Optional[List[LoRAModulePath]],
47 - prompt_adapters: Optional[List[PromptAdapterPath]] = None,
69 + prompt_adapters: Optional[List[PromptAdapterPath]],
70 + request_logger: Optional[RequestLogger],
71 ):
72 super().__init__()
73
@@ -78,6 +101,8 @@ class OpenAIServing:
101 prompt_adapter_local_path=prompt_adapter.local_path,
102 prompt_adapter_num_virtual_tokens=num_virtual_tokens))
103
104 + self.request_logger = request_logger
105 +
106 async def show_available_models(self) -> ModelList:
107 """Show available models. Right now we only have one model."""
108 model_cards = [
@@ -126,9 +151,8 @@ class OpenAIServing:
151 return json_str
152
153 async def _check_model(
129 - self, request: Union[ChatCompletionRequest, CompletionRequest,
130 - DetokenizeRequest, EmbeddingRequest,
131 - TokenizeRequest]
154 + self,
155 + request: AnyRequest,
156 ) -> Optional[ErrorResponse]:
157 if request.model in self.served_model_names:
158 return None
@@ -144,64 +168,65 @@ class OpenAIServing:
168 err_type="NotFoundError",
169 status_code=HTTPStatus.NOT_FOUND)
170
147 - def _maybe_get_adapter(
148 - self, request: Union[CompletionRequest, ChatCompletionRequest,
149 - EmbeddingRequest, TokenizeRequest,
150 - DetokenizeRequest]
151 - ) -> Tuple[Optional[str], Optional[Union[LoRARequest,
152 - PromptAdapterRequest]]]:
171 + def _maybe_get_adapters(
172 + self, request: AnyRequest
173 + ) -> Union[Tuple[None, None], Tuple[LoRARequest, None], Tuple[
174 + None, PromptAdapterRequest]]:
175 if request.model in self.served_model_names:
176 return None, None
177 for lora in self.lora_requests:
178 if request.model == lora.lora_name:
157 - return 'LoRA', lora
179 + return lora, None
180 for prompt_adapter in self.prompt_adapter_requests:
181 if request.model == prompt_adapter.prompt_adapter_name:
160 - return 'PromptAdapter', prompt_adapter
182 + return None, prompt_adapter
183 # if _check_model has been called earlier, this will be unreachable
184 raise ValueError(f"The model `{request.model}` does not exist.")
185
164 - async def _validate_prompt_and_tokenize(
165 - self,
166 - request: Union[ChatCompletionRequest, CompletionRequest,
167 - DetokenizeRequest, EmbeddingRequest,
168 - TokenizeRequest],
169 - tokenizer: "PreTrainedTokenizer",
170 - prompt: Optional[str] = None,
171 - prompt_ids: Optional[List[int]] = None,
172 - truncate_prompt_tokens: Optional[Annotated[int,
173 - Field(ge=1)]] = None,
174 - add_special_tokens: Optional[bool] = True
175 - ) -> Tuple[List[int], str]:
176 - if not (prompt or prompt_ids):
177 - raise ValueError("Either prompt or prompt_ids should be provided.")
178 - if prompt and prompt_ids:
179 - raise ValueError(
180 - "Only one of prompt or prompt_ids should be provided.")
181 -
182 - if prompt_ids is None:
183 - # When using OpenAIServingChat for chat completions, for
184 - # most models the special tokens (e.g., BOS) have already
185 - # been added by the chat template. Therefore, we do not
186 - # need to add them again.
187 - # Set add_special_tokens to False (by default) to avoid
188 - # adding the BOS tokens again.
189 - tokenizer_kwargs: Dict[str, Any] = {
190 - "add_special_tokens": add_special_tokens
191 - }
192 - if truncate_prompt_tokens is not None:
193 - tokenizer_kwargs.update({
194 - "truncation": True,
195 - "max_length": truncate_prompt_tokens,
196 - })
197 - input_ids = tokenizer(prompt, **tokenizer_kwargs).input_ids
198 - elif truncate_prompt_tokens is not None:
199 - input_ids = prompt_ids[-truncate_prompt_tokens:]
186 + def _normalize_prompt_text_to_input(
187 + self,
188 + request: AnyRequest,
189 + tokenizer: AnyTokenizer,
190 + prompt: str,
191 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
192 + add_special_tokens: bool,
193 + ) -> TextTokensPrompt:
194 + if truncate_prompt_tokens is None:
195 + encoded = tokenizer(prompt, add_special_tokens=add_special_tokens)
196 else:
197 + encoded = tokenizer(prompt,
198 + add_special_tokens=add_special_tokens,
199 + truncation=True,
200 + max_length=truncate_prompt_tokens)
201 +
202 + input_ids = encoded.input_ids
203 +
204 + input_text = prompt
205 +
206 + return self._validate_input(request, input_ids, input_text)
207 +
208 + def _normalize_prompt_tokens_to_input(
209 + self,
210 + request: AnyRequest,
211 + tokenizer: AnyTokenizer,
212 + prompt_ids: List[int],
213 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]],
214 + ) -> TextTokensPrompt:
215 + if truncate_prompt_tokens is None:
216 input_ids = prompt_ids
217 + else:
218 + input_ids = prompt_ids[-truncate_prompt_tokens:]
219 +
220 + input_text = tokenizer.decode(input_ids)
221
203 - input_text = prompt if prompt is not None else tokenizer.decode(
204 - input_ids)
222 + return self._validate_input(request, input_ids, input_text)
223 +
224 + def _validate_input(
225 + self,
226 + request: AnyRequest,
227 + input_ids: List[int],
228 + input_text: str,
229 + ) -> TextTokensPrompt:
230 token_num = len(input_ids)
231
232 # Note: EmbeddingRequest doesn't have max_tokens
@@ -211,13 +236,16 @@ class OpenAIServing:
236 f"This model's maximum context length is "
237 f"{self.max_model_len} tokens. However, you requested "
238 f"{token_num} tokens in the input for embedding "
214 - f"generation. Please reduce the length of the input.", )
215 - return input_ids, input_text
239 + f"generation. Please reduce the length of the input.")
240 + return TextTokensPrompt(prompt=input_text,
241 + prompt_token_ids=input_ids)
242
243 # Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens
244 # and does not require model context length validation
219 - if isinstance(request, (TokenizeRequest, DetokenizeRequest)):
220 - return input_ids, input_text
245 + if isinstance(request, (TokenizeCompletionRequest, TokenizeChatRequest,
246 + DetokenizeRequest)):
247 + return TextTokensPrompt(prompt=input_text,
248 + prompt_token_ids=input_ids)
249
250 if request.max_tokens is None:
251 if token_num >= self.max_model_len:
@@ -225,7 +253,7 @@ class OpenAIServing:
253 f"This model's maximum context length is "
254 f"{self.max_model_len} tokens. However, you requested "
255 f"{token_num} tokens in the messages, "
228 - f"Please reduce the length of the messages.", )
256 + f"Please reduce the length of the messages.")
257 request.max_tokens = self.max_model_len - token_num
258
259 if token_num + request.max_tokens > self.max_model_len:
@@ -235,13 +263,132 @@ class OpenAIServing:
263 f"{request.max_tokens + token_num} tokens "
264 f"({token_num} in the messages, "
265 f"{request.max_tokens} in the completion). "
238 - f"Please reduce the length of the messages or completion.", )
266 + f"Please reduce the length of the messages or completion.")
267 +
268 + return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids)
269 +
270 + def _tokenize_prompt_input(
271 + self,
272 + request: AnyRequest,
273 + tokenizer: AnyTokenizer,
274 + prompt_input: Union[str, List[int]],
275 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
276 + add_special_tokens: bool = True,
277 + ) -> TextTokensPrompt:
278 + """
279 + A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs`
280 + that assumes single input.
281 + """
282 + return next(
283 + self._tokenize_prompt_inputs(
284 + request,
285 + tokenizer,
286 + [prompt_input],
287 + truncate_prompt_tokens=truncate_prompt_tokens,
288 + add_special_tokens=add_special_tokens,
289 + ))
290 +
291 + def _tokenize_prompt_inputs(
292 + self,
293 + request: AnyRequest,
294 + tokenizer: AnyTokenizer,
295 + prompt_inputs: Iterable[Union[str, List[int]]],
296 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
297 + add_special_tokens: bool = True,
298 + ) -> Iterator[TextTokensPrompt]:
299 + """
300 + A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs`
301 + that assumes multiple inputs.
302 + """
303 + for text in prompt_inputs:
304 + if isinstance(text, str):
305 + yield self._normalize_prompt_text_to_input(
306 + request,
307 + tokenizer,
308 + prompt=text,
309 + truncate_prompt_tokens=truncate_prompt_tokens,
310 + add_special_tokens=add_special_tokens,
311 + )
312 + else:
313 + yield self._normalize_prompt_tokens_to_input(
314 + request,
315 + tokenizer,
316 + prompt_ids=text,
317 + truncate_prompt_tokens=truncate_prompt_tokens,
318 + )
319 +
320 + def _tokenize_prompt_input_or_inputs(
321 + self,
322 + request: AnyRequest,
323 + tokenizer: AnyTokenizer,
324 + input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
325 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
326 + add_special_tokens: bool = True,
327 + ) -> Iterator[TextTokensPrompt]:
328 + """
329 + Tokenize/detokenize depending on the input format.
330 +
331 + According to `OpenAI API <https://platform.openai.com/docs/api-reference/embeddings/create>`_
332 + , each input can be a string or array of tokens. Note that each request
333 + can pass one or more inputs.
334 + """
335 + for prompt_input in parse_and_batch_prompt(input_or_inputs):
336 + # Although our type checking is based on mypy,
337 + # VSCode Pyright extension should still work properly
338 + # "is True" is required for Pyright to perform type narrowing
339 + # See: https://github.com/microsoft/pyright/issues/7672
340 + if prompt_input["is_tokens"] is False:
341 + yield self._normalize_prompt_text_to_input(
342 + request,
343 + tokenizer,
344 + prompt=prompt_input["content"],
345 + truncate_prompt_tokens=truncate_prompt_tokens,
346 + add_special_tokens=add_special_tokens,
347 + )
348 + else:
349 + yield self._normalize_prompt_tokens_to_input(
350 + request,
351 + tokenizer,
352 + prompt_ids=prompt_input["content"],
353 + truncate_prompt_tokens=truncate_prompt_tokens,
354 + )
355 +
356 + def _log_inputs(
357 + self,
358 + request_id: str,
359 + inputs: Union[str, List[int], TextTokensPrompt],
360 + params: Optional[Union[SamplingParams, PoolingParams]],
361 + lora_request: Optional[LoRARequest],
362 + prompt_adapter_request: Optional[PromptAdapterRequest],
363 + ) -> None:
364 + if self.request_logger is None:
365 + return
366 +
367 + if isinstance(inputs, str):
368 + prompt = inputs
369 + prompt_token_ids = None
370 + elif isinstance(inputs, list):
371 + prompt = None
372 + prompt_token_ids = inputs
373 else:
240 - return input_ids, input_text
374 + prompt = inputs["prompt"]
375 + prompt_token_ids = inputs["prompt_token_ids"]
376 +
377 + self.request_logger.log_inputs(
378 + request_id,
379 + prompt,
380 + prompt_token_ids,
381 + params=params,
382 + lora_request=lora_request,
383 + prompt_adapter_request=prompt_adapter_request,
384 + )
385
386 @staticmethod
243 - def _get_decoded_token(logprob: Logprob, token_id: int,
244 - tokenizer: PreTrainedTokenizer) -> str:
387 + def _get_decoded_token(
388 + logprob: Logprob,
389 + token_id: int,
390 + tokenizer: AnyTokenizer,
391 + ) -> str:
392 if logprob.decoded_token is not None:
393 return logprob.decoded_token
394 return tokenizer.decode(token_id)
vllm/entrypoints/openai/serving_tokenization.py
+79 -27
@@ -1,83 +1,135 @@
1 -from typing import List, Optional
1 +from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 from vllm.engine.async_llm_engine import AsyncLLMEngine
5 +# yapf conflicts with isort for this block
6 +# yapf: disable
7 from vllm.entrypoints.chat_utils import (ConversationMessage,
8 load_chat_template,
9 parse_chat_message_content)
10 +from vllm.entrypoints.logger import RequestLogger
11 from vllm.entrypoints.openai.protocol import (DetokenizeRequest,
12 DetokenizeResponse,
13 + ErrorResponse,
14 + TokenizeChatRequest,
15 TokenizeRequest,
16 TokenizeResponse)
17 +# yapf: enable
18 from vllm.entrypoints.openai.serving_engine import (LoRAModulePath,
19 OpenAIServing)
20 +from vllm.utils import random_uuid
21
22
23 class OpenAIServingTokenization(OpenAIServing):
24
18 - def __init__(self,
19 - engine: AsyncLLMEngine,
20 - model_config: ModelConfig,
21 - served_model_names: List[str],
22 - lora_modules: Optional[List[LoRAModulePath]] = None,
23 - chat_template: Optional[str] = None):
25 + def __init__(
26 + self,
27 + engine: AsyncLLMEngine,
28 + model_config: ModelConfig,
29 + served_model_names: List[str],
30 + *,
31 + lora_modules: Optional[List[LoRAModulePath]],
32 + request_logger: Optional[RequestLogger],
33 + chat_template: Optional[str],
34 + ):
35 super().__init__(engine=engine,
36 model_config=model_config,
37 served_model_names=served_model_names,
27 - lora_modules=lora_modules)
38 + lora_modules=lora_modules,
39 + prompt_adapters=None,
40 + request_logger=request_logger)
41
42 # If this is None we use the tokenizer's default chat template
43 self.chat_template = load_chat_template(chat_template)
44
32 - async def create_tokenize(self,
33 - request: TokenizeRequest) -> TokenizeResponse:
45 + async def create_tokenize(
46 + self,
47 + request: TokenizeRequest,
48 + ) -> Union[TokenizeResponse, ErrorResponse]:
49 error_check_ret = await self._check_model(request)
50 if error_check_ret is not None:
51 return error_check_ret
52
38 - if not (request.prompt or request.messages):
39 - return self.create_error_response(
40 - "Either `prompt` or `messages` should be provided.")
53 + request_id = f"tokn-{random_uuid()}"
54
42 - if (request.prompt and request.messages):
43 - return self.create_error_response(
44 - "Only one of `prompt` or `messages` should be provided.")
55 + (
56 + lora_request,
57 + prompt_adapter_request,
58 + ) = self._maybe_get_adapters(request)
59
46 - _, lora_request = self._maybe_get_adapter(request)
60 tokenizer = await self.engine.get_tokenizer(lora_request)
48 - if request.messages:
61 +
62 + if isinstance(request, TokenizeChatRequest):
63 + model_config = self.model_config
64 +
65 conversation: List[ConversationMessage] = []
66
67 for message in request.messages:
52 - result = parse_chat_message_content(message, self.model_config,
68 + result = parse_chat_message_content(message, model_config,
69 tokenizer)
70 conversation.extend(result.messages)
71
56 - request.prompt = tokenizer.apply_chat_template(
72 + prompt = tokenizer.apply_chat_template(
73 add_generation_prompt=request.add_generation_prompt,
74 conversation=conversation,
75 tokenize=False,
76 chat_template=self.chat_template)
77 + assert isinstance(prompt, str)
78 + else:
79 + prompt = request.prompt
80 +
81 + self._log_inputs(request_id,
82 + prompt,
83 + params=None,
84 + lora_request=lora_request,
85 + prompt_adapter_request=prompt_adapter_request)
86
62 - (input_ids, input_text) = await self._validate_prompt_and_tokenize(
87 + # Silently ignore prompt adapter since it does not affect tokenization
88 +
89 + prompt_input = self._tokenize_prompt_input(
90 request,
91 tokenizer,
65 - prompt=request.prompt,
66 - add_special_tokens=request.add_special_tokens)
92 + prompt,
93 + add_special_tokens=request.add_special_tokens,
94 + )
95 + input_ids = prompt_input["prompt_token_ids"]
96
97 return TokenizeResponse(tokens=input_ids,
98 count=len(input_ids),
99 max_model_len=self.max_model_len)
100
101 async def create_detokenize(
73 - self, request: DetokenizeRequest) -> DetokenizeResponse:
102 + self,
103 + request: DetokenizeRequest,
104 + ) -> Union[DetokenizeResponse, ErrorResponse]:
105 error_check_ret = await self._check_model(request)
106 if error_check_ret is not None:
107 return error_check_ret
108
78 - _, lora_request = self._maybe_get_adapter(request)
109 + request_id = f"tokn-{random_uuid()}"
110 +
111 + (
112 + lora_request,
113 + prompt_adapter_request,
114 + ) = self._maybe_get_adapters(request)
115 +
116 tokenizer = await self.engine.get_tokenizer(lora_request)
80 - (input_ids, input_text) = await self._validate_prompt_and_tokenize(
81 - request, tokenizer, prompt_ids=request.tokens)
117 +
118 + self._log_inputs(request_id,
119 + request.tokens,
120 + params=None,
121 + lora_request=lora_request,
122 + prompt_adapter_request=prompt_adapter_request)
123 +
124 + if prompt_adapter_request is not None:
125 + raise NotImplementedError("Prompt adapter is not supported "
126 + "for tokenization")
127 +
128 + prompt_input = self._tokenize_prompt_input(
129 + request,
130 + tokenizer,
131 + request.tokens,
132 + )
133 + input_text = prompt_input["prompt"]
134
135 return DetokenizeResponse(prompt=input_text)
vllm/inputs/__init__.py
+3 -4
@@ -1,6 +1,5 @@
1 from .data import (LLMInputs, ParsedText, ParsedTokens, PromptInputs,
2 - PromptStrictInputs, TextPrompt, TextTokensPrompt,
3 - TokensPrompt, parse_and_batch_prompt)
2 + TextPrompt, TokensPrompt, parse_and_batch_prompt)
3 from .registry import InputContext, InputRegistry
4
5 INPUT_REGISTRY = InputRegistry()
@@ -14,6 +13,6 @@ See also:
13
14 __all__ = [
15 "ParsedText", "ParsedTokens", "parse_and_batch_prompt", "TextPrompt",
17 - "TokensPrompt", "TextTokensPrompt", "PromptStrictInputs", "PromptInputs",
18 - "LLMInputs", "INPUT_REGISTRY", "InputContext", "InputRegistry"
16 + "TokensPrompt", "PromptInputs", "LLMInputs", "INPUT_REGISTRY",
17 + "InputContext", "InputRegistry"
18 ]
vllm/inputs/data.py
+1 -23
@@ -92,25 +92,7 @@ class TokensPrompt(TypedDict):
92 """
93
94
95 -class TextTokensPrompt(TypedDict):
96 - """It is assumed that :attr:`prompt` is consistent with
97 - :attr:`prompt_token_ids`. This is currently used in
98 - :class:`AsyncLLMEngine` for logging both the text and token IDs."""
99 -
100 - prompt: str
101 - """The prompt text."""
102 -
103 - prompt_token_ids: List[int]
104 - """The token IDs of the prompt."""
105 -
106 - multi_modal_data: NotRequired["MultiModalDataDict"]
107 - """
108 - Optional multi-modal data to pass to the model,
109 - if the model supports it.
110 - """
111 -
112 -
113 -PromptStrictInputs = Union[str, TextPrompt, TokensPrompt]
95 +PromptInputs = Union[str, TextPrompt, TokensPrompt]
96 """
97 The inputs to the LLM, which can take one of the following forms:
98
@@ -118,10 +100,6 @@ The inputs to the LLM, which can take one of the following forms:
100 - A tokenized prompt (:class:`TokensPrompt`)
101 """
102
121 -PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
122 -"""Same as :const:`PromptStrictInputs` but additionally accepts
123 -:class:`TextTokensPrompt`."""
124 -
103
104 class LLMInputs(TypedDict):
105 """
vllm/sequence.py
+3 -2
@@ -5,7 +5,8 @@ import math
5 from abc import ABC, abstractmethod
6 from collections import defaultdict
7 from dataclasses import dataclass, field
8 -from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
8 +from typing import (TYPE_CHECKING, Dict, List, Mapping, Optional, Set, Tuple,
9 + Union)
10
11 import torch
12
@@ -438,7 +439,7 @@ class SequenceGroup:
439 embeddings: Optional[List[float]] = None,
440 pooling_params: Optional[PoolingParams] = None,
441 encoder_seq: Optional[Sequence] = None,
441 - trace_headers: Optional[Dict[str, str]] = None,
442 + trace_headers: Optional[Mapping[str, str]] = None,
443 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
444 ) -> None:
445 self.request_id = request_id