@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0057894ef7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Rename `PromptInputs` and `inputs`(#8673)

Cyrus Leung committed Sep 21, 2024 at 10:00 UTC 0057894ef7f8db0d51385aa7254219d7fbd6c784
18 files changed +157 -162
benchmarks/benchmark_latency.py
+4 -4
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14 -from vllm.inputs import PromptInputs
14 +from vllm.inputs import PromptType
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
17
@@ -61,7 +61,7 @@ def main(args: argparse.Namespace):
61 dummy_prompt_token_ids = np.random.randint(10000,
62 size=(args.batch_size,
63 args.input_len))
64 - dummy_inputs: List[PromptInputs] = [{
64 + dummy_prompts: List[PromptType] = [{
65 "prompt_token_ids": batch
66 } for batch in dummy_prompt_token_ids.tolist()]
67
@@ -74,13 +74,13 @@ def main(args: argparse.Namespace):
74 ],
75 on_trace_ready=torch.profiler.tensorboard_trace_handler(
76 str(profile_dir))) as p:
77 - llm.generate(dummy_inputs,
77 + llm.generate(dummy_prompts,
78 sampling_params=sampling_params,
79 use_tqdm=False)
80 print(p.key_averages())
81 else:
82 start_time = time.perf_counter()
83 - llm.generate(dummy_inputs,
83 + llm.generate(dummy_prompts,
84 sampling_params=sampling_params,
85 use_tqdm=False)
86 end_time = time.perf_counter()
docs/source/dev/multimodal/multimodal_index.rst
+1 -1
@@ -8,7 +8,7 @@ Multi-Modality
8 vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
9
10 Multi-modal inputs can be passed alongside text and token prompts to :ref:`supported models <supported_vlms>`
11 -via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptInputs`.
11 +via the ``multi_modal_data`` field in :class:`vllm.inputs.PromptType`.
12
13 Currently, vLLM only has built-in support for image data. You can extend vLLM to process additional modalities
14 by following :ref:`this guide <adding_multimodal_plugin>`.
docs/source/dev/offline_inference/llm_inputs.rst
+1 -1
@@ -1,7 +1,7 @@
1 LLM Inputs
2 ==========
3
4 -.. autodata:: vllm.inputs.PromptInputs
4 +.. autodata:: vllm.inputs.PromptType
5
6 .. autoclass:: vllm.inputs.TextPrompt
7 :show-inheritance:
docs/source/models/vlm.rst
+1 -1
@@ -27,7 +27,7 @@ The :class:`~vllm.LLM` class can be instantiated in much the same way as languag
27 We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28 the above snippet. Specifically, ``image_feature_size`` can no longer be specified as we now calculate that internally for each model.
29
30 -To pass an image to the model, note the following in :class:`vllm.inputs.PromptInputs`:
30 +To pass an image to the model, note the following in :class:`vllm.inputs.PromptType`:
31
32 * ``prompt``: The prompt should follow the format that is documented on HuggingFace.
33 * ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
tests/mq_llm_engine/test_error_handling.py
+6 -6
@@ -61,7 +61,7 @@ async def test_evil_forward(tmp_socket):
61
62 # Throws an error in first forward pass.
63 with pytest.raises(RAISED_ERROR):
64 - async for _ in client.generate(inputs="Hello my name is",
64 + async for _ in client.generate(prompt="Hello my name is",
65 sampling_params=SamplingParams(),
66 request_id=uuid.uuid4()):
67 pass
@@ -69,7 +69,7 @@ async def test_evil_forward(tmp_socket):
69
70 # Engine is errored, should get ENGINE_DEAD_ERROR.
71 with pytest.raises(MQEngineDeadError):
72 - async for _ in client.generate(inputs="Hello my name is",
72 + async for _ in client.generate(prompt="Hello my name is",
73 sampling_params=SamplingParams(),
74 request_id=uuid.uuid4()):
75 pass
@@ -118,7 +118,7 @@ async def test_failed_health_check(tmp_socket):
118
119 # Generate call should throw ENGINE_DEAD_ERROR
120 with pytest.raises(MQEngineDeadError):
121 - async for _ in client.generate(inputs="Hello my name is",
121 + async for _ in client.generate(prompt="Hello my name is",
122 sampling_params=SamplingParams(),
123 request_id=uuid.uuid4()):
124 pass
@@ -165,7 +165,7 @@ async def test_failed_abort(tmp_socket):
165 # with reference to the original KeyError("foo")
166 with pytest.raises(MQEngineDeadError) as execinfo:
167 async for _ in client.generate(
168 - inputs="Hello my name is",
168 + prompt="Hello my name is",
169 sampling_params=SamplingParams(max_tokens=2000),
170 request_id=uuid.uuid4()):
171 pass
@@ -190,7 +190,7 @@ async def test_bad_request(tmp_socket):
190
191 # Invalid request should fail, but not crash the server.
192 with pytest.raises(ValueError):
193 - async for _ in client.generate(inputs="Hello my name is",
193 + async for _ in client.generate(prompt="Hello my name is",
194 sampling_params=SamplingParams(),
195 request_id="abcd-1",
196 lora_request=LoRARequest(
@@ -199,7 +199,7 @@ async def test_bad_request(tmp_socket):
199 pass
200
201 # This request should be okay.
202 - async for _ in client.generate(inputs="Hello my name is",
202 + async for _ in client.generate(prompt="Hello my name is",
203 sampling_params=SamplingParams(),
204 request_id="abcd-2"):
205 pass
tests/mq_llm_engine/utils.py
+1 -1
@@ -20,7 +20,7 @@ async def generate(
20 count = 0
21 async for out in client.generate(
22 request_id=request_id,
23 - inputs="Hello my name is Robert and",
23 + prompt="Hello my name is Robert and",
24 sampling_params=SamplingParams(max_tokens=num_tokens,
25 temperature=0)):
26
vllm/__init__.py
+2 -2
@@ -5,7 +5,7 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 -from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
8 +from vllm.inputs import PromptType, TextPrompt, TokensPrompt
9 from vllm.model_executor.models import ModelRegistry
10 from vllm.outputs import (CompletionOutput, EmbeddingOutput,
11 EmbeddingRequestOutput, RequestOutput)
@@ -19,7 +19,7 @@ __all__ = [
19 "__version__",
20 "LLM",
21 "ModelRegistry",
22 - "PromptInputs",
22 + "PromptType",
23 "TextPrompt",
24 "TokensPrompt",
25 "SamplingParams",
vllm/engine/async_llm_engine.py
+11 -13
@@ -17,7 +17,7 @@ from vllm.engine.metrics_types import StatLoggerBase
17 from vllm.executor.executor_base import ExecutorAsyncBase
18 from vllm.executor.gpu_executor import GPUExecutorAsync
19 from vllm.executor.ray_utils import initialize_ray_cluster
20 -from vllm.inputs import PromptInputs
20 +from vllm.inputs import PromptType
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -405,7 +405,7 @@ class _AsyncLLMEngine(LLMEngine):
405 async def add_request_async(
406 self,
407 request_id: str,
408 - inputs: PromptInputs,
408 + prompt: PromptType,
409 params: Union[SamplingParams, PoolingParams],
410 arrival_time: Optional[float] = None,
411 lora_request: Optional[LoRARequest] = None,
@@ -420,7 +420,7 @@ class _AsyncLLMEngine(LLMEngine):
420 arrival_time = time.time()
421
422 preprocessed_inputs = await self.input_preprocessor.preprocess_async(
423 - inputs,
423 + prompt,
424 request_id=request_id,
425 lora_request=lora_request,
426 prompt_adapter_request=prompt_adapter_request,
@@ -777,7 +777,7 @@ class AsyncLLMEngine:
777 async def add_request(
778 self,
779 request_id: str,
780 - inputs: PromptInputs,
780 + prompt: PromptType,
781 params: Union[SamplingParams, PoolingParams],
782 arrival_time: Optional[float] = None,
783 lora_request: Optional[LoRARequest] = None,
@@ -797,7 +797,7 @@ class AsyncLLMEngine:
797 stream = self._request_tracker.add_request(
798 request_id,
799 verbose=self.log_requests,
800 - inputs=inputs,
800 + prompt=prompt,
801 params=params,
802 arrival_time=arrival_time or time.time(),
803 lora_request=lora_request,
@@ -808,7 +808,7 @@ class AsyncLLMEngine:
808
809 async def generate(
810 self,
811 - inputs: PromptInputs,
811 + prompt: PromptType,
812 sampling_params: SamplingParams,
813 request_id: str,
814 lora_request: Optional[LoRARequest] = None,
@@ -822,8 +822,7 @@ class AsyncLLMEngine:
822 from the LLMEngine to the caller.
823
824 Args:
825 - inputs: The inputs to the LLM. See
826 - :class:`~vllm.inputs.PromptInputs`
825 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
826 for more details about the format of each input.
827 sampling_params: The sampling parameters of the request.
828 request_id: The unique id of the request.
@@ -881,7 +880,7 @@ class AsyncLLMEngine:
880 """
881 async for output in await self.add_request(
882 request_id,
884 - inputs,
883 + prompt,
884 sampling_params,
885 lora_request=lora_request,
886 trace_headers=trace_headers,
@@ -891,7 +890,7 @@ class AsyncLLMEngine:
890
891 async def encode(
892 self,
894 - inputs: PromptInputs,
893 + prompt: PromptType,
894 pooling_params: PoolingParams,
895 request_id: str,
896 lora_request: Optional[LoRARequest] = None,
@@ -904,8 +903,7 @@ class AsyncLLMEngine:
903 from the LLMEngine to the caller.
904
905 Args:
907 - inputs: The inputs to the LLM. See
908 - :class:`~vllm.inputs.PromptInputs`
906 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
907 for more details about the format of each input.
908 pooling_params: The pooling parameters of the request.
909 request_id: The unique id of the request.
@@ -959,7 +957,7 @@ class AsyncLLMEngine:
957 """
958 async for output in await self.add_request(
959 request_id,
962 - inputs,
960 + prompt,
961 pooling_params,
962 lora_request=lora_request,
963 trace_headers=trace_headers,
vllm/engine/llm_engine.py
+4 -5
@@ -29,7 +29,7 @@ from vllm.executor.executor_base import ExecutorBase
29 from vllm.executor.gpu_executor import GPUExecutor
30 from vllm.executor.ray_utils import initialize_ray_cluster
31 from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
32 - InputRegistry, LLMInputs, PromptInputs)
32 + InputRegistry, LLMInputs, PromptType)
33 from vllm.inputs.preprocess import InputPreprocessor
34 from vllm.logger import init_logger
35 from vllm.lora.request import LoRARequest
@@ -680,7 +680,7 @@ class LLMEngine:
680 def add_request(
681 self,
682 request_id: str,
683 - inputs: PromptInputs,
683 + prompt: PromptType,
684 params: Union[SamplingParams, PoolingParams],
685 arrival_time: Optional[float] = None,
686 lora_request: Optional[LoRARequest] = None,
@@ -695,8 +695,7 @@ class LLMEngine:
695
696 Args:
697 request_id: The unique ID of the request.
698 - inputs: The inputs to the LLM. See
699 - :class:`~vllm.inputs.PromptInputs`
698 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
699 for more details about the format of each input.
700 params: Parameters for sampling or pooling.
701 :class:`~vllm.SamplingParams` for text generation.
@@ -736,7 +735,7 @@ class LLMEngine:
735 arrival_time = time.time()
736
737 preprocessed_inputs = self.input_preprocessor.preprocess(
739 - inputs,
738 + prompt,
739 request_id=request_id,
740 lora_request=lora_request,
741 prompt_adapter_request=prompt_adapter_request,
vllm/engine/multiprocessing/__init__.py
+2 -2
@@ -3,7 +3,7 @@ from enum import Enum
3 from typing import List, Mapping, Optional, Union
4
5 from vllm import PoolingParams
6 -from vllm.inputs import PromptInputs
6 +from vllm.inputs import PromptType
7 from vllm.lora.request import LoRARequest
8 from vllm.outputs import RequestOutput
9 from vllm.prompt_adapter.request import PromptAdapterRequest
@@ -23,7 +23,7 @@ class MQEngineDeadError(RuntimeError):
23
24 @dataclass
25 class RPCProcessRequest:
26 - inputs: PromptInputs
26 + prompt: PromptType
27 params: Union[SamplingParams, PoolingParams]
28 request_id: str
29 lora_request: Optional[LoRARequest] = None
vllm/engine/multiprocessing/client.py
+9 -11
@@ -25,7 +25,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
25 RPCStartupResponse)
26 # yapf: enable
27 from vllm.envs import VLLM_RPC_TIMEOUT
28 -from vllm.inputs import PromptInputs
28 +from vllm.inputs import PromptType
29 from vllm.logger import init_logger
30 from vllm.lora.request import LoRARequest
31 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -375,7 +375,7 @@ class MQLLMEngineClient:
375
376 def generate(
377 self,
378 - inputs: PromptInputs,
378 + prompt: PromptType,
379 sampling_params: SamplingParams,
380 request_id: str,
381 lora_request: Optional[LoRARequest] = None,
@@ -389,8 +389,7 @@ class MQLLMEngineClient:
389 from the LLMEngine to the caller.
390
391 Args:
392 - inputs: The inputs to the LLM. See
393 - :class:`~vllm.inputs.PromptInputs`
392 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
393 for more details about the format of each input.
394 sampling_params: The sampling parameters of the request.
395 request_id: The unique id of the request.
@@ -399,13 +398,13 @@ class MQLLMEngineClient:
398 prompt_adapter_request: Prompt Adapter request to use
399 for generation, if any.
400 """
402 - return self._process_request(inputs, sampling_params, request_id,
401 + return self._process_request(prompt, sampling_params, request_id,
402 lora_request, trace_headers,
403 prompt_adapter_request)
404
405 def encode(
406 self,
408 - inputs: PromptInputs,
407 + prompt: PromptType,
408 pooling_params: PoolingParams,
409 request_id: str,
410 lora_request: Optional[LoRARequest] = None,
@@ -418,8 +417,7 @@ class MQLLMEngineClient:
417 from the LLMEngine to the caller.
418
419 Args:
421 - inputs: The inputs to the LLM. See
422 - :class:`~vllm.inputs.PromptInputs`
420 + prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
421 for more details about the format of each input.
422 pooling_params: The pooling parameters of the request.
423 request_id: The unique id of the request.
@@ -430,12 +428,12 @@ class MQLLMEngineClient:
428 The output `EmbeddingRequestOutput` objects from the LLMEngine
429 for the request.
430 """
433 - return self._process_request(inputs, pooling_params, request_id,
431 + return self._process_request(prompt, pooling_params, request_id,
432 lora_request, trace_headers)
433
434 async def _process_request(
435 self,
438 - inputs: PromptInputs,
436 + prompt: PromptType,
437 params: Union[SamplingParams, PoolingParams],
438 request_id: str,
439 lora_request: Optional[LoRARequest] = None,
@@ -468,7 +466,7 @@ class MQLLMEngineClient:
466
467 request_bytes = pickle.dumps(
468 RPCProcessRequest(
471 - inputs=inputs,
469 + prompt=prompt,
470 params=params,
471 request_id=request_id,
472 lora_request=lora_request,
vllm/engine/multiprocessing/engine.py
+1 -1
@@ -245,7 +245,7 @@ class MQLLMEngine:
245 try:
246 self.engine.add_request(
247 request_id=request_id,
248 - inputs=request.inputs,
248 + prompt=request.prompt,
249 params=request.params,
250 lora_request=request.lora_request,
251 trace_headers=request.trace_headers,
vllm/engine/protocol.py
+4 -4
@@ -3,7 +3,7 @@ from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
3
4 from vllm.config import DecodingConfig, ModelConfig
5 from vllm.core.scheduler import SchedulerOutputs
6 -from vllm.inputs.data import PromptInputs
6 +from vllm.inputs.data import PromptType
7 from vllm.lora.request import LoRARequest
8 from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
@@ -35,19 +35,19 @@ class EngineClient(Protocol):
35
36 def generate(
37 self,
38 - inputs: PromptInputs,
38 + prompt: PromptType,
39 sampling_params: SamplingParams,
40 request_id: str,
41 lora_request: Optional[LoRARequest] = None,
42 trace_headers: Optional[Mapping[str, str]] = None,
43 prompt_adapter_request: Optional[PromptAdapterRequest] = None
44 ) -> AsyncGenerator[RequestOutput, None]:
45 - """Generates outputs for a request"""
45 + """Generate outputs for a request."""
46 ...
47
48 def encode(
49 self,
50 - inputs: PromptInputs,
50 + prompt: PromptType,
51 pooling_params: PoolingParams,
52 request_id: str,
53 lora_request: Optional[LoRARequest] = None,
vllm/entrypoints/llm.py
+42 -38
@@ -10,7 +10,7 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
10 apply_hf_chat_template,
11 apply_mistral_chat_template,
12 parse_chat_messages)
13 -from vllm.inputs import PromptInputs, TextPrompt, TokensPrompt
13 +from vllm.inputs import PromptType, TextPrompt, TokensPrompt
14 from vllm.inputs.parse import parse_and_batch_prompt
15 from vllm.logger import init_logger
16 from vllm.lora.request import LoRARequest
@@ -258,8 +258,8 @@ class LLM:
258 @overload
259 def generate(
260 self,
261 - inputs: Union[PromptInputs, Sequence[PromptInputs]],
262 - /, # We may enable `inputs` keyword after removing the old API
261 + prompts: Union[PromptType, Sequence[PromptType]],
262 + /,
263 *,
264 sampling_params: Optional[Union[SamplingParams,
265 Sequence[SamplingParams]]] = None,
@@ -276,7 +276,7 @@ class LLM:
276 )
277 def generate(
278 self,
279 - prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
279 + prompts: Union[Union[PromptType, Sequence[PromptType]],
280 Optional[Union[str, List[str]]]] = None,
281 sampling_params: Optional[Union[SamplingParams,
282 Sequence[SamplingParams]]] = None,
@@ -294,7 +294,9 @@ class LLM:
294 into a single list and pass it to this method.
295
296 Args:
297 - inputs: A list of inputs to generate completions for.
297 + prompts: The prompts to the LLM. You may pass a sequence of prompts
298 + for batch inference. See :class:`~vllm.inputs.PromptType`
299 + for more details about the format of each prompts.
300 sampling_params: The sampling parameters for text generation. If
301 None, we use the default sampling parameters.
302 When it is a single value, it is applied to every prompt.
@@ -320,12 +322,13 @@ class LLM:
322 "models (XForCausalLM, XForConditionalGeneration).")
323
324 if prompt_token_ids is not None:
323 - inputs = self._convert_v1_inputs(
325 + parsed_prompts = self._convert_v1_inputs(
326 prompts=cast(Optional[Union[str, List[str]]], prompts),
327 prompt_token_ids=prompt_token_ids,
328 )
329 else:
328 - inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
330 + parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
331 + prompts)
332
333 if isinstance(guided_options_request, dict):
334 if len(guided_options_request) > 1:
@@ -340,7 +343,7 @@ class LLM:
343 sampling_params = SamplingParams()
344
345 self._validate_and_add_requests(
343 - inputs=inputs,
346 + prompts=parsed_prompts,
347 params=sampling_params,
348 lora_request=lora_request,
349 prompt_adapter_request=prompt_adapter_request,
@@ -396,9 +399,9 @@ class LLM:
399 conversation, mm_data = parse_chat_messages(messages, model_config,
400 tokenizer)
401
399 - prompt: Union[str, List[int]]
402 + prompt_data: Union[str, List[int]]
403 if isinstance(tokenizer, MistralTokenizer):
401 - prompt = apply_mistral_chat_template(
404 + prompt_data = apply_mistral_chat_template(
405 tokenizer,
406 messages=messages,
407 chat_template=chat_template,
@@ -406,7 +409,7 @@ class LLM:
409 tools=tools,
410 )
411 else:
409 - prompt = apply_hf_chat_template(
412 + prompt_data = apply_hf_chat_template(
413 tokenizer,
414 conversation=conversation,
415 chat_template=chat_template,
@@ -414,17 +417,17 @@ class LLM:
417 tools=tools,
418 )
419
417 - inputs: PromptInputs
418 - if is_list_of(prompt, int):
419 - inputs = TokensPrompt(prompt_token_ids=prompt)
420 + prompt: PromptType
421 + if is_list_of(prompt_data, int):
422 + prompt = TokensPrompt(prompt_token_ids=prompt_data)
423 else:
421 - inputs = TextPrompt(prompt=prompt)
424 + prompt = TextPrompt(prompt=prompt_data)
425
426 if mm_data is not None:
424 - inputs["multi_modal_data"] = mm_data
427 + prompt["multi_modal_data"] = mm_data
428
429 return self.generate(
427 - inputs,
430 + prompt,
431 sampling_params=sampling_params,
432 use_tqdm=use_tqdm,
433 lora_request=lora_request,
@@ -494,8 +497,8 @@ class LLM:
497 @overload
498 def encode(
499 self,
497 - inputs: Union[PromptInputs, Sequence[PromptInputs]],
498 - /, # We may enable `inputs` keyword after removing the old API
500 + prompts: Union[PromptType, Sequence[PromptType]],
501 + /,
502 *,
503 pooling_params: Optional[Union[PoolingParams,
504 Sequence[PoolingParams]]] = None,
@@ -512,7 +515,7 @@ class LLM:
515 )
516 def encode(
517 self,
515 - prompts: Union[Union[PromptInputs, Sequence[PromptInputs]],
518 + prompts: Union[Union[PromptType, Sequence[PromptType]],
519 Optional[Union[str, List[str]]]] = None,
520 pooling_params: Optional[Union[PoolingParams,
521 Sequence[PoolingParams]]] = None,
@@ -528,9 +531,9 @@ class LLM:
531 into a single list and pass it to this method.
532
533 Args:
531 - inputs: The inputs to the LLM. You may pass a sequence of inputs for
532 - batch inference. See :class:`~vllm.inputs.PromptInputs`
533 - for more details about the format of each input.
534 + prompts: The prompts to the LLM. You may pass a sequence of prompts
535 + for batch inference. See :class:`~vllm.inputs.PromptType`
536 + for more details about the format of each prompts.
537 pooling_params: The pooling parameters for pooling. If None, we
538 use the default pooling parameters.
539 use_tqdm: Whether to use tqdm to display the progress bar.
@@ -553,19 +556,20 @@ class LLM:
556 )
557
558 if prompt_token_ids is not None:
556 - inputs = self._convert_v1_inputs(
559 + parsed_prompts = self._convert_v1_inputs(
560 prompts=cast(Optional[Union[str, List[str]]], prompts),
561 prompt_token_ids=prompt_token_ids,
562 )
563 else:
561 - inputs = cast(Union[PromptInputs, Sequence[PromptInputs]], prompts)
564 + parsed_prompts = cast(Union[PromptType, Sequence[PromptType]],
565 + prompts)
566
567 if pooling_params is None:
568 # Use default pooling params.
569 pooling_params = PoolingParams()
570
571 self._validate_and_add_requests(
568 - inputs=inputs,
572 + prompts=parsed_prompts,
573 params=pooling_params,
574 lora_request=lora_request,
575 prompt_adapter_request=prompt_adapter_request,
@@ -609,9 +613,9 @@ class LLM:
613 raise ValueError("Either prompts or prompt_token_ids must be "
614 "provided.")
615
612 - inputs: List[PromptInputs] = []
616 + parsed_prompts: List[PromptType] = []
617 for i in range(num_requests):
614 - item: PromptInputs
618 + item: PromptType
619
620 if prompts is not None:
621 item = TextPrompt(prompt=prompts[i])
@@ -620,24 +624,24 @@ class LLM:
624 else:
625 raise AssertionError
626
623 - inputs.append(item)
627 + parsed_prompts.append(item)
628
625 - return inputs
629 + return parsed_prompts
630
631 def _validate_and_add_requests(
632 self,
629 - inputs: Union[PromptInputs, Sequence[PromptInputs]],
633 + prompts: Union[PromptType, Sequence[PromptType]],
634 params: Union[SamplingParams, Sequence[SamplingParams], PoolingParams,
635 Sequence[PoolingParams]],
636 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
637 prompt_adapter_request: Optional[PromptAdapterRequest],
638 guided_options: Optional[GuidedDecodingRequest] = None,
639 ) -> None:
636 - if isinstance(inputs, (str, dict)):
640 + if isinstance(prompts, (str, dict)):
641 # Convert a single prompt to a list.
638 - inputs = [inputs]
642 + prompts = [prompts]
643
640 - num_requests = len(inputs)
644 + num_requests = len(prompts)
645 if isinstance(params, list) and len(params) != num_requests:
646 raise ValueError("The lengths of prompts and params "
647 "must be the same.")
@@ -654,9 +658,9 @@ class LLM:
658 sp.output_kind = RequestOutputKind.FINAL_ONLY
659
660 # Add requests to the engine.
657 - for i, request_inputs in enumerate(inputs):
661 + for i, prompt in enumerate(prompts):
662 self._add_request(
659 - request_inputs,
663 + prompt,
664 params[i] if isinstance(params, Sequence) else params,
665 lora_request=lora_request[i] if isinstance(
666 lora_request, Sequence) else lora_request,
@@ -665,7 +669,7 @@ class LLM:
669
670 def _add_request(
671 self,
668 - inputs: PromptInputs,
672 + prompt: PromptType,
673 params: Union[SamplingParams, PoolingParams],
674 lora_request: Optional[LoRARequest] = None,
675 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -673,7 +677,7 @@ class LLM:
677 request_id = str(next(self.request_counter))
678 self.llm_engine.add_request(
679 request_id,
676 - inputs,
680 + prompt,
681 params,
682 lora_request=lora_request,
683 prompt_adapter_request=prompt_adapter_request,
vllm/inputs/__init__.py
+3 -3
@@ -1,5 +1,5 @@
1 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
2 - LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
2 + LLMInputs, PromptType, SingletonPrompt, TextPrompt,
3 TokensPrompt, build_explicit_enc_dec_prompt,
4 to_enc_dec_tuple_list, zip_enc_dec_prompts)
5 from .registry import InputContext, InputRegistry
@@ -16,8 +16,8 @@ See also:
16 __all__ = [
17 "TextPrompt",
18 "TokensPrompt",
19 - "PromptInputs",
20 - "SingletonPromptInputs",
19 + "PromptType",
20 + "SingletonPrompt",
21 "ExplicitEncoderDecoderPrompt",
22 "LLMInputs",
23 "EncoderDecoderLLMInputs",
vllm/inputs/data.py
+11 -15
@@ -33,7 +33,7 @@ class TokensPrompt(TypedDict):
33 """
34
35
36 -SingletonPromptInputs = Union[str, TextPrompt, TokensPrompt]
36 +SingletonPrompt = Union[str, TextPrompt, TokensPrompt]
37 """
38 Set of possible schemas for a single LLM input:
39
@@ -46,7 +46,7 @@ which may be utilized for encoder/decoder models when
46 the user desires to express both the encoder & decoder
47 prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
48
49 -A prompt of type :class:`SingletonPromptInputs` may be employed
49 +A prompt of type :class:`SingletonPromptType` may be employed
50 as (1) input to a decoder-only model, (2) input to
51 the encoder of an encoder/decoder model, in the scenario
52 where the decoder-prompt is not specified explicitly, or
@@ -55,12 +55,12 @@ more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
55 """
56
57 _T1_co = TypeVar("_T1_co",
58 - bound=SingletonPromptInputs,
59 - default=SingletonPromptInputs,
58 + bound=SingletonPrompt,
59 + default=SingletonPrompt,
60 covariant=True)
61 _T2_co = TypeVar("_T2_co",
62 - bound=SingletonPromptInputs,
63 - default=SingletonPromptInputs,
62 + bound=SingletonPrompt,
63 + default=SingletonPrompt,
64 covariant=True)
65
66
@@ -72,7 +72,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
72
73 The encoder and decoder prompts, respectively,
74 may formatted according to any of the
75 - :class:`SingletonPromptInputs` schemas, and are not
75 + :class:`SingletonPromptType` schemas, and are not
76 required to have the same schema.
77
78 Only the encoder prompt may have multi-modal data.
@@ -81,7 +81,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
81 be used as an input to a decoder-only model,
82 and that the `encoder_prompt` and `decoder_prompt`
83 fields of this data structure themselves must be
84 - :class:`SingletonPromptInputs` instances.
84 + :class:`SingletonPromptType` instances.
85 """
86
87 encoder_prompt: _T1_co
@@ -89,7 +89,7 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
89 decoder_prompt: Optional[_T2_co]
90
91
92 -PromptInputs = Union[SingletonPromptInputs, ExplicitEncoderDecoderPrompt]
92 +PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
93 """
94 Set of possible schemas for an LLM input, including
95 both decoder-only and encoder/decoder input types:
@@ -140,12 +140,8 @@ class EncoderDecoderLLMInputs(LLMInputs):
140 """
141
142
143 -_T1 = TypeVar("_T1",
144 - bound=SingletonPromptInputs,
145 - default=SingletonPromptInputs)
146 -_T2 = TypeVar("_T2",
147 - bound=SingletonPromptInputs,
148 - default=SingletonPromptInputs)
143 +_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
144 +_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
145
146
147 def build_explicit_enc_dec_prompt(
vllm/inputs/parse.py
+11 -11
@@ -5,7 +5,7 @@ from typing_extensions import TypeIs
5 from vllm.utils import is_list_of
6
7 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 - LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
8 + LLMInputs, PromptType, SingletonPrompt, TextPrompt,
9 TokensPrompt)
10
11
@@ -81,23 +81,23 @@ class ParsedTokensPrompt(TypedDict):
81
82
83 def parse_singleton_prompt(
84 - inputs: SingletonPromptInputs,
84 + prompt: SingletonPrompt,
85 ) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86 - if isinstance(inputs, str):
87 - return ParsedStrPrompt(type="str", content=inputs)
88 - elif isinstance(inputs, dict):
89 - if "prompt_token_ids" in inputs:
86 + if isinstance(prompt, str):
87 + return ParsedStrPrompt(type="str", content=prompt)
88 + elif isinstance(prompt, dict):
89 + if "prompt_token_ids" in prompt:
90 return ParsedTokensPrompt(type="tokens",
91 - content=inputs) # type: ignore
92 - elif "prompt" in inputs:
93 - return ParsedTextPrompt(type="text", content=inputs)
91 + content=prompt) # type: ignore
92 + elif "prompt" in prompt:
93 + return ParsedTextPrompt(type="text", content=prompt)
94
95 raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98 def is_explicit_encoder_decoder_prompt(
99 - inputs: PromptInputs) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 - return isinstance(inputs, dict) and "encoder_prompt" in inputs
99 + prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
100 + return isinstance(prompt, dict) and "encoder_prompt" in prompt
101
102
103 def is_valid_encoder_decoder_llm_inputs(
vllm/inputs/preprocess.py
+43 -43
@@ -9,8 +9,8 @@ from vllm.lora.request import LoRARequest
9 from vllm.prompt_adapter.request import PromptAdapterRequest
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
12 -from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13 - SingletonPromptInputs)
12 +from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptType,
13 + SingletonPrompt)
14 from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15
16 if TYPE_CHECKING:
@@ -206,7 +206,7 @@ class InputPreprocessor:
206
207 def _extract_prompt_components(
208 self,
209 - inputs: SingletonPromptInputs,
209 + prompt: SingletonPrompt,
210 request_id: str,
211 lora_request: Optional[LoRARequest] = None,
212 ) -> PromptComponents:
@@ -216,7 +216,7 @@ class InputPreprocessor:
216 Arguments:
217
218 * request_id
219 - * inputs: single encoder or decoder input prompt
219 + * prompt: single encoder or decoder input prompt
220 * lora_request: this is only valid for decoder prompts
221
222 Returns:
@@ -226,24 +226,24 @@ class InputPreprocessor:
226 * multi_modal_data
227 '''
228
229 - parsed = parse_singleton_prompt(inputs)
229 + parsed = parse_singleton_prompt(prompt)
230
231 if parsed["type"] == "str":
232 - prompt = parsed["content"]
232 + prompt_text = parsed["content"]
233 prompt_token_ids = self._tokenize_prompt(
234 - prompt,
234 + prompt_text,
235 request_id=request_id,
236 lora_request=lora_request,
237 )
238 multi_modal_data = None
239 elif parsed["type"] == "tokens":
240 - prompt = None
240 + prompt_text = None
241 prompt_token_ids = parsed["content"]["prompt_token_ids"]
242 multi_modal_data = parsed["content"].get("multi_modal_data")
243 elif parsed["type"] == "text":
244 - prompt = parsed["content"]["prompt"]
244 + prompt_text = parsed["content"]["prompt"]
245 prompt_token_ids = self._tokenize_prompt(
246 - prompt,
246 + prompt_text,
247 request_id=request_id,
248 lora_request=lora_request,
249 )
@@ -251,33 +251,33 @@ class InputPreprocessor:
251 else:
252 assert_never(parsed)
253
254 - return prompt, prompt_token_ids, multi_modal_data
254 + return prompt_text, prompt_token_ids, multi_modal_data
255
256 async def _extract_prompt_components_async(
257 self,
258 - inputs: SingletonPromptInputs,
258 + prompt: SingletonPrompt,
259 request_id: str,
260 lora_request: Optional[LoRARequest] = None,
261 ) -> PromptComponents:
262 """Async version of :meth:`_extract_prompt_components`."""
263 - parsed = parse_singleton_prompt(inputs)
263 + parsed = parse_singleton_prompt(prompt)
264
265 if parsed["type"] == "str":
266 - prompt = parsed["content"]
266 + prompt_text = parsed["content"]
267 prompt_token_ids = await self._tokenize_prompt_async(
268 - prompt,
268 + prompt_text,
269 request_id=request_id,
270 lora_request=lora_request,
271 )
272 multi_modal_data = None
273 elif parsed["type"] == "tokens":
274 - prompt = None
274 + prompt_text = None
275 prompt_token_ids = parsed["content"]["prompt_token_ids"]
276 multi_modal_data = parsed["content"].get("multi_modal_data")
277 elif parsed["type"] == "text":
278 - prompt = parsed["content"]["prompt"]
278 + prompt_text = parsed["content"]["prompt"]
279 prompt_token_ids = await self._tokenize_prompt_async(
280 - prompt,
280 + prompt_text,
281 request_id=request_id,
282 lora_request=lora_request,
283 )
@@ -285,7 +285,7 @@ class InputPreprocessor:
285 else:
286 assert_never(parsed)
287
288 - return prompt, prompt_token_ids, multi_modal_data
288 + return prompt_text, prompt_token_ids, multi_modal_data
289
290 def _build_enc_dec_llm_inputs(
291 self,
@@ -311,7 +311,7 @@ class InputPreprocessor:
311
312 def _process_encoder_decoder_prompt(
313 self,
314 - inputs: PromptInputs,
314 + prompt: PromptType,
315 request_id: str,
316 ) -> EncoderDecoderLLMInputs:
317 '''
@@ -339,7 +339,7 @@ class InputPreprocessor:
339
340 Arguments:
341
342 - * inputs: an input prompt
342 + * prompt: an input prompt
343 * request_id
344
345 Returns:
@@ -350,13 +350,13 @@ class InputPreprocessor:
350 encoder_comps: PromptComponents
351 decoder_comps: DecoderPromptComponents
352
353 - if is_explicit_encoder_decoder_prompt(inputs):
353 + if is_explicit_encoder_decoder_prompt(prompt):
354 encoder_comps = self._extract_prompt_components(
355 - inputs["encoder_prompt"],
355 + prompt["encoder_prompt"],
356 request_id=request_id,
357 )
358
359 - if (decoder_input := inputs["decoder_prompt"]) is None:
359 + if (decoder_input := prompt["decoder_prompt"]) is None:
360 decoder_comps = None, None, None
361 else:
362 decoder_comps = self._extract_prompt_components(
@@ -365,7 +365,7 @@ class InputPreprocessor:
365 )
366 else:
367 encoder_comps = self._extract_prompt_components(
368 - inputs,
368 + prompt,
369 request_id=request_id,
370 )
371
@@ -375,20 +375,20 @@ class InputPreprocessor:
375
376 async def _process_encoder_decoder_prompt_async(
377 self,
378 - inputs: PromptInputs,
378 + prompt: PromptType,
379 request_id: str,
380 ) -> EncoderDecoderLLMInputs:
381 """Async version of :meth:`_process_encoder_decoder_prompt`."""
382 encoder_comps: PromptComponents
383 decoder_comps: DecoderPromptComponents
384
385 - if is_explicit_encoder_decoder_prompt(inputs):
385 + if is_explicit_encoder_decoder_prompt(prompt):
386 encoder_task = self._extract_prompt_components_async(
387 - inputs["encoder_prompt"],
387 + prompt["encoder_prompt"],
388 request_id=request_id,
389 )
390
391 - if (decoder_input := inputs["decoder_prompt"]) is None:
391 + if (decoder_input := prompt["decoder_prompt"]) is None:
392 encoder_comps = await encoder_task
393 decoder_comps = None, None, None
394 else:
@@ -401,7 +401,7 @@ class InputPreprocessor:
401 encoder_task, decoder_task)
402 else:
403 encoder_comps = await self._extract_prompt_components_async(
404 - inputs,
404 + prompt,
405 request_id=request_id,
406 )
407
@@ -425,7 +425,7 @@ class InputPreprocessor:
425
426 def _process_decoder_only_prompt(
427 self,
428 - inputs: SingletonPromptInputs,
428 + prompt: SingletonPrompt,
429 request_id: str,
430 lora_request: Optional[LoRARequest] = None,
431 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -436,7 +436,7 @@ class InputPreprocessor:
436
437 Arguments:
438
439 - * inputs: input prompt
439 + * prompt: input prompt
440 * request_id
441 * lora_request
442 * prompt_adapter_request
@@ -447,7 +447,7 @@ class InputPreprocessor:
447 '''
448
449 prompt_comps = self._extract_prompt_components(
450 - inputs,
450 + prompt,
451 request_id=request_id,
452 lora_request=lora_request,
453 )
@@ -459,14 +459,14 @@ class InputPreprocessor:
459
460 async def _process_decoder_only_prompt_async(
461 self,
462 - inputs: SingletonPromptInputs,
462 + prompt: SingletonPrompt,
463 request_id: str,
464 lora_request: Optional[LoRARequest] = None,
465 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466 ) -> LLMInputs:
467 """Async version of :meth:`_process_decoder_only_prompt`."""
468 prompt_comps = await self._extract_prompt_components_async(
469 - inputs,
469 + prompt,
470 request_id=request_id,
471 lora_request=lora_request,
472 )
@@ -478,7 +478,7 @@ class InputPreprocessor:
478
479 def preprocess(
480 self,
481 - inputs: PromptInputs,
481 + prompt: PromptType,
482 request_id: str,
483 lora_request: Optional[LoRARequest] = None,
484 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -488,17 +488,17 @@ class InputPreprocessor:
488 # Encoder-decoder model requires special mapping of
489 # input prompts to encoder & decoder
490 return self._process_encoder_decoder_prompt(
491 - inputs,
491 + prompt,
492 request_id=request_id,
493 )
494
495 - if is_explicit_encoder_decoder_prompt(inputs):
495 + if is_explicit_encoder_decoder_prompt(prompt):
496 raise ValueError("Cannot pass encoder-decoder prompt "
497 "to decoder-only models")
498
499 # Decoder-only operation
500 return self._process_decoder_only_prompt(
501 - inputs,
501 + prompt,
502 request_id=request_id,
503 lora_request=lora_request,
504 prompt_adapter_request=prompt_adapter_request,
@@ -506,7 +506,7 @@ class InputPreprocessor:
506
507 async def preprocess_async(
508 self,
509 - inputs: PromptInputs,
509 + prompt: PromptType,
510 request_id: str,
511 lora_request: Optional[LoRARequest] = None,
512 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
@@ -516,17 +516,17 @@ class InputPreprocessor:
516 # Encoder-decoder model requires special mapping of
517 # input prompts to encoder & decoder
518 return await self._process_encoder_decoder_prompt_async(
519 - inputs,
519 + prompt,
520 request_id=request_id,
521 )
522
523 - if is_explicit_encoder_decoder_prompt(inputs):
523 + if is_explicit_encoder_decoder_prompt(prompt):
524 raise ValueError("Cannot pass encoder-decoder prompt "
525 "to decoder-only models")
526
527 # Decoder-only operation
528 return await self._process_decoder_only_prompt_async(
529 - inputs,
529 + prompt,
530 request_id=request_id,
531 lora_request=lora_request,
532 prompt_adapter_request=prompt_adapter_request,