A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Factor out input preprocessing to a separate class (#7329)
Cyrus Leung committed
Sep 13, 2024 at 10:56 UTC
5ec9c0fb3c667c30117eb1fd743e0e7c13ccf997
5 files changed
+590
-540
tests/engine/test_skip_tokenizer_init.py
+3
-2
@@ -11,9 +11,10 @@ def test_skip_tokenizer_initialization(model: str):
11
# token ids.
12
llm = LLM(model=model, skip_tokenizer_init=True)
13
sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
14
- with pytest.raises(ValueError) as err:
14
+
15
+ with pytest.raises(ValueError, match="cannot pass text prompts when"):
16
llm.generate("abc", sampling_params)
16
- assert "prompts must be None if" in str(err.value)
17
+
18
outputs = llm.generate({"prompt_token_ids": [1, 2, 3]},
19
sampling_params=sampling_params)
20
assert len(outputs) > 0
vllm/engine/async_llm_engine.py
+4
-141
@@ -4,22 +4,17 @@ from functools import partial
4
from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
5
Mapping, Optional, Set, Tuple, Type, Union)
6
7
-from typing_extensions import assert_never
8
-
7
import vllm.envs as envs
8
from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
9
ParallelConfig, SchedulerConfig)
10
from vllm.core.scheduler import SchedulerOutputs
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
from vllm.engine.async_timeout import asyncio_timeout
15
-from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
16
- PromptComponents, SchedulerOutputState)
13
+from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
14
from vllm.engine.metrics_types import StatLoggerBase
15
from vllm.executor.executor_base import ExecutorAsyncBase
16
from vllm.executor.ray_utils import initialize_ray_cluster
20
-from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
21
- SingletonPromptInputs)
22
-from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
17
+from vllm.inputs import PromptInputs
18
from vllm.logger import init_logger
19
from vllm.lora.request import LoRARequest
20
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -403,139 +398,6 @@ class _AsyncLLMEngine(LLMEngine):
398
"""Stop the remote worker execution loop."""
399
await self.model_executor.stop_remote_worker_execution_loop_async()
400
406
- async def _tokenize_prompt_async(
407
- self,
408
- prompt: str,
409
- request_id: str,
410
- lora_request: Optional[LoRARequest],
411
- ) -> List[int]:
412
- """Async version of :meth:`_tokenize_prompt`."""
413
- tokenizer = self.get_tokenizer_group(
414
- missing_msg="prompts must be None if skip_tokenizer_init is True")
415
-
416
- return await tokenizer.encode_async(request_id=request_id,
417
- prompt=prompt,
418
- lora_request=lora_request)
419
-
420
- async def _extract_prompt_components_async(
421
- self,
422
- inputs: SingletonPromptInputs,
423
- request_id: str,
424
- lora_request: Optional[LoRARequest] = None,
425
- ) -> PromptComponents:
426
- """Async version of :meth:`_extract_prompt_components`."""
427
- if isinstance(inputs, str):
428
- prompt = inputs
429
- prompt_token_ids = await self._tokenize_prompt_async(
430
- prompt,
431
- request_id=request_id,
432
- lora_request=lora_request,
433
- )
434
- multi_modal_data = None
435
- elif isinstance(inputs, dict):
436
- if "prompt_token_ids" in inputs:
437
- prompt = None
438
- prompt_token_ids = inputs["prompt_token_ids"]
439
- else:
440
- # NOTE: This extra assignment is required to pass mypy
441
- prompt = parsed_prompt = inputs["prompt"]
442
- prompt_token_ids = await self._tokenize_prompt_async(
443
- parsed_prompt,
444
- request_id=request_id,
445
- lora_request=lora_request,
446
- )
447
-
448
- multi_modal_data = inputs.get("multi_modal_data")
449
- else:
450
- assert_never(inputs)
451
-
452
- return prompt, prompt_token_ids, multi_modal_data
453
-
454
- async def _process_encoder_decoder_prompt_async(
455
- self,
456
- inputs: PromptInputs,
457
- request_id: str,
458
- ) -> EncoderDecoderLLMInputs:
459
- """Async version of :meth:`_process_encoder_decoder_prompt`."""
460
- encoder_comps: PromptComponents
461
- decoder_comps: DecoderPromptComponents
462
-
463
- if is_explicit_encoder_decoder_prompt(inputs):
464
- encoder_task = self._extract_prompt_components_async(
465
- inputs["encoder_prompt"],
466
- request_id=request_id,
467
- )
468
-
469
- if (decoder_input := inputs["decoder_prompt"]) is None:
470
- encoder_comps = await encoder_task
471
- decoder_comps = None, None, None
472
- else:
473
- decoder_task = self._extract_prompt_components_async(
474
- decoder_input,
475
- request_id=request_id,
476
- )
477
-
478
- encoder_comps, decoder_comps = await asyncio.gather(
479
- encoder_task, decoder_task)
480
- else:
481
- encoder_comps = await self._extract_prompt_components_async(
482
- inputs,
483
- request_id=request_id,
484
- )
485
-
486
- decoder_comps = None, None, None
487
-
488
- return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
489
-
490
- async def _process_decoder_only_prompt_async(
491
- self,
492
- inputs: SingletonPromptInputs,
493
- request_id: str,
494
- lora_request: Optional[LoRARequest] = None,
495
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
496
- ) -> LLMInputs:
497
- """Async version of :meth:`_process_decoder_only_prompt`."""
498
- prompt_comps = await self._extract_prompt_components_async(
499
- inputs,
500
- request_id=request_id,
501
- lora_request=lora_request,
502
- )
503
-
504
- return self._build_decoder_only_llm_inputs(
505
- prompt_comps,
506
- prompt_adapter_request=prompt_adapter_request,
507
- )
508
-
509
- async def process_model_inputs_async(
510
- self,
511
- inputs: PromptInputs,
512
- request_id: str,
513
- lora_request: Optional[LoRARequest] = None,
514
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
515
- ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
516
- """Async version of :meth:`process_model_inputs`."""
517
- if self.is_encoder_decoder_model():
518
- # Encoder-decoder model requires special mapping of
519
- # input prompts to encoder & decoder
520
- model_inputs = await self._process_encoder_decoder_prompt_async(
521
- inputs,
522
- request_id=request_id,
523
- )
524
- else:
525
- if is_explicit_encoder_decoder_prompt(inputs):
526
- raise ValueError("Cannot pass encoder-decoder prompt "
527
- "to decoder-only models")
528
-
529
- # Decoder-only operation
530
- model_inputs = await self._process_decoder_only_prompt_async(
531
- inputs,
532
- request_id=request_id,
533
- lora_request=lora_request,
534
- prompt_adapter_request=prompt_adapter_request,
535
- )
536
-
537
- return self.input_processor(model_inputs)
538
-
401
async def add_request_async(
402
self,
403
request_id: str,
@@ -553,12 +415,13 @@ class _AsyncLLMEngine(LLMEngine):
415
if arrival_time is None:
416
arrival_time = time.time()
417
556
- processed_inputs = await self.process_model_inputs_async(
418
+ preprocessed_inputs = await self.input_preprocessor.preprocess_async(
419
inputs,
420
request_id=request_id,
421
lora_request=lora_request,
422
prompt_adapter_request=prompt_adapter_request,
423
)
424
+ processed_inputs = self.input_processor(preprocessed_inputs)
425
426
self._add_processed_request(
427
request_id=request_id,
vllm/engine/llm_engine.py
+13
-394
@@ -6,10 +6,10 @@ from dataclasses import dataclass
6
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7
Iterable, List, Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
9
-from typing import Set, Tuple, Type, Union
9
+from typing import Set, Type, Union
10
11
import torch
12
-from typing_extensions import TypeVar, assert_never
12
+from typing_extensions import TypeVar
13
14
import vllm.envs as envs
15
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -28,13 +28,11 @@ from vllm.engine.output_processor.util import create_output_by_sequence_group
28
from vllm.executor.executor_base import ExecutorBase
29
from vllm.executor.ray_utils import initialize_ray_cluster
30
from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
31
- InputRegistry, LLMInputs, PromptInputs,
32
- SingletonPromptInputs)
33
-from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
31
+ InputRegistry, LLMInputs, PromptInputs)
32
+from vllm.inputs.preprocess import InputPreprocessor
33
from vllm.logger import init_logger
34
from vllm.lora.request import LoRARequest
35
from vllm.model_executor.layers.sampler import SamplerOutput
37
-from vllm.multimodal import MultiModalDataDict
36
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
37
RequestOutputFactory)
38
from vllm.pooling_params import PoolingParams
@@ -75,11 +73,6 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
73
_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
74
_O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
75
78
-PromptComponents = Tuple[Optional[str], List[int],
79
- Optional[MultiModalDataDict]]
80
-DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
81
- Optional[MultiModalDataDict]]
82
-
76
77
@dataclass
78
class SchedulerOutputState:
@@ -313,6 +306,9 @@ class LLMEngine:
306
self.generation_config_fields = _load_generation_config_dict(
307
model_config)
308
309
+ self.input_preprocessor = InputPreprocessor(model_config,
310
+ self.tokenizer)
311
+
312
self.input_registry = input_registry
313
self.input_processor = input_registry.create_input_processor(
314
model_config)
@@ -571,19 +567,15 @@ class LLMEngine:
567
if model_executor := getattr(self, "model_executor", None):
568
model_executor.shutdown()
569
574
- MISSING_TOKENIZER_GROUP_MSG = ("Unable to get tokenizer because "
575
- "skip_tokenizer_init is True")
576
-
570
def get_tokenizer_group(
571
self,
572
group_type: Type[_G] = BaseTokenizerGroup,
580
- *,
581
- missing_msg: str = MISSING_TOKENIZER_GROUP_MSG,
573
) -> _G:
574
tokenizer_group = self.tokenizer
575
576
if tokenizer_group is None:
586
- raise ValueError(missing_msg)
577
+ raise ValueError("Unable to get tokenizer because "
578
+ "skip_tokenizer_init is True")
579
if not isinstance(tokenizer_group, group_type):
580
raise TypeError("Invalid type of tokenizer group. "
581
f"Expected type: {group_type}, but "
@@ -615,52 +607,6 @@ class LLMEngine:
607
self.prompt_adapter_config.verify_with_model_config(
608
self.model_config)
609
618
- def _get_bos_token_id(self,
619
- lora_request: Optional[LoRARequest] = None
620
- ) -> Optional[int]:
621
- if self.tokenizer is None:
622
- logger.warning("Using None for BOS token id because tokenizer "
623
- "is not initialized")
624
- return None
625
-
626
- return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id
627
-
628
- def _get_eos_token_id(self,
629
- lora_request: Optional[LoRARequest] = None
630
- ) -> Optional[int]:
631
- if self.tokenizer is None:
632
- logger.warning("Using None for EOS token id because tokenizer "
633
- "is not initialized")
634
- return None
635
-
636
- return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
637
-
638
- def _get_decoder_start_token_id(self) -> Optional[int]:
639
- '''
640
- Obtain the decoder start token id employed by an encoder/decoder
641
- model. Returns None for non-encoder/decoder models or if the
642
- model config is unavailable.
643
- '''
644
-
645
- if not self.is_encoder_decoder_model():
646
- logger.warning("Using None for decoder start token id because "
647
- "this is not an encoder/decoder model.")
648
- return None
649
-
650
- if (self.model_config is None or self.model_config.hf_config is None):
651
- logger.warning("Using None for decoder start token id because "
652
- "model config is not available.")
653
- return None
654
-
655
- dec_start_token_id = getattr(self.model_config.hf_config,
656
- 'decoder_start_token_id', None)
657
- if dec_start_token_id is None:
658
- logger.warning("Falling back on <BOS> for decoder start token id "
659
- "because decoder start token id is not available.")
660
- dec_start_token_id = self._get_bos_token_id()
661
-
662
- return dec_start_token_id
663
-
610
def _add_processed_request(
611
self,
612
request_id: str,
@@ -675,7 +621,7 @@ class LLMEngine:
621
# Create the sequences.
622
block_size = self.cache_config.block_size
623
seq_id = next(self.seq_counter)
678
- eos_token_id = self._get_eos_token_id(lora_request)
624
+ eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
625
626
seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
627
lora_request, prompt_adapter_request)
@@ -725,334 +671,6 @@ class LLMEngine:
671
def stop_remote_worker_execution_loop(self) -> None:
672
self.model_executor.stop_remote_worker_execution_loop()
673
728
- _LLMInputComponentsType = Tuple[str, List[int]]
729
-
730
- def _prepare_decoder_input_ids_for_generation(
731
- self,
732
- decoder_input_ids: Optional[List[int]],
733
- ) -> List[int]:
734
- """
735
- Prepares `decoder_input_ids` for generation with encoder-decoder models.
736
-
737
- Based on
738
-
739
- https://github.com/huggingface/transformers/blob/
740
- 4037a2b5b1278736e566aec12e169100275545ea/
741
- src/transformers/generation/utils.py
742
-
743
- specifically GenerationMixin._prepare_decoder_input_ids_for_generation()
744
-
745
- Arguments:
746
-
747
- * decoder_input_ids: input token ids to preprocess
748
-
749
- Returns:
750
-
751
- * Processed token list
752
- """
753
-
754
- decoder_start_token_id = self._get_decoder_start_token_id()
755
- assert decoder_start_token_id is not None
756
-
757
- if decoder_input_ids is None:
758
- # no decoder prompt input ->
759
- # use decoder_start_token_id as decoder_input_ids
760
- decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
761
-
762
- if (len(decoder_input_ids) == 0
763
- or decoder_input_ids[0] != decoder_start_token_id):
764
- decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
765
-
766
- return decoder_input_ids
767
-
768
- def _tokenize_prompt(
769
- self,
770
- prompt: str,
771
- request_id: str,
772
- lora_request: Optional[LoRARequest],
773
- ) -> List[int]:
774
- '''
775
- Wrapper around application of the model's tokenizer.
776
-
777
- Arguments:
778
-
779
- * prompt
780
- * request_id
781
- * lora_request
782
-
783
- Returns:
784
-
785
- * prompt token ids
786
- '''
787
-
788
- tokenizer = self.get_tokenizer_group(
789
- missing_msg="prompts must be None if skip_tokenizer_init is True")
790
-
791
- return tokenizer.encode(request_id=request_id,
792
- prompt=prompt,
793
- lora_request=lora_request)
794
-
795
- def _extract_prompt_components(
796
- self,
797
- inputs: SingletonPromptInputs,
798
- request_id: str,
799
- lora_request: Optional[LoRARequest] = None,
800
- ) -> PromptComponents:
801
- '''
802
- Extract the components of any single encoder or decoder input prompt.
803
-
804
- Arguments:
805
-
806
- * request_id
807
- * inputs: single encoder or decoder input prompt
808
- * lora_request: this is only valid for decoder prompts
809
-
810
- Returns:
811
-
812
- * prompt
813
- * prompt_token_ids
814
- * multi_modal_data
815
- '''
816
-
817
- if isinstance(inputs, str):
818
- prompt = inputs
819
- prompt_token_ids = self._tokenize_prompt(
820
- prompt,
821
- request_id=request_id,
822
- lora_request=lora_request,
823
- )
824
- multi_modal_data = None
825
- elif isinstance(inputs, dict):
826
- if "prompt_token_ids" in inputs:
827
- prompt = None
828
- prompt_token_ids = inputs["prompt_token_ids"]
829
- else:
830
- # NOTE: This extra assignment is required to pass mypy
831
- prompt = parsed_prompt = inputs["prompt"]
832
- prompt_token_ids = self._tokenize_prompt(
833
- parsed_prompt,
834
- request_id=request_id,
835
- lora_request=lora_request,
836
- )
837
-
838
- multi_modal_data = inputs.get("multi_modal_data")
839
- else:
840
- assert_never(inputs)
841
-
842
- return prompt, prompt_token_ids, multi_modal_data
843
-
844
- def _apply_prompt_adapter(
845
- self,
846
- prompt_token_ids: List[int],
847
- prompt_adapter_request: Optional[PromptAdapterRequest],
848
- ) -> List[int]:
849
- if prompt_adapter_request:
850
- prompt_token_ids = (
851
- [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
852
- + prompt_token_ids)
853
-
854
- return prompt_token_ids
855
-
856
- def _get_default_enc_dec_decoder_prompt(self) -> List[int]:
857
- '''
858
- Specifically for encoder/decoder models:
859
- generate a default decoder prompt for when
860
- the user specifies only the encoder prompt.
861
-
862
- Encoder/decoder models utilize the decoder
863
- prompt in different ways; as new models are
864
- added, it is intended that this function
865
- will be extended to produce differing
866
- default decoder prompts, depending on the
867
- model variety.
868
-
869
- Absent a special case, the default behavior
870
- of this method is to mirror the behavior of
871
- the HuggingFace (HF) GenerationMixin for a None
872
- decoder prompt, which is to employ a logit processor
873
- setting to force the first decoded token to be <BOS>.
874
- Here, this behavior is approximated by having the
875
- "default" decoder prompt be <BOS>.
876
-
877
- However, it is possible that in the future
878
- other models may have different or more
879
- complex logic for the default decoder prompt.
880
- This motivates having a special helper method
881
- for default decoder prompts.
882
-
883
- Returns:
884
-
885
- * prompt_token_ids
886
- '''
887
-
888
- bos_token_id = self._get_bos_token_id()
889
- assert bos_token_id is not None
890
- return [bos_token_id]
891
-
892
- def _build_enc_dec_llm_inputs(
893
- self,
894
- encoder_comps: PromptComponents,
895
- decoder_comps: DecoderPromptComponents,
896
- ) -> EncoderDecoderLLMInputs:
897
- encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
898
- decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
899
-
900
- if encoder_mm_data is not None or decoder_mm_data is not None:
901
- raise ValueError("Multi-modal encoder-decoder models are "
902
- "not supported yet")
903
-
904
- decoder_prompt_ids = (
905
- self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
906
-
907
- return EncoderDecoderLLMInputs(
908
- prompt_token_ids=decoder_prompt_ids,
909
- prompt=decoder_prompt,
910
- encoder_prompt_token_ids=encoder_prompt_ids,
911
- encoder_prompt=encoder_prompt,
912
- )
913
-
914
- def _process_encoder_decoder_prompt(
915
- self,
916
- inputs: PromptInputs,
917
- request_id: str,
918
- ) -> EncoderDecoderLLMInputs:
919
- '''
920
- For encoder/decoder models only:
921
- Process an input prompt into an
922
- :class:`EncoderDecoderLLMInputs` instance.
923
-
924
- There are two types of input prompts:
925
- singleton prompts which carry only the
926
- encoder prompt, and explicit encoder/decoder
927
- prompts which carry both the encoder and the
928
- decoder prompts as member variables.
929
-
930
- This function handles the following scenarios:
931
- * Singleton encoder prompt: extract encoder prompt
932
- token ids & infer default decoder prompt token ids
933
- * Explicit encoder/decoder prompt: extract encoder
934
- and decoder prompt token ids
935
-
936
- Note that for Explicit encoder/decoder prompts,
937
- each sub-prompt (encoder or decoder prompt) can
938
- have any possible singleton type; thus this
939
- method relies on helper functions to obtain
940
- token ids for the sub-prompts.
941
-
942
- Arguments:
943
-
944
- * inputs: an input prompt
945
- * request_id
946
-
947
- Returns:
948
-
949
- * :class:`EncoderDecoderLLMInputs` instance
950
- '''
951
-
952
- encoder_comps: PromptComponents
953
- decoder_comps: DecoderPromptComponents
954
-
955
- if is_explicit_encoder_decoder_prompt(inputs):
956
- encoder_comps = self._extract_prompt_components(
957
- inputs["encoder_prompt"],
958
- request_id=request_id,
959
- )
960
-
961
- if (decoder_input := inputs["decoder_prompt"]) is None:
962
- decoder_comps = None, None, None
963
- else:
964
- decoder_comps = self._extract_prompt_components(
965
- decoder_input,
966
- request_id=request_id,
967
- )
968
- else:
969
- encoder_comps = self._extract_prompt_components(
970
- inputs,
971
- request_id=request_id,
972
- )
973
-
974
- decoder_comps = None, None, None
975
-
976
- return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
977
-
978
- def _build_decoder_only_llm_inputs(
979
- self,
980
- prompt_comps: PromptComponents,
981
- prompt_adapter_request: Optional[PromptAdapterRequest],
982
- ) -> LLMInputs:
983
- prompt, prompt_token_ids, multi_modal_data = prompt_comps
984
-
985
- prompt_token_ids = self._apply_prompt_adapter(
986
- prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
987
-
988
- return LLMInputs(prompt_token_ids=prompt_token_ids,
989
- prompt=prompt,
990
- multi_modal_data=multi_modal_data)
991
-
992
- def _process_decoder_only_prompt(
993
- self,
994
- inputs: SingletonPromptInputs,
995
- request_id: str,
996
- lora_request: Optional[LoRARequest] = None,
997
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
998
- ) -> LLMInputs:
999
- '''
1000
- For decoder-only models:
1001
- Process an input prompt into an :class:`LLMInputs` instance.
1002
-
1003
- Arguments:
1004
-
1005
- * inputs: input prompt
1006
- * request_id
1007
- * lora_request
1008
- * prompt_adapter_request
1009
-
1010
- Returns:
1011
-
1012
- * :class:`LLMInputs` instance
1013
- '''
1014
-
1015
- prompt_comps = self._extract_prompt_components(
1016
- inputs,
1017
- request_id=request_id,
1018
- lora_request=lora_request,
1019
- )
1020
-
1021
- return self._build_decoder_only_llm_inputs(
1022
- prompt_comps,
1023
- prompt_adapter_request=prompt_adapter_request,
1024
- )
1025
-
1026
- def process_model_inputs(
1027
- self,
1028
- inputs: PromptInputs,
1029
- request_id: str,
1030
- lora_request: Optional[LoRARequest] = None,
1031
- prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1032
- ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
1033
-
1034
- if self.is_encoder_decoder_model():
1035
- # Encoder-decoder model requires special mapping of
1036
- # input prompts to encoder & decoder
1037
- model_inputs = self._process_encoder_decoder_prompt(
1038
- inputs,
1039
- request_id=request_id,
1040
- )
1041
- else:
1042
- if is_explicit_encoder_decoder_prompt(inputs):
1043
- raise ValueError("Cannot pass encoder-decoder prompt "
1044
- "to decoder-only models")
1045
-
1046
- # Decoder-only operation
1047
- model_inputs = self._process_decoder_only_prompt(
1048
- inputs,
1049
- request_id=request_id,
1050
- lora_request=lora_request,
1051
- prompt_adapter_request=prompt_adapter_request,
1052
- )
1053
-
1054
- return self.input_processor(model_inputs)
1055
-
674
def add_request(
675
self,
676
request_id: str,
@@ -1111,12 +729,13 @@ class LLMEngine:
729
if arrival_time is None:
730
arrival_time = time.time()
731
1114
- processed_inputs = self.process_model_inputs(
732
+ preprocessed_inputs = self.input_preprocessor.preprocess(
733
inputs,
734
request_id=request_id,
735
lora_request=lora_request,
736
prompt_adapter_request=prompt_adapter_request,
737
)
738
+ processed_inputs = self.input_processor(preprocessed_inputs)
739
740
self._add_processed_request(
741
request_id=request_id,
@@ -2043,7 +1662,7 @@ class LLMEngine:
1662
metrics.model_execute_time)
1663
1664
def is_encoder_decoder_model(self):
2046
- return self.model_config.is_encoder_decoder_model
1665
+ return self.input_preprocessor.is_encoder_decoder_model()
1666
1667
def is_embedding_model(self):
1668
return self.model_config.is_embedding_model
vllm/inputs/parse.py
+34
-3
@@ -5,7 +5,8 @@ from typing_extensions import TypeIs
5
from vllm.utils import is_list_of
6
7
from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8
- LLMInputs, PromptInputs)
8
+ LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
9
+ TokensPrompt)
10
11
12
class ParsedText(TypedDict):
@@ -60,8 +61,38 @@ def parse_and_batch_prompt(
61
for elem in prompt
62
]
63
63
- raise ValueError("prompt must be a string, array of strings, "
64
- "array of tokens, or array of token arrays")
64
+ raise TypeError("prompt must be a string, array of strings, "
65
+ "array of tokens, or array of token arrays")
66
+
67
+
68
+class ParsedStrPrompt(TypedDict):
69
+ type: Literal["str"]
70
+ content: str
71
+
72
+
73
+class ParsedTextPrompt(TypedDict):
74
+ type: Literal["text"]
75
+ content: TextPrompt
76
+
77
+
78
+class ParsedTokensPrompt(TypedDict):
79
+ type: Literal["tokens"]
80
+ content: TokensPrompt
81
+
82
+
83
+def parse_singleton_prompt(
84
+ inputs: SingletonPromptInputs,
85
+) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86
+ if isinstance(inputs, str):
87
+ return ParsedStrPrompt(type="str", content=inputs)
88
+ elif isinstance(inputs, dict):
89
+ if "prompt_token_ids" in inputs:
90
+ return ParsedTokensPrompt(type="tokens",
91
+ content=inputs) # type: ignore
92
+ elif "prompt" in inputs:
93
+ return ParsedTextPrompt(type="text", content=inputs)
94
+
95
+ raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98
def is_explicit_encoder_decoder_prompt(
vllm/inputs/preprocess.py
new
+536
@@ -0,0 +1,536 @@
1
+import asyncio
2
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
3
+
4
+from typing_extensions import assert_never
5
+
6
+from vllm.config import ModelConfig
7
+from vllm.logger import init_logger
8
+from vllm.lora.request import LoRARequest
9
+from vllm.prompt_adapter.request import PromptAdapterRequest
10
+from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11
+
12
+from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13
+ SingletonPromptInputs)
14
+from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15
+
16
+if TYPE_CHECKING:
17
+ from vllm.multimodal import MultiModalDataDict
18
+
19
+logger = init_logger(__name__)
20
+
21
+PromptComponents = Tuple[Optional[str], List[int],
22
+ Optional["MultiModalDataDict"]]
23
+DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
24
+ Optional["MultiModalDataDict"]]
25
+
26
+
27
+class InputPreprocessor:
28
+
29
+ def __init__(
30
+ self,
31
+ model_config: ModelConfig,
32
+ tokenizer: Optional[BaseTokenizerGroup],
33
+ ) -> None:
34
+ super().__init__()
35
+
36
+ self.model_config = model_config
37
+ self.tokenizer = tokenizer
38
+
39
+ def get_tokenizer_group(self) -> BaseTokenizerGroup:
40
+ if self.tokenizer is None:
41
+ raise ValueError("You cannot pass text prompts when "
42
+ "`skip_tokenizer_init` is True")
43
+
44
+ return self.tokenizer
45
+
46
+ def get_bos_token_id(self,
47
+ lora_request: Optional[LoRARequest] = None
48
+ ) -> Optional[int]:
49
+ if self.tokenizer is None:
50
+ logger.warning("Using None for BOS token id because tokenizer "
51
+ "is not initialized")
52
+ return None
53
+
54
+ return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id
55
+
56
+ def get_eos_token_id(self,
57
+ lora_request: Optional[LoRARequest] = None
58
+ ) -> Optional[int]:
59
+ if self.tokenizer is None:
60
+ logger.warning("Using None for EOS token id because tokenizer "
61
+ "is not initialized")
62
+ return None
63
+
64
+ return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
65
+
66
+ def get_decoder_start_token_id(self) -> Optional[int]:
67
+ '''
68
+ Obtain the decoder start token id employed by an encoder/decoder
69
+ model. Returns None for non-encoder/decoder models or if the
70
+ model config is unavailable.
71
+ '''
72
+
73
+ if not self.is_encoder_decoder_model():
74
+ logger.warning("Using None for decoder start token id because "
75
+ "this is not an encoder/decoder model.")
76
+ return None
77
+
78
+ if (self.model_config is None or self.model_config.hf_config is None):
79
+ logger.warning("Using None for decoder start token id because "
80
+ "model config is not available.")
81
+ return None
82
+
83
+ dec_start_token_id = getattr(self.model_config.hf_config,
84
+ 'decoder_start_token_id', None)
85
+ if dec_start_token_id is None:
86
+ logger.warning("Falling back on <BOS> for decoder start token id "
87
+ "because decoder start token id is not available.")
88
+ dec_start_token_id = self.get_bos_token_id()
89
+
90
+ return dec_start_token_id
91
+
92
+ def _get_default_enc_dec_decoder_prompt(self) -> List[int]:
93
+ '''
94
+ Specifically for encoder/decoder models:
95
+ generate a default decoder prompt for when
96
+ the user specifies only the encoder prompt.
97
+
98
+ Encoder/decoder models utilize the decoder
99
+ prompt in different ways; as new models are
100
+ added, it is intended that this function
101
+ will be extended to produce differing
102
+ default decoder prompts, depending on the
103
+ model variety.
104
+
105
+ Absent a special case, the default behavior
106
+ of this method is to mirror the behavior of
107
+ the HuggingFace (HF) GenerationMixin for a None
108
+ decoder prompt, which is to employ a logit processor
109
+ setting to force the first decoded token to be <BOS>.
110
+ Here, this behavior is approximated by having the
111
+ "default" decoder prompt be <BOS>.
112
+
113
+ However, it is possible that in the future
114
+ other models may have different or more
115
+ complex logic for the default decoder prompt.
116
+ This motivates having a special helper method
117
+ for default decoder prompts.
118
+
119
+ Returns:
120
+
121
+ * prompt_token_ids
122
+ '''
123
+
124
+ bos_token_id = self.get_bos_token_id()
125
+ assert bos_token_id is not None
126
+ return [bos_token_id]
127
+
128
+ def _prepare_decoder_input_ids_for_generation(
129
+ self,
130
+ decoder_input_ids: Optional[List[int]],
131
+ ) -> List[int]:
132
+ """
133
+ Prepares `decoder_input_ids` for generation with encoder-decoder models.
134
+
135
+ Based on
136
+
137
+ https://github.com/huggingface/transformers/blob/
138
+ 4037a2b5b1278736e566aec12e169100275545ea/
139
+ src/transformers/generation/utils.py
140
+
141
+ specifically GenerationMixin._prepare_decoder_input_ids_for_generation()
142
+
143
+ Arguments:
144
+
145
+ * decoder_input_ids: input token ids to preprocess
146
+
147
+ Returns:
148
+
149
+ * Processed token list
150
+ """
151
+
152
+ decoder_start_token_id = self.get_decoder_start_token_id()
153
+ assert decoder_start_token_id is not None
154
+
155
+ if decoder_input_ids is None:
156
+ # no decoder prompt input ->
157
+ # use decoder_start_token_id as decoder_input_ids
158
+ decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
159
+
160
+ if (len(decoder_input_ids) == 0
161
+ or decoder_input_ids[0] != decoder_start_token_id):
162
+ decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
163
+
164
+ return decoder_input_ids
165
+
166
+ def _apply_prompt_adapter(
167
+ self,
168
+ prompt_token_ids: List[int],
169
+ prompt_adapter_request: Optional[PromptAdapterRequest],
170
+ ) -> List[int]:
171
+ if prompt_adapter_request:
172
+ prompt_token_ids = (
173
+ [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
174
+ + prompt_token_ids)
175
+
176
+ return prompt_token_ids
177
+
178
+ def _tokenize_prompt(
179
+ self,
180
+ prompt: str,
181
+ request_id: str,
182
+ lora_request: Optional[LoRARequest],
183
+ ) -> List[int]:
184
+ """
185
+ Apply the model's tokenizer to a text prompt, returning the
186
+ corresponding token IDs.
187
+ """
188
+ tokenizer = self.get_tokenizer_group()
189
+
190
+ return tokenizer.encode(request_id=request_id,
191
+ prompt=prompt,
192
+ lora_request=lora_request)
193
+
194
+ async def _tokenize_prompt_async(
195
+ self,
196
+ prompt: str,
197
+ request_id: str,
198
+ lora_request: Optional[LoRARequest],
199
+ ) -> List[int]:
200
+ """Async version of :meth:`_tokenize_prompt`."""
201
+ tokenizer = self.get_tokenizer_group()
202
+
203
+ return await tokenizer.encode_async(request_id=request_id,
204
+ prompt=prompt,
205
+ lora_request=lora_request)
206
+
207
+ def _extract_prompt_components(
208
+ self,
209
+ inputs: SingletonPromptInputs,
210
+ request_id: str,
211
+ lora_request: Optional[LoRARequest] = None,
212
+ ) -> PromptComponents:
213
+ '''
214
+ Extract the components of any single encoder or decoder input prompt.
215
+
216
+ Arguments:
217
+
218
+ * request_id
219
+ * inputs: single encoder or decoder input prompt
220
+ * lora_request: this is only valid for decoder prompts
221
+
222
+ Returns:
223
+
224
+ * prompt
225
+ * prompt_token_ids
226
+ * multi_modal_data
227
+ '''
228
+
229
+ parsed = parse_singleton_prompt(inputs)
230
+
231
+ if parsed["type"] == "str":
232
+ prompt = parsed["content"]
233
+ prompt_token_ids = self._tokenize_prompt(
234
+ prompt,
235
+ request_id=request_id,
236
+ lora_request=lora_request,
237
+ )
238
+ multi_modal_data = None
239
+ elif parsed["type"] == "tokens":
240
+ prompt = None
241
+ prompt_token_ids = parsed["content"]["prompt_token_ids"]
242
+ multi_modal_data = parsed["content"].get("multi_modal_data")
243
+ elif parsed["type"] == "text":
244
+ prompt = parsed["content"]["prompt"]
245
+ prompt_token_ids = self._tokenize_prompt(
246
+ prompt,
247
+ request_id=request_id,
248
+ lora_request=lora_request,
249
+ )
250
+ multi_modal_data = parsed["content"].get("multi_modal_data")
251
+ else:
252
+ assert_never(parsed)
253
+
254
+ return prompt, prompt_token_ids, multi_modal_data
255
+
256
+ async def _extract_prompt_components_async(
257
+ self,
258
+ inputs: SingletonPromptInputs,
259
+ request_id: str,
260
+ lora_request: Optional[LoRARequest] = None,
261
+ ) -> PromptComponents:
262
+ """Async version of :meth:`_extract_prompt_components`."""
263
+ parsed = parse_singleton_prompt(inputs)
264
+
265
+ if parsed["type"] == "str":
266
+ prompt = parsed["content"]
267
+ prompt_token_ids = await self._tokenize_prompt_async(
268
+ prompt,
269
+ request_id=request_id,
270
+ lora_request=lora_request,
271
+ )
272
+ multi_modal_data = None
273
+ elif parsed["type"] == "tokens":
274
+ prompt = None
275
+ prompt_token_ids = parsed["content"]["prompt_token_ids"]
276
+ multi_modal_data = parsed["content"].get("multi_modal_data")
277
+ elif parsed["type"] == "text":
278
+ prompt = parsed["content"]["prompt"]
279
+ prompt_token_ids = await self._tokenize_prompt_async(
280
+ prompt,
281
+ request_id=request_id,
282
+ lora_request=lora_request,
283
+ )
284
+ multi_modal_data = parsed["content"].get("multi_modal_data")
285
+ else:
286
+ assert_never(parsed)
287
+
288
+ return prompt, prompt_token_ids, multi_modal_data
289
+
290
+ def _build_enc_dec_llm_inputs(
291
+ self,
292
+ encoder_comps: PromptComponents,
293
+ decoder_comps: DecoderPromptComponents,
294
+ ) -> EncoderDecoderLLMInputs:
295
+ encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
296
+ decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
297
+
298
+ if encoder_mm_data is not None or decoder_mm_data is not None:
299
+ raise ValueError("Multi-modal encoder-decoder models are "
300
+ "not supported yet")
301
+
302
+ decoder_prompt_ids = (
303
+ self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
304
+
305
+ return EncoderDecoderLLMInputs(
306
+ prompt_token_ids=decoder_prompt_ids,
307
+ prompt=decoder_prompt,
308
+ encoder_prompt_token_ids=encoder_prompt_ids,
309
+ encoder_prompt=encoder_prompt,
310
+ )
311
+
312
+ def _process_encoder_decoder_prompt(
313
+ self,
314
+ inputs: PromptInputs,
315
+ request_id: str,
316
+ ) -> EncoderDecoderLLMInputs:
317
+ '''
318
+ For encoder/decoder models only:
319
+ Process an input prompt into an
320
+ :class:`EncoderDecoderLLMInputs` instance.
321
+
322
+ There are two types of input prompts:
323
+ singleton prompts which carry only the
324
+ encoder prompt, and explicit encoder/decoder
325
+ prompts which carry both the encoder and the
326
+ decoder prompts as member variables.
327
+
328
+ This function handles the following scenarios:
329
+ * Singleton encoder prompt: extract encoder prompt
330
+ token ids & infer default decoder prompt token ids
331
+ * Explicit encoder/decoder prompt: extract encoder
332
+ and decoder prompt token ids
333
+
334
+ Note that for Explicit encoder/decoder prompts,
335
+ each sub-prompt (encoder or decoder prompt) can
336
+ have any possible singleton type; thus this
337
+ method relies on helper functions to obtain
338
+ token ids for the sub-prompts.
339
+
340
+ Arguments:
341
+
342
+ * inputs: an input prompt
343
+ * request_id
344
+
345
+ Returns:
346
+
347
+ * :class:`EncoderDecoderLLMInputs` instance
348
+ '''
349
+
350
+ encoder_comps: PromptComponents
351
+ decoder_comps: DecoderPromptComponents
352
+
353
+ if is_explicit_encoder_decoder_prompt(inputs):
354
+ encoder_comps = self._extract_prompt_components(
355
+ inputs["encoder_prompt"],
356
+ request_id=request_id,
357
+ )
358
+
359
+ if (decoder_input := inputs["decoder_prompt"]) is None:
360
+ decoder_comps = None, None, None
361
+ else:
362
+ decoder_comps = self._extract_prompt_components(
363
+ decoder_input,
364
+ request_id=request_id,
365
+ )
366
+ else:
367
+ encoder_comps = self._extract_prompt_components(
368
+ inputs,
369
+ request_id=request_id,
370
+ )
371
+
372
+ decoder_comps = None, None, None
373
+
374
+ return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
375
+
376
+ async def _process_encoder_decoder_prompt_async(
377
+ self,
378
+ inputs: PromptInputs,
379
+ request_id: str,
380
+ ) -> EncoderDecoderLLMInputs:
381
+ """Async version of :meth:`_process_encoder_decoder_prompt`."""
382
+ encoder_comps: PromptComponents
383
+ decoder_comps: DecoderPromptComponents
384
+
385
+ if is_explicit_encoder_decoder_prompt(inputs):
386
+ encoder_task = self._extract_prompt_components_async(
387
+ inputs["encoder_prompt"],
388
+ request_id=request_id,
389
+ )
390
+
391
+ if (decoder_input := inputs["decoder_prompt"]) is None:
392
+ encoder_comps = await encoder_task
393
+ decoder_comps = None, None, None
394
+ else:
395
+ decoder_task = self._extract_prompt_components_async(
396
+ decoder_input,
397
+ request_id=request_id,
398
+ )
399
+
400
+ encoder_comps, decoder_comps = await asyncio.gather(
401
+ encoder_task, decoder_task)
402
+ else:
403
+ encoder_comps = await self._extract_prompt_components_async(
404
+ inputs,
405
+ request_id=request_id,
406
+ )
407
+
408
+ decoder_comps = None, None, None
409
+
410
+ return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
411
+
412
+ def _build_decoder_only_llm_inputs(
413
+ self,
414
+ prompt_comps: PromptComponents,
415
+ prompt_adapter_request: Optional[PromptAdapterRequest],
416
+ ) -> LLMInputs:
417
+ prompt, prompt_token_ids, multi_modal_data = prompt_comps
418
+
419
+ prompt_token_ids = self._apply_prompt_adapter(
420
+ prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
421
+
422
+ return LLMInputs(prompt_token_ids=prompt_token_ids,
423
+ prompt=prompt,
424
+ multi_modal_data=multi_modal_data)
425
+
426
+ def _process_decoder_only_prompt(
427
+ self,
428
+ inputs: SingletonPromptInputs,
429
+ request_id: str,
430
+ lora_request: Optional[LoRARequest] = None,
431
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
432
+ ) -> LLMInputs:
433
+ '''
434
+ For decoder-only models:
435
+ Process an input prompt into an :class:`LLMInputs` instance.
436
+
437
+ Arguments:
438
+
439
+ * inputs: input prompt
440
+ * request_id
441
+ * lora_request
442
+ * prompt_adapter_request
443
+
444
+ Returns:
445
+
446
+ * :class:`LLMInputs` instance
447
+ '''
448
+
449
+ prompt_comps = self._extract_prompt_components(
450
+ inputs,
451
+ request_id=request_id,
452
+ lora_request=lora_request,
453
+ )
454
+
455
+ return self._build_decoder_only_llm_inputs(
456
+ prompt_comps,
457
+ prompt_adapter_request=prompt_adapter_request,
458
+ )
459
+
460
+ async def _process_decoder_only_prompt_async(
461
+ self,
462
+ inputs: SingletonPromptInputs,
463
+ request_id: str,
464
+ lora_request: Optional[LoRARequest] = None,
465
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466
+ ) -> LLMInputs:
467
+ """Async version of :meth:`_process_decoder_only_prompt`."""
468
+ prompt_comps = await self._extract_prompt_components_async(
469
+ inputs,
470
+ request_id=request_id,
471
+ lora_request=lora_request,
472
+ )
473
+
474
+ return self._build_decoder_only_llm_inputs(
475
+ prompt_comps,
476
+ prompt_adapter_request=prompt_adapter_request,
477
+ )
478
+
479
+ def preprocess(
480
+ self,
481
+ inputs: PromptInputs,
482
+ request_id: str,
483
+ lora_request: Optional[LoRARequest] = None,
484
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
485
+ ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
486
+ """Preprocess the input prompt."""
487
+ if self.is_encoder_decoder_model():
488
+ # Encoder-decoder model requires special mapping of
489
+ # input prompts to encoder & decoder
490
+ return self._process_encoder_decoder_prompt(
491
+ inputs,
492
+ request_id=request_id,
493
+ )
494
+
495
+ if is_explicit_encoder_decoder_prompt(inputs):
496
+ raise ValueError("Cannot pass encoder-decoder prompt "
497
+ "to decoder-only models")
498
+
499
+ # Decoder-only operation
500
+ return self._process_decoder_only_prompt(
501
+ inputs,
502
+ request_id=request_id,
503
+ lora_request=lora_request,
504
+ prompt_adapter_request=prompt_adapter_request,
505
+ )
506
+
507
+ async def preprocess_async(
508
+ self,
509
+ inputs: PromptInputs,
510
+ request_id: str,
511
+ lora_request: Optional[LoRARequest] = None,
512
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None,
513
+ ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
514
+ """Async version of :meth:`preprocess`."""
515
+ if self.is_encoder_decoder_model():
516
+ # Encoder-decoder model requires special mapping of
517
+ # input prompts to encoder & decoder
518
+ return await self._process_encoder_decoder_prompt_async(
519
+ inputs,
520
+ request_id=request_id,
521
+ )
522
+
523
+ if is_explicit_encoder_decoder_prompt(inputs):
524
+ raise ValueError("Cannot pass encoder-decoder prompt "
525
+ "to decoder-only models")
526
+
527
+ # Decoder-only operation
528
+ return await self._process_decoder_only_prompt_async(
529
+ inputs,
530
+ request_id=request_id,
531
+ lora_request=lora_request,
532
+ prompt_adapter_request=prompt_adapter_request,
533
+ )
534
+
535
+ def is_encoder_decoder_model(self):
536
+ return self.model_config.is_encoder_decoder_model