@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5ec9c0fb3c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Factor out input preprocessing to a separate class (#7329)

Cyrus Leung committed Sep 13, 2024 at 10:56 UTC 5ec9c0fb3c667c30117eb1fd743e0e7c13ccf997
5 files changed +590 -540
tests/engine/test_skip_tokenizer_init.py
+3 -2
@@ -11,9 +11,10 @@ def test_skip_tokenizer_initialization(model: str):
11 # token ids.
12 llm = LLM(model=model, skip_tokenizer_init=True)
13 sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True)
14 - with pytest.raises(ValueError) as err:
14 +
15 + with pytest.raises(ValueError, match="cannot pass text prompts when"):
16 llm.generate("abc", sampling_params)
16 - assert "prompts must be None if" in str(err.value)
17 +
18 outputs = llm.generate({"prompt_token_ids": [1, 2, 3]},
19 sampling_params=sampling_params)
20 assert len(outputs) > 0
vllm/engine/async_llm_engine.py
+4 -141
@@ -4,22 +4,17 @@ from functools import partial
4 from typing import (Any, AsyncGenerator, Callable, Dict, Iterable, List,
5 Mapping, Optional, Set, Tuple, Type, Union)
6
7 -from typing_extensions import assert_never
8 -
7 import vllm.envs as envs
8 from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
9 ParallelConfig, SchedulerConfig)
10 from vllm.core.scheduler import SchedulerOutputs
11 from vllm.engine.arg_utils import AsyncEngineArgs
12 from vllm.engine.async_timeout import asyncio_timeout
15 -from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
16 - PromptComponents, SchedulerOutputState)
13 +from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
14 from vllm.engine.metrics_types import StatLoggerBase
15 from vllm.executor.executor_base import ExecutorAsyncBase
16 from vllm.executor.ray_utils import initialize_ray_cluster
20 -from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
21 - SingletonPromptInputs)
22 -from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
17 +from vllm.inputs import PromptInputs
18 from vllm.logger import init_logger
19 from vllm.lora.request import LoRARequest
20 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -403,139 +398,6 @@ class _AsyncLLMEngine(LLMEngine):
398 """Stop the remote worker execution loop."""
399 await self.model_executor.stop_remote_worker_execution_loop_async()
400
406 - async def _tokenize_prompt_async(
407 - self,
408 - prompt: str,
409 - request_id: str,
410 - lora_request: Optional[LoRARequest],
411 - ) -> List[int]:
412 - """Async version of :meth:`_tokenize_prompt`."""
413 - tokenizer = self.get_tokenizer_group(
414 - missing_msg="prompts must be None if skip_tokenizer_init is True")
415 -
416 - return await tokenizer.encode_async(request_id=request_id,
417 - prompt=prompt,
418 - lora_request=lora_request)
419 -
420 - async def _extract_prompt_components_async(
421 - self,
422 - inputs: SingletonPromptInputs,
423 - request_id: str,
424 - lora_request: Optional[LoRARequest] = None,
425 - ) -> PromptComponents:
426 - """Async version of :meth:`_extract_prompt_components`."""
427 - if isinstance(inputs, str):
428 - prompt = inputs
429 - prompt_token_ids = await self._tokenize_prompt_async(
430 - prompt,
431 - request_id=request_id,
432 - lora_request=lora_request,
433 - )
434 - multi_modal_data = None
435 - elif isinstance(inputs, dict):
436 - if "prompt_token_ids" in inputs:
437 - prompt = None
438 - prompt_token_ids = inputs["prompt_token_ids"]
439 - else:
440 - # NOTE: This extra assignment is required to pass mypy
441 - prompt = parsed_prompt = inputs["prompt"]
442 - prompt_token_ids = await self._tokenize_prompt_async(
443 - parsed_prompt,
444 - request_id=request_id,
445 - lora_request=lora_request,
446 - )
447 -
448 - multi_modal_data = inputs.get("multi_modal_data")
449 - else:
450 - assert_never(inputs)
451 -
452 - return prompt, prompt_token_ids, multi_modal_data
453 -
454 - async def _process_encoder_decoder_prompt_async(
455 - self,
456 - inputs: PromptInputs,
457 - request_id: str,
458 - ) -> EncoderDecoderLLMInputs:
459 - """Async version of :meth:`_process_encoder_decoder_prompt`."""
460 - encoder_comps: PromptComponents
461 - decoder_comps: DecoderPromptComponents
462 -
463 - if is_explicit_encoder_decoder_prompt(inputs):
464 - encoder_task = self._extract_prompt_components_async(
465 - inputs["encoder_prompt"],
466 - request_id=request_id,
467 - )
468 -
469 - if (decoder_input := inputs["decoder_prompt"]) is None:
470 - encoder_comps = await encoder_task
471 - decoder_comps = None, None, None
472 - else:
473 - decoder_task = self._extract_prompt_components_async(
474 - decoder_input,
475 - request_id=request_id,
476 - )
477 -
478 - encoder_comps, decoder_comps = await asyncio.gather(
479 - encoder_task, decoder_task)
480 - else:
481 - encoder_comps = await self._extract_prompt_components_async(
482 - inputs,
483 - request_id=request_id,
484 - )
485 -
486 - decoder_comps = None, None, None
487 -
488 - return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
489 -
490 - async def _process_decoder_only_prompt_async(
491 - self,
492 - inputs: SingletonPromptInputs,
493 - request_id: str,
494 - lora_request: Optional[LoRARequest] = None,
495 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
496 - ) -> LLMInputs:
497 - """Async version of :meth:`_process_decoder_only_prompt`."""
498 - prompt_comps = await self._extract_prompt_components_async(
499 - inputs,
500 - request_id=request_id,
501 - lora_request=lora_request,
502 - )
503 -
504 - return self._build_decoder_only_llm_inputs(
505 - prompt_comps,
506 - prompt_adapter_request=prompt_adapter_request,
507 - )
508 -
509 - async def process_model_inputs_async(
510 - self,
511 - inputs: PromptInputs,
512 - request_id: str,
513 - lora_request: Optional[LoRARequest] = None,
514 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
515 - ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
516 - """Async version of :meth:`process_model_inputs`."""
517 - if self.is_encoder_decoder_model():
518 - # Encoder-decoder model requires special mapping of
519 - # input prompts to encoder & decoder
520 - model_inputs = await self._process_encoder_decoder_prompt_async(
521 - inputs,
522 - request_id=request_id,
523 - )
524 - else:
525 - if is_explicit_encoder_decoder_prompt(inputs):
526 - raise ValueError("Cannot pass encoder-decoder prompt "
527 - "to decoder-only models")
528 -
529 - # Decoder-only operation
530 - model_inputs = await self._process_decoder_only_prompt_async(
531 - inputs,
532 - request_id=request_id,
533 - lora_request=lora_request,
534 - prompt_adapter_request=prompt_adapter_request,
535 - )
536 -
537 - return self.input_processor(model_inputs)
538 -
401 async def add_request_async(
402 self,
403 request_id: str,
@@ -553,12 +415,13 @@ class _AsyncLLMEngine(LLMEngine):
415 if arrival_time is None:
416 arrival_time = time.time()
417
556 - processed_inputs = await self.process_model_inputs_async(
418 + preprocessed_inputs = await self.input_preprocessor.preprocess_async(
419 inputs,
420 request_id=request_id,
421 lora_request=lora_request,
422 prompt_adapter_request=prompt_adapter_request,
423 )
424 + processed_inputs = self.input_processor(preprocessed_inputs)
425
426 self._add_processed_request(
427 request_id=request_id,
vllm/engine/llm_engine.py
+13 -394
@@ -6,10 +6,10 @@ from dataclasses import dataclass
6 from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7 Iterable, List, Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
9 -from typing import Set, Tuple, Type, Union
9 +from typing import Set, Type, Union
10
11 import torch
12 -from typing_extensions import TypeVar, assert_never
12 +from typing_extensions import TypeVar
13
14 import vllm.envs as envs
15 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
@@ -28,13 +28,11 @@ from vllm.engine.output_processor.util import create_output_by_sequence_group
28 from vllm.executor.executor_base import ExecutorBase
29 from vllm.executor.ray_utils import initialize_ray_cluster
30 from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
31 - InputRegistry, LLMInputs, PromptInputs,
32 - SingletonPromptInputs)
33 -from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
31 + InputRegistry, LLMInputs, PromptInputs)
32 +from vllm.inputs.preprocess import InputPreprocessor
33 from vllm.logger import init_logger
34 from vllm.lora.request import LoRARequest
35 from vllm.model_executor.layers.sampler import SamplerOutput
37 -from vllm.multimodal import MultiModalDataDict
36 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
37 RequestOutputFactory)
38 from vllm.pooling_params import PoolingParams
@@ -75,11 +73,6 @@ def _load_generation_config_dict(model_config: ModelConfig) -> Dict[str, Any]:
73 _G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
74 _O = TypeVar("_O", RequestOutput, EmbeddingRequestOutput)
75
78 -PromptComponents = Tuple[Optional[str], List[int],
79 - Optional[MultiModalDataDict]]
80 -DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
81 - Optional[MultiModalDataDict]]
82 -
76
77 @dataclass
78 class SchedulerOutputState:
@@ -313,6 +306,9 @@ class LLMEngine:
306 self.generation_config_fields = _load_generation_config_dict(
307 model_config)
308
309 + self.input_preprocessor = InputPreprocessor(model_config,
310 + self.tokenizer)
311 +
312 self.input_registry = input_registry
313 self.input_processor = input_registry.create_input_processor(
314 model_config)
@@ -571,19 +567,15 @@ class LLMEngine:
567 if model_executor := getattr(self, "model_executor", None):
568 model_executor.shutdown()
569
574 - MISSING_TOKENIZER_GROUP_MSG = ("Unable to get tokenizer because "
575 - "skip_tokenizer_init is True")
576 -
570 def get_tokenizer_group(
571 self,
572 group_type: Type[_G] = BaseTokenizerGroup,
580 - *,
581 - missing_msg: str = MISSING_TOKENIZER_GROUP_MSG,
573 ) -> _G:
574 tokenizer_group = self.tokenizer
575
576 if tokenizer_group is None:
586 - raise ValueError(missing_msg)
577 + raise ValueError("Unable to get tokenizer because "
578 + "skip_tokenizer_init is True")
579 if not isinstance(tokenizer_group, group_type):
580 raise TypeError("Invalid type of tokenizer group. "
581 f"Expected type: {group_type}, but "
@@ -615,52 +607,6 @@ class LLMEngine:
607 self.prompt_adapter_config.verify_with_model_config(
608 self.model_config)
609
618 - def _get_bos_token_id(self,
619 - lora_request: Optional[LoRARequest] = None
620 - ) -> Optional[int]:
621 - if self.tokenizer is None:
622 - logger.warning("Using None for BOS token id because tokenizer "
623 - "is not initialized")
624 - return None
625 -
626 - return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id
627 -
628 - def _get_eos_token_id(self,
629 - lora_request: Optional[LoRARequest] = None
630 - ) -> Optional[int]:
631 - if self.tokenizer is None:
632 - logger.warning("Using None for EOS token id because tokenizer "
633 - "is not initialized")
634 - return None
635 -
636 - return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
637 -
638 - def _get_decoder_start_token_id(self) -> Optional[int]:
639 - '''
640 - Obtain the decoder start token id employed by an encoder/decoder
641 - model. Returns None for non-encoder/decoder models or if the
642 - model config is unavailable.
643 - '''
644 -
645 - if not self.is_encoder_decoder_model():
646 - logger.warning("Using None for decoder start token id because "
647 - "this is not an encoder/decoder model.")
648 - return None
649 -
650 - if (self.model_config is None or self.model_config.hf_config is None):
651 - logger.warning("Using None for decoder start token id because "
652 - "model config is not available.")
653 - return None
654 -
655 - dec_start_token_id = getattr(self.model_config.hf_config,
656 - 'decoder_start_token_id', None)
657 - if dec_start_token_id is None:
658 - logger.warning("Falling back on <BOS> for decoder start token id "
659 - "because decoder start token id is not available.")
660 - dec_start_token_id = self._get_bos_token_id()
661 -
662 - return dec_start_token_id
663 -
610 def _add_processed_request(
611 self,
612 request_id: str,
@@ -675,7 +621,7 @@ class LLMEngine:
621 # Create the sequences.
622 block_size = self.cache_config.block_size
623 seq_id = next(self.seq_counter)
678 - eos_token_id = self._get_eos_token_id(lora_request)
624 + eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)
625
626 seq = Sequence(seq_id, processed_inputs, block_size, eos_token_id,
627 lora_request, prompt_adapter_request)
@@ -725,334 +671,6 @@ class LLMEngine:
671 def stop_remote_worker_execution_loop(self) -> None:
672 self.model_executor.stop_remote_worker_execution_loop()
673
728 - _LLMInputComponentsType = Tuple[str, List[int]]
729 -
730 - def _prepare_decoder_input_ids_for_generation(
731 - self,
732 - decoder_input_ids: Optional[List[int]],
733 - ) -> List[int]:
734 - """
735 - Prepares `decoder_input_ids` for generation with encoder-decoder models.
736 -
737 - Based on
738 -
739 - https://github.com/huggingface/transformers/blob/
740 - 4037a2b5b1278736e566aec12e169100275545ea/
741 - src/transformers/generation/utils.py
742 -
743 - specifically GenerationMixin._prepare_decoder_input_ids_for_generation()
744 -
745 - Arguments:
746 -
747 - * decoder_input_ids: input token ids to preprocess
748 -
749 - Returns:
750 -
751 - * Processed token list
752 - """
753 -
754 - decoder_start_token_id = self._get_decoder_start_token_id()
755 - assert decoder_start_token_id is not None
756 -
757 - if decoder_input_ids is None:
758 - # no decoder prompt input ->
759 - # use decoder_start_token_id as decoder_input_ids
760 - decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
761 -
762 - if (len(decoder_input_ids) == 0
763 - or decoder_input_ids[0] != decoder_start_token_id):
764 - decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
765 -
766 - return decoder_input_ids
767 -
768 - def _tokenize_prompt(
769 - self,
770 - prompt: str,
771 - request_id: str,
772 - lora_request: Optional[LoRARequest],
773 - ) -> List[int]:
774 - '''
775 - Wrapper around application of the model's tokenizer.
776 -
777 - Arguments:
778 -
779 - * prompt
780 - * request_id
781 - * lora_request
782 -
783 - Returns:
784 -
785 - * prompt token ids
786 - '''
787 -
788 - tokenizer = self.get_tokenizer_group(
789 - missing_msg="prompts must be None if skip_tokenizer_init is True")
790 -
791 - return tokenizer.encode(request_id=request_id,
792 - prompt=prompt,
793 - lora_request=lora_request)
794 -
795 - def _extract_prompt_components(
796 - self,
797 - inputs: SingletonPromptInputs,
798 - request_id: str,
799 - lora_request: Optional[LoRARequest] = None,
800 - ) -> PromptComponents:
801 - '''
802 - Extract the components of any single encoder or decoder input prompt.
803 -
804 - Arguments:
805 -
806 - * request_id
807 - * inputs: single encoder or decoder input prompt
808 - * lora_request: this is only valid for decoder prompts
809 -
810 - Returns:
811 -
812 - * prompt
813 - * prompt_token_ids
814 - * multi_modal_data
815 - '''
816 -
817 - if isinstance(inputs, str):
818 - prompt = inputs
819 - prompt_token_ids = self._tokenize_prompt(
820 - prompt,
821 - request_id=request_id,
822 - lora_request=lora_request,
823 - )
824 - multi_modal_data = None
825 - elif isinstance(inputs, dict):
826 - if "prompt_token_ids" in inputs:
827 - prompt = None
828 - prompt_token_ids = inputs["prompt_token_ids"]
829 - else:
830 - # NOTE: This extra assignment is required to pass mypy
831 - prompt = parsed_prompt = inputs["prompt"]
832 - prompt_token_ids = self._tokenize_prompt(
833 - parsed_prompt,
834 - request_id=request_id,
835 - lora_request=lora_request,
836 - )
837 -
838 - multi_modal_data = inputs.get("multi_modal_data")
839 - else:
840 - assert_never(inputs)
841 -
842 - return prompt, prompt_token_ids, multi_modal_data
843 -
844 - def _apply_prompt_adapter(
845 - self,
846 - prompt_token_ids: List[int],
847 - prompt_adapter_request: Optional[PromptAdapterRequest],
848 - ) -> List[int]:
849 - if prompt_adapter_request:
850 - prompt_token_ids = (
851 - [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
852 - + prompt_token_ids)
853 -
854 - return prompt_token_ids
855 -
856 - def _get_default_enc_dec_decoder_prompt(self) -> List[int]:
857 - '''
858 - Specifically for encoder/decoder models:
859 - generate a default decoder prompt for when
860 - the user specifies only the encoder prompt.
861 -
862 - Encoder/decoder models utilize the decoder
863 - prompt in different ways; as new models are
864 - added, it is intended that this function
865 - will be extended to produce differing
866 - default decoder prompts, depending on the
867 - model variety.
868 -
869 - Absent a special case, the default behavior
870 - of this method is to mirror the behavior of
871 - the HuggingFace (HF) GenerationMixin for a None
872 - decoder prompt, which is to employ a logit processor
873 - setting to force the first decoded token to be <BOS>.
874 - Here, this behavior is approximated by having the
875 - "default" decoder prompt be <BOS>.
876 -
877 - However, it is possible that in the future
878 - other models may have different or more
879 - complex logic for the default decoder prompt.
880 - This motivates having a special helper method
881 - for default decoder prompts.
882 -
883 - Returns:
884 -
885 - * prompt_token_ids
886 - '''
887 -
888 - bos_token_id = self._get_bos_token_id()
889 - assert bos_token_id is not None
890 - return [bos_token_id]
891 -
892 - def _build_enc_dec_llm_inputs(
893 - self,
894 - encoder_comps: PromptComponents,
895 - decoder_comps: DecoderPromptComponents,
896 - ) -> EncoderDecoderLLMInputs:
897 - encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
898 - decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
899 -
900 - if encoder_mm_data is not None or decoder_mm_data is not None:
901 - raise ValueError("Multi-modal encoder-decoder models are "
902 - "not supported yet")
903 -
904 - decoder_prompt_ids = (
905 - self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
906 -
907 - return EncoderDecoderLLMInputs(
908 - prompt_token_ids=decoder_prompt_ids,
909 - prompt=decoder_prompt,
910 - encoder_prompt_token_ids=encoder_prompt_ids,
911 - encoder_prompt=encoder_prompt,
912 - )
913 -
914 - def _process_encoder_decoder_prompt(
915 - self,
916 - inputs: PromptInputs,
917 - request_id: str,
918 - ) -> EncoderDecoderLLMInputs:
919 - '''
920 - For encoder/decoder models only:
921 - Process an input prompt into an
922 - :class:`EncoderDecoderLLMInputs` instance.
923 -
924 - There are two types of input prompts:
925 - singleton prompts which carry only the
926 - encoder prompt, and explicit encoder/decoder
927 - prompts which carry both the encoder and the
928 - decoder prompts as member variables.
929 -
930 - This function handles the following scenarios:
931 - * Singleton encoder prompt: extract encoder prompt
932 - token ids & infer default decoder prompt token ids
933 - * Explicit encoder/decoder prompt: extract encoder
934 - and decoder prompt token ids
935 -
936 - Note that for Explicit encoder/decoder prompts,
937 - each sub-prompt (encoder or decoder prompt) can
938 - have any possible singleton type; thus this
939 - method relies on helper functions to obtain
940 - token ids for the sub-prompts.
941 -
942 - Arguments:
943 -
944 - * inputs: an input prompt
945 - * request_id
946 -
947 - Returns:
948 -
949 - * :class:`EncoderDecoderLLMInputs` instance
950 - '''
951 -
952 - encoder_comps: PromptComponents
953 - decoder_comps: DecoderPromptComponents
954 -
955 - if is_explicit_encoder_decoder_prompt(inputs):
956 - encoder_comps = self._extract_prompt_components(
957 - inputs["encoder_prompt"],
958 - request_id=request_id,
959 - )
960 -
961 - if (decoder_input := inputs["decoder_prompt"]) is None:
962 - decoder_comps = None, None, None
963 - else:
964 - decoder_comps = self._extract_prompt_components(
965 - decoder_input,
966 - request_id=request_id,
967 - )
968 - else:
969 - encoder_comps = self._extract_prompt_components(
970 - inputs,
971 - request_id=request_id,
972 - )
973 -
974 - decoder_comps = None, None, None
975 -
976 - return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
977 -
978 - def _build_decoder_only_llm_inputs(
979 - self,
980 - prompt_comps: PromptComponents,
981 - prompt_adapter_request: Optional[PromptAdapterRequest],
982 - ) -> LLMInputs:
983 - prompt, prompt_token_ids, multi_modal_data = prompt_comps
984 -
985 - prompt_token_ids = self._apply_prompt_adapter(
986 - prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
987 -
988 - return LLMInputs(prompt_token_ids=prompt_token_ids,
989 - prompt=prompt,
990 - multi_modal_data=multi_modal_data)
991 -
992 - def _process_decoder_only_prompt(
993 - self,
994 - inputs: SingletonPromptInputs,
995 - request_id: str,
996 - lora_request: Optional[LoRARequest] = None,
997 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
998 - ) -> LLMInputs:
999 - '''
1000 - For decoder-only models:
1001 - Process an input prompt into an :class:`LLMInputs` instance.
1002 -
1003 - Arguments:
1004 -
1005 - * inputs: input prompt
1006 - * request_id
1007 - * lora_request
1008 - * prompt_adapter_request
1009 -
1010 - Returns:
1011 -
1012 - * :class:`LLMInputs` instance
1013 - '''
1014 -
1015 - prompt_comps = self._extract_prompt_components(
1016 - inputs,
1017 - request_id=request_id,
1018 - lora_request=lora_request,
1019 - )
1020 -
1021 - return self._build_decoder_only_llm_inputs(
1022 - prompt_comps,
1023 - prompt_adapter_request=prompt_adapter_request,
1024 - )
1025 -
1026 - def process_model_inputs(
1027 - self,
1028 - inputs: PromptInputs,
1029 - request_id: str,
1030 - lora_request: Optional[LoRARequest] = None,
1031 - prompt_adapter_request: Optional[PromptAdapterRequest] = None,
1032 - ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
1033 -
1034 - if self.is_encoder_decoder_model():
1035 - # Encoder-decoder model requires special mapping of
1036 - # input prompts to encoder & decoder
1037 - model_inputs = self._process_encoder_decoder_prompt(
1038 - inputs,
1039 - request_id=request_id,
1040 - )
1041 - else:
1042 - if is_explicit_encoder_decoder_prompt(inputs):
1043 - raise ValueError("Cannot pass encoder-decoder prompt "
1044 - "to decoder-only models")
1045 -
1046 - # Decoder-only operation
1047 - model_inputs = self._process_decoder_only_prompt(
1048 - inputs,
1049 - request_id=request_id,
1050 - lora_request=lora_request,
1051 - prompt_adapter_request=prompt_adapter_request,
1052 - )
1053 -
1054 - return self.input_processor(model_inputs)
1055 -
674 def add_request(
675 self,
676 request_id: str,
@@ -1111,12 +729,13 @@ class LLMEngine:
729 if arrival_time is None:
730 arrival_time = time.time()
731
1114 - processed_inputs = self.process_model_inputs(
732 + preprocessed_inputs = self.input_preprocessor.preprocess(
733 inputs,
734 request_id=request_id,
735 lora_request=lora_request,
736 prompt_adapter_request=prompt_adapter_request,
737 )
738 + processed_inputs = self.input_processor(preprocessed_inputs)
739
740 self._add_processed_request(
741 request_id=request_id,
@@ -2043,7 +1662,7 @@ class LLMEngine:
1662 metrics.model_execute_time)
1663
1664 def is_encoder_decoder_model(self):
2046 - return self.model_config.is_encoder_decoder_model
1665 + return self.input_preprocessor.is_encoder_decoder_model()
1666
1667 def is_embedding_model(self):
1668 return self.model_config.is_embedding_model
vllm/inputs/parse.py
+34 -3
@@ -5,7 +5,8 @@ from typing_extensions import TypeIs
5 from vllm.utils import is_list_of
6
7 from .data import (EncoderDecoderLLMInputs, ExplicitEncoderDecoderPrompt,
8 - LLMInputs, PromptInputs)
8 + LLMInputs, PromptInputs, SingletonPromptInputs, TextPrompt,
9 + TokensPrompt)
10
11
12 class ParsedText(TypedDict):
@@ -60,8 +61,38 @@ def parse_and_batch_prompt(
61 for elem in prompt
62 ]
63
63 - raise ValueError("prompt must be a string, array of strings, "
64 - "array of tokens, or array of token arrays")
64 + raise TypeError("prompt must be a string, array of strings, "
65 + "array of tokens, or array of token arrays")
66 +
67 +
68 +class ParsedStrPrompt(TypedDict):
69 + type: Literal["str"]
70 + content: str
71 +
72 +
73 +class ParsedTextPrompt(TypedDict):
74 + type: Literal["text"]
75 + content: TextPrompt
76 +
77 +
78 +class ParsedTokensPrompt(TypedDict):
79 + type: Literal["tokens"]
80 + content: TokensPrompt
81 +
82 +
83 +def parse_singleton_prompt(
84 + inputs: SingletonPromptInputs,
85 +) -> Union[ParsedStrPrompt, ParsedTextPrompt, ParsedTokensPrompt]:
86 + if isinstance(inputs, str):
87 + return ParsedStrPrompt(type="str", content=inputs)
88 + elif isinstance(inputs, dict):
89 + if "prompt_token_ids" in inputs:
90 + return ParsedTokensPrompt(type="tokens",
91 + content=inputs) # type: ignore
92 + elif "prompt" in inputs:
93 + return ParsedTextPrompt(type="text", content=inputs)
94 +
95 + raise TypeError("inputs must be a string, TextPrompt, or TokensPrompt")
96
97
98 def is_explicit_encoder_decoder_prompt(
vllm/inputs/preprocess.py new
+536
@@ -0,0 +1,536 @@
1 +import asyncio
2 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union
3 +
4 +from typing_extensions import assert_never
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.logger import init_logger
8 +from vllm.lora.request import LoRARequest
9 +from vllm.prompt_adapter.request import PromptAdapterRequest
10 +from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11 +
12 +from .data import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
13 + SingletonPromptInputs)
14 +from .parse import is_explicit_encoder_decoder_prompt, parse_singleton_prompt
15 +
16 +if TYPE_CHECKING:
17 + from vllm.multimodal import MultiModalDataDict
18 +
19 +logger = init_logger(__name__)
20 +
21 +PromptComponents = Tuple[Optional[str], List[int],
22 + Optional["MultiModalDataDict"]]
23 +DecoderPromptComponents = Tuple[Optional[str], Optional[List[int]],
24 + Optional["MultiModalDataDict"]]
25 +
26 +
27 +class InputPreprocessor:
28 +
29 + def __init__(
30 + self,
31 + model_config: ModelConfig,
32 + tokenizer: Optional[BaseTokenizerGroup],
33 + ) -> None:
34 + super().__init__()
35 +
36 + self.model_config = model_config
37 + self.tokenizer = tokenizer
38 +
39 + def get_tokenizer_group(self) -> BaseTokenizerGroup:
40 + if self.tokenizer is None:
41 + raise ValueError("You cannot pass text prompts when "
42 + "`skip_tokenizer_init` is True")
43 +
44 + return self.tokenizer
45 +
46 + def get_bos_token_id(self,
47 + lora_request: Optional[LoRARequest] = None
48 + ) -> Optional[int]:
49 + if self.tokenizer is None:
50 + logger.warning("Using None for BOS token id because tokenizer "
51 + "is not initialized")
52 + return None
53 +
54 + return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id
55 +
56 + def get_eos_token_id(self,
57 + lora_request: Optional[LoRARequest] = None
58 + ) -> Optional[int]:
59 + if self.tokenizer is None:
60 + logger.warning("Using None for EOS token id because tokenizer "
61 + "is not initialized")
62 + return None
63 +
64 + return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
65 +
66 + def get_decoder_start_token_id(self) -> Optional[int]:
67 + '''
68 + Obtain the decoder start token id employed by an encoder/decoder
69 + model. Returns None for non-encoder/decoder models or if the
70 + model config is unavailable.
71 + '''
72 +
73 + if not self.is_encoder_decoder_model():
74 + logger.warning("Using None for decoder start token id because "
75 + "this is not an encoder/decoder model.")
76 + return None
77 +
78 + if (self.model_config is None or self.model_config.hf_config is None):
79 + logger.warning("Using None for decoder start token id because "
80 + "model config is not available.")
81 + return None
82 +
83 + dec_start_token_id = getattr(self.model_config.hf_config,
84 + 'decoder_start_token_id', None)
85 + if dec_start_token_id is None:
86 + logger.warning("Falling back on <BOS> for decoder start token id "
87 + "because decoder start token id is not available.")
88 + dec_start_token_id = self.get_bos_token_id()
89 +
90 + return dec_start_token_id
91 +
92 + def _get_default_enc_dec_decoder_prompt(self) -> List[int]:
93 + '''
94 + Specifically for encoder/decoder models:
95 + generate a default decoder prompt for when
96 + the user specifies only the encoder prompt.
97 +
98 + Encoder/decoder models utilize the decoder
99 + prompt in different ways; as new models are
100 + added, it is intended that this function
101 + will be extended to produce differing
102 + default decoder prompts, depending on the
103 + model variety.
104 +
105 + Absent a special case, the default behavior
106 + of this method is to mirror the behavior of
107 + the HuggingFace (HF) GenerationMixin for a None
108 + decoder prompt, which is to employ a logit processor
109 + setting to force the first decoded token to be <BOS>.
110 + Here, this behavior is approximated by having the
111 + "default" decoder prompt be <BOS>.
112 +
113 + However, it is possible that in the future
114 + other models may have different or more
115 + complex logic for the default decoder prompt.
116 + This motivates having a special helper method
117 + for default decoder prompts.
118 +
119 + Returns:
120 +
121 + * prompt_token_ids
122 + '''
123 +
124 + bos_token_id = self.get_bos_token_id()
125 + assert bos_token_id is not None
126 + return [bos_token_id]
127 +
128 + def _prepare_decoder_input_ids_for_generation(
129 + self,
130 + decoder_input_ids: Optional[List[int]],
131 + ) -> List[int]:
132 + """
133 + Prepares `decoder_input_ids` for generation with encoder-decoder models.
134 +
135 + Based on
136 +
137 + https://github.com/huggingface/transformers/blob/
138 + 4037a2b5b1278736e566aec12e169100275545ea/
139 + src/transformers/generation/utils.py
140 +
141 + specifically GenerationMixin._prepare_decoder_input_ids_for_generation()
142 +
143 + Arguments:
144 +
145 + * decoder_input_ids: input token ids to preprocess
146 +
147 + Returns:
148 +
149 + * Processed token list
150 + """
151 +
152 + decoder_start_token_id = self.get_decoder_start_token_id()
153 + assert decoder_start_token_id is not None
154 +
155 + if decoder_input_ids is None:
156 + # no decoder prompt input ->
157 + # use decoder_start_token_id as decoder_input_ids
158 + decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
159 +
160 + if (len(decoder_input_ids) == 0
161 + or decoder_input_ids[0] != decoder_start_token_id):
162 + decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
163 +
164 + return decoder_input_ids
165 +
166 + def _apply_prompt_adapter(
167 + self,
168 + prompt_token_ids: List[int],
169 + prompt_adapter_request: Optional[PromptAdapterRequest],
170 + ) -> List[int]:
171 + if prompt_adapter_request:
172 + prompt_token_ids = (
173 + [0] * prompt_adapter_request.prompt_adapter_num_virtual_tokens
174 + + prompt_token_ids)
175 +
176 + return prompt_token_ids
177 +
178 + def _tokenize_prompt(
179 + self,
180 + prompt: str,
181 + request_id: str,
182 + lora_request: Optional[LoRARequest],
183 + ) -> List[int]:
184 + """
185 + Apply the model's tokenizer to a text prompt, returning the
186 + corresponding token IDs.
187 + """
188 + tokenizer = self.get_tokenizer_group()
189 +
190 + return tokenizer.encode(request_id=request_id,
191 + prompt=prompt,
192 + lora_request=lora_request)
193 +
194 + async def _tokenize_prompt_async(
195 + self,
196 + prompt: str,
197 + request_id: str,
198 + lora_request: Optional[LoRARequest],
199 + ) -> List[int]:
200 + """Async version of :meth:`_tokenize_prompt`."""
201 + tokenizer = self.get_tokenizer_group()
202 +
203 + return await tokenizer.encode_async(request_id=request_id,
204 + prompt=prompt,
205 + lora_request=lora_request)
206 +
207 + def _extract_prompt_components(
208 + self,
209 + inputs: SingletonPromptInputs,
210 + request_id: str,
211 + lora_request: Optional[LoRARequest] = None,
212 + ) -> PromptComponents:
213 + '''
214 + Extract the components of any single encoder or decoder input prompt.
215 +
216 + Arguments:
217 +
218 + * request_id
219 + * inputs: single encoder or decoder input prompt
220 + * lora_request: this is only valid for decoder prompts
221 +
222 + Returns:
223 +
224 + * prompt
225 + * prompt_token_ids
226 + * multi_modal_data
227 + '''
228 +
229 + parsed = parse_singleton_prompt(inputs)
230 +
231 + if parsed["type"] == "str":
232 + prompt = parsed["content"]
233 + prompt_token_ids = self._tokenize_prompt(
234 + prompt,
235 + request_id=request_id,
236 + lora_request=lora_request,
237 + )
238 + multi_modal_data = None
239 + elif parsed["type"] == "tokens":
240 + prompt = None
241 + prompt_token_ids = parsed["content"]["prompt_token_ids"]
242 + multi_modal_data = parsed["content"].get("multi_modal_data")
243 + elif parsed["type"] == "text":
244 + prompt = parsed["content"]["prompt"]
245 + prompt_token_ids = self._tokenize_prompt(
246 + prompt,
247 + request_id=request_id,
248 + lora_request=lora_request,
249 + )
250 + multi_modal_data = parsed["content"].get("multi_modal_data")
251 + else:
252 + assert_never(parsed)
253 +
254 + return prompt, prompt_token_ids, multi_modal_data
255 +
256 + async def _extract_prompt_components_async(
257 + self,
258 + inputs: SingletonPromptInputs,
259 + request_id: str,
260 + lora_request: Optional[LoRARequest] = None,
261 + ) -> PromptComponents:
262 + """Async version of :meth:`_extract_prompt_components`."""
263 + parsed = parse_singleton_prompt(inputs)
264 +
265 + if parsed["type"] == "str":
266 + prompt = parsed["content"]
267 + prompt_token_ids = await self._tokenize_prompt_async(
268 + prompt,
269 + request_id=request_id,
270 + lora_request=lora_request,
271 + )
272 + multi_modal_data = None
273 + elif parsed["type"] == "tokens":
274 + prompt = None
275 + prompt_token_ids = parsed["content"]["prompt_token_ids"]
276 + multi_modal_data = parsed["content"].get("multi_modal_data")
277 + elif parsed["type"] == "text":
278 + prompt = parsed["content"]["prompt"]
279 + prompt_token_ids = await self._tokenize_prompt_async(
280 + prompt,
281 + request_id=request_id,
282 + lora_request=lora_request,
283 + )
284 + multi_modal_data = parsed["content"].get("multi_modal_data")
285 + else:
286 + assert_never(parsed)
287 +
288 + return prompt, prompt_token_ids, multi_modal_data
289 +
290 + def _build_enc_dec_llm_inputs(
291 + self,
292 + encoder_comps: PromptComponents,
293 + decoder_comps: DecoderPromptComponents,
294 + ) -> EncoderDecoderLLMInputs:
295 + encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
296 + decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
297 +
298 + if encoder_mm_data is not None or decoder_mm_data is not None:
299 + raise ValueError("Multi-modal encoder-decoder models are "
300 + "not supported yet")
301 +
302 + decoder_prompt_ids = (
303 + self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
304 +
305 + return EncoderDecoderLLMInputs(
306 + prompt_token_ids=decoder_prompt_ids,
307 + prompt=decoder_prompt,
308 + encoder_prompt_token_ids=encoder_prompt_ids,
309 + encoder_prompt=encoder_prompt,
310 + )
311 +
312 + def _process_encoder_decoder_prompt(
313 + self,
314 + inputs: PromptInputs,
315 + request_id: str,
316 + ) -> EncoderDecoderLLMInputs:
317 + '''
318 + For encoder/decoder models only:
319 + Process an input prompt into an
320 + :class:`EncoderDecoderLLMInputs` instance.
321 +
322 + There are two types of input prompts:
323 + singleton prompts which carry only the
324 + encoder prompt, and explicit encoder/decoder
325 + prompts which carry both the encoder and the
326 + decoder prompts as member variables.
327 +
328 + This function handles the following scenarios:
329 + * Singleton encoder prompt: extract encoder prompt
330 + token ids & infer default decoder prompt token ids
331 + * Explicit encoder/decoder prompt: extract encoder
332 + and decoder prompt token ids
333 +
334 + Note that for Explicit encoder/decoder prompts,
335 + each sub-prompt (encoder or decoder prompt) can
336 + have any possible singleton type; thus this
337 + method relies on helper functions to obtain
338 + token ids for the sub-prompts.
339 +
340 + Arguments:
341 +
342 + * inputs: an input prompt
343 + * request_id
344 +
345 + Returns:
346 +
347 + * :class:`EncoderDecoderLLMInputs` instance
348 + '''
349 +
350 + encoder_comps: PromptComponents
351 + decoder_comps: DecoderPromptComponents
352 +
353 + if is_explicit_encoder_decoder_prompt(inputs):
354 + encoder_comps = self._extract_prompt_components(
355 + inputs["encoder_prompt"],
356 + request_id=request_id,
357 + )
358 +
359 + if (decoder_input := inputs["decoder_prompt"]) is None:
360 + decoder_comps = None, None, None
361 + else:
362 + decoder_comps = self._extract_prompt_components(
363 + decoder_input,
364 + request_id=request_id,
365 + )
366 + else:
367 + encoder_comps = self._extract_prompt_components(
368 + inputs,
369 + request_id=request_id,
370 + )
371 +
372 + decoder_comps = None, None, None
373 +
374 + return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
375 +
376 + async def _process_encoder_decoder_prompt_async(
377 + self,
378 + inputs: PromptInputs,
379 + request_id: str,
380 + ) -> EncoderDecoderLLMInputs:
381 + """Async version of :meth:`_process_encoder_decoder_prompt`."""
382 + encoder_comps: PromptComponents
383 + decoder_comps: DecoderPromptComponents
384 +
385 + if is_explicit_encoder_decoder_prompt(inputs):
386 + encoder_task = self._extract_prompt_components_async(
387 + inputs["encoder_prompt"],
388 + request_id=request_id,
389 + )
390 +
391 + if (decoder_input := inputs["decoder_prompt"]) is None:
392 + encoder_comps = await encoder_task
393 + decoder_comps = None, None, None
394 + else:
395 + decoder_task = self._extract_prompt_components_async(
396 + decoder_input,
397 + request_id=request_id,
398 + )
399 +
400 + encoder_comps, decoder_comps = await asyncio.gather(
401 + encoder_task, decoder_task)
402 + else:
403 + encoder_comps = await self._extract_prompt_components_async(
404 + inputs,
405 + request_id=request_id,
406 + )
407 +
408 + decoder_comps = None, None, None
409 +
410 + return self._build_enc_dec_llm_inputs(encoder_comps, decoder_comps)
411 +
412 + def _build_decoder_only_llm_inputs(
413 + self,
414 + prompt_comps: PromptComponents,
415 + prompt_adapter_request: Optional[PromptAdapterRequest],
416 + ) -> LLMInputs:
417 + prompt, prompt_token_ids, multi_modal_data = prompt_comps
418 +
419 + prompt_token_ids = self._apply_prompt_adapter(
420 + prompt_token_ids, prompt_adapter_request=prompt_adapter_request)
421 +
422 + return LLMInputs(prompt_token_ids=prompt_token_ids,
423 + prompt=prompt,
424 + multi_modal_data=multi_modal_data)
425 +
426 + def _process_decoder_only_prompt(
427 + self,
428 + inputs: SingletonPromptInputs,
429 + request_id: str,
430 + lora_request: Optional[LoRARequest] = None,
431 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
432 + ) -> LLMInputs:
433 + '''
434 + For decoder-only models:
435 + Process an input prompt into an :class:`LLMInputs` instance.
436 +
437 + Arguments:
438 +
439 + * inputs: input prompt
440 + * request_id
441 + * lora_request
442 + * prompt_adapter_request
443 +
444 + Returns:
445 +
446 + * :class:`LLMInputs` instance
447 + '''
448 +
449 + prompt_comps = self._extract_prompt_components(
450 + inputs,
451 + request_id=request_id,
452 + lora_request=lora_request,
453 + )
454 +
455 + return self._build_decoder_only_llm_inputs(
456 + prompt_comps,
457 + prompt_adapter_request=prompt_adapter_request,
458 + )
459 +
460 + async def _process_decoder_only_prompt_async(
461 + self,
462 + inputs: SingletonPromptInputs,
463 + request_id: str,
464 + lora_request: Optional[LoRARequest] = None,
465 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
466 + ) -> LLMInputs:
467 + """Async version of :meth:`_process_decoder_only_prompt`."""
468 + prompt_comps = await self._extract_prompt_components_async(
469 + inputs,
470 + request_id=request_id,
471 + lora_request=lora_request,
472 + )
473 +
474 + return self._build_decoder_only_llm_inputs(
475 + prompt_comps,
476 + prompt_adapter_request=prompt_adapter_request,
477 + )
478 +
479 + def preprocess(
480 + self,
481 + inputs: PromptInputs,
482 + request_id: str,
483 + lora_request: Optional[LoRARequest] = None,
484 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
485 + ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
486 + """Preprocess the input prompt."""
487 + if self.is_encoder_decoder_model():
488 + # Encoder-decoder model requires special mapping of
489 + # input prompts to encoder & decoder
490 + return self._process_encoder_decoder_prompt(
491 + inputs,
492 + request_id=request_id,
493 + )
494 +
495 + if is_explicit_encoder_decoder_prompt(inputs):
496 + raise ValueError("Cannot pass encoder-decoder prompt "
497 + "to decoder-only models")
498 +
499 + # Decoder-only operation
500 + return self._process_decoder_only_prompt(
501 + inputs,
502 + request_id=request_id,
503 + lora_request=lora_request,
504 + prompt_adapter_request=prompt_adapter_request,
505 + )
506 +
507 + async def preprocess_async(
508 + self,
509 + inputs: PromptInputs,
510 + request_id: str,
511 + lora_request: Optional[LoRARequest] = None,
512 + prompt_adapter_request: Optional[PromptAdapterRequest] = None,
513 + ) -> Union[LLMInputs, EncoderDecoderLLMInputs]:
514 + """Async version of :meth:`preprocess`."""
515 + if self.is_encoder_decoder_model():
516 + # Encoder-decoder model requires special mapping of
517 + # input prompts to encoder & decoder
518 + return await self._process_encoder_decoder_prompt_async(
519 + inputs,
520 + request_id=request_id,
521 + )
522 +
523 + if is_explicit_encoder_decoder_prompt(inputs):
524 + raise ValueError("Cannot pass encoder-decoder prompt "
525 + "to decoder-only models")
526 +
527 + # Decoder-only operation
528 + return await self._process_decoder_only_prompt_async(
529 + inputs,
530 + request_id=request_id,
531 + lora_request=lora_request,
532 + prompt_adapter_request=prompt_adapter_request,
533 + )
534 +
535 + def is_encoder_decoder_model(self):
536 + return self.model_config.is_encoder_decoder_model