@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 295c4730a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Raise error when using encoder/decoder model with cpu backend (#8355)

Kevin Lin committed Sep 12, 2024 at 00:45 UTC 295c4730a85ce419e5b46e256240d69ad1cce619
2 files changed +9 -1
vllm/utils.py
+4
@@ -82,6 +82,9 @@ STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
82 "currently supported with encoder/"
83 "decoder models.")
84
85 +STR_NOT_IMPL_ENC_DEC_CPU = ("CPU is not currently supported with "
86 + "encoder/decoder models.")
87 +
88 # Efficiently import all enc/dec error strings
89 # rather than having to import all of the above
90 STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
@@ -97,6 +100,7 @@ STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
100 "STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH": STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
101 "STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
102 "STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
103 + "STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU
104 }
105
106 # Constants related to forcing the attention backend selection
vllm/worker/cpu_model_runner.py
+5 -1
@@ -15,7 +15,7 @@ from vllm.model_executor.model_loader import get_model
15 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16 MultiModalInputs)
17 from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
18 -from vllm.utils import make_tensor_with_pad
18 +from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
19 from vllm.worker.model_runner_base import (
20 ModelRunnerBase, ModelRunnerInputBase,
21 _add_attn_metadata_broadcastable_dict,
@@ -121,6 +121,10 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
121 # Lazy initialization.
122 self.model: nn.Module # Set after init_Model
123
124 + if self.model_config.is_encoder_decoder_model:
125 + raise NotImplementedError(
126 + STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
127 +
128 def load_model(self) -> None:
129 self.model = get_model(model_config=self.model_config,
130 load_config=self.load_config,