A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Raise error when using encoder/decoder model with cpu backend (#8355)
Kevin Lin committed
Sep 12, 2024 at 00:45 UTC
295c4730a85ce419e5b46e256240d69ad1cce619
2 files changed
+9
-1
vllm/utils.py
+4
@@ -82,6 +82,9 @@ STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER = ("Prompt adapters are not "
82
"currently supported with encoder/"
83
"decoder models.")
84
85
+STR_NOT_IMPL_ENC_DEC_CPU = ("CPU is not currently supported with "
86
+ "encoder/decoder models.")
87
+
88
# Efficiently import all enc/dec error strings
89
# rather than having to import all of the above
90
STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
@@ -97,6 +100,7 @@ STR_NOT_IMPL_ENC_DEC_ERR_STRS = {
100
"STR_NOT_IMPL_ENC_DEC_CUDA_GRAPH": STR_NOT_IMPL_ENC_DEC_CUDAGRAPH,
101
"STR_NOT_IMPL_ENC_DEC_BACKEND": STR_NOT_IMPL_ENC_DEC_BACKEND,
102
"STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER": STR_NOT_IMPL_ENC_DEC_PROMPT_ADAPTER,
103
+ "STR_NOT_IMPL_ENC_DEC_CPU": STR_NOT_IMPL_ENC_DEC_CPU
104
}
105
106
# Constants related to forcing the attention backend selection
vllm/worker/cpu_model_runner.py
+5
-1
@@ -15,7 +15,7 @@ from vllm.model_executor.model_loader import get_model
15
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16
MultiModalInputs)
17
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
18
-from vllm.utils import make_tensor_with_pad
18
+from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS, make_tensor_with_pad
19
from vllm.worker.model_runner_base import (
20
ModelRunnerBase, ModelRunnerInputBase,
21
_add_attn_metadata_broadcastable_dict,
@@ -121,6 +121,10 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
121
# Lazy initialization.
122
self.model: nn.Module # Set after init_Model
123
124
+ if self.model_config.is_encoder_decoder_model:
125
+ raise NotImplementedError(
126
+ STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_CPU'])
127
+
128
def load_model(self) -> None:
129
self.model = get_model(model_config=self.model_config,
130
load_config=self.load_config,