A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[bugfix] fix chatglm dummy_data_for_glmv (#9955)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 2, 2024 at 08:03 UTC
74b529ceeead8d4b44ded858f7c28bca9c1629ba
1 file changed
+7
-9
vllm/model_executor/models/chatglm.py
+7
-9
@@ -14,8 +14,8 @@ from torch.nn import LayerNorm
14
from vllm.attention import Attention, AttentionMetadata
15
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
17
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
18
- token_inputs)
17
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
18
+ InputContext, token_inputs)
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.activation import SiluAndMul
21
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -31,8 +31,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
31
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32
from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
-from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
35
- MultiModalInputs)
34
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
35
from vllm.multimodal.base import MultiModalData
36
from vllm.multimodal.utils import cached_get_tokenizer
37
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -117,16 +116,15 @@ def get_max_glmv_image_tokens(ctx: InputContext):
116
raise NotImplementedError(msg)
117
118
120
-def dummy_data_for_glmv(
121
- ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
122
-) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
119
+def dummy_data_for_glmv(ctx: InputContext, seq_len: int,
120
+ mm_counts: Mapping[str, int]) -> DummyData:
121
hf_config = ctx.get_hf_config(ChatGLMConfig)
122
vision_config = getattr(hf_config, 'vision_config', None)
123
124
if vision_config is None:
125
token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len)
126
seq_data = SequenceData(token_ids)
129
- return seq_data, None
127
+ return DummyData(seq_data, None)
128
elif isinstance(vision_config, dict):
129
image_size = vision_config["image_size"]
130
image_placeholder_length = calculate_image_placeholder(vision_config)
@@ -141,7 +139,7 @@ def dummy_data_for_glmv(
139
"image": Image.new("RGB", (image_size, image_size), color=0)
140
}
141
144
- return seq_data, mm_data
142
+ return DummyData(seq_data, mm_data)
143
144
msg = f"Unsupported vision config: {type(vision_config)}"
145
raise NotImplementedError(msg)