@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74b529ceee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[bugfix] fix chatglm dummy_data_for_glmv (#9955)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 2, 2024 at 08:03 UTC 74b529ceeead8d4b44ded858f7c28bca9c1629ba
1 file changed +7 -9
vllm/model_executor/models/chatglm.py
+7 -9
@@ -14,8 +14,8 @@ from torch.nn import LayerNorm
14 from vllm.attention import Attention, AttentionMetadata
15 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
17 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
18 - token_inputs)
17 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
18 + InputContext, token_inputs)
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.activation import SiluAndMul
21 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -31,8 +31,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
31 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32 from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 -from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
35 - MultiModalInputs)
34 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
35 from vllm.multimodal.base import MultiModalData
36 from vllm.multimodal.utils import cached_get_tokenizer
37 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
@@ -117,16 +116,15 @@ def get_max_glmv_image_tokens(ctx: InputContext):
116 raise NotImplementedError(msg)
117
118
120 -def dummy_data_for_glmv(
121 - ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
122 -) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
119 +def dummy_data_for_glmv(ctx: InputContext, seq_len: int,
120 + mm_counts: Mapping[str, int]) -> DummyData:
121 hf_config = ctx.get_hf_config(ChatGLMConfig)
122 vision_config = getattr(hf_config, 'vision_config', None)
123
124 if vision_config is None:
125 token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len)
126 seq_data = SequenceData(token_ids)
129 - return seq_data, None
127 + return DummyData(seq_data, None)
128 elif isinstance(vision_config, dict):
129 image_size = vision_config["image_size"]
130 image_placeholder_length = calculate_image_placeholder(vision_config)
@@ -141,7 +139,7 @@ def dummy_data_for_glmv(
139 "image": Image.new("RGB", (image_size, image_size), color=0)
140 }
141
144 - return seq_data, mm_data
142 + return DummyData(seq_data, mm_data)
143
144 msg = f"Unsupported vision config: {type(vision_config)}"
145 raise NotImplementedError(msg)