@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 836e8ef6ee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix] Fix PP for ChatGLM and Molmo (#9422)

Cyrus Leung committed Oct 24, 2024 at 14:12 UTC 836e8ef6eeafcd1e24b25c990da6331f48a95fd2
7 files changed +197 -124
docs/source/models/supported_models.rst
+1 -1
@@ -425,7 +425,7 @@ Text Generation
425 -
426 * - :code:`MolmoForCausalLM`
427 - Molmo
428 - - Image
428 + - T + I
429 - :code:`allenai/Molmo-7B-D-0924`, :code:`allenai/Molmo-72B-0924`, etc.
430 -
431 - ✅︎
tests/distributed/test_pipeline_parallel.py
+18 -19
@@ -118,11 +118,8 @@ class PPTestSettings:
118 # The values displayed here are only a rough indicator of the size of the model
119
120 # yapf: disable
121 -GENERATION_MODEL_SETTINGS = {
122 - # [DETAILED TESTS]
123 - "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
124 - "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
125 - # [FAST TESTS]
121 +TEXT_GENERATION_MODELS = {
122 + # [Decoder-only]
123 # Uses Llama
124 # "BAAI/AquilaChat-7B": PPTestSettings.fast(),
125 "Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(tp_base=8, trust_remote_code=True), # noqa: E501
@@ -151,6 +148,7 @@ GENERATION_MODEL_SETTINGS = {
148 "core42/jais-13b-chat": PPTestSettings.fast(),
149 # TODO: Implement PP
150 # "ai21labs/AI21-Jamba-1.5-Mini": PPTestSettings.fast(),
151 + "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
152 "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(trust_remote_code=True),
153 "openbmb/MiniCPM3-4B": PPTestSettings.fast(trust_remote_code=True),
154 # Uses Llama
@@ -163,6 +161,7 @@ GENERATION_MODEL_SETTINGS = {
161 "facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
162 "OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
163 "microsoft/phi-2": PPTestSettings.fast(),
164 + "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
165 "microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
166 "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
167 "adept/persimmon-8b-chat": PPTestSettings.fast(),
@@ -174,40 +173,40 @@ GENERATION_MODEL_SETTINGS = {
173 "upstage/solar-pro-preview-instruct": PPTestSettings.fast(tp_base=2),
174 # FIXME: Cannot load tokenizer in latest transformers version
175 # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
176 + # [Encoder-only]
177 + # TODO: Implement PP
178 + # "facebook/bart-base": PPTestSettings.fast(),
179 }
180
179 -EMBEDDING_MODEL_SETTINGS = { # type: ignore[var-annotated]
180 - # [FAST TESTS]
181 +EMBEDDING_MODELS = { # type: ignore[var-annotated]
182 + # [Text-only]
183 "intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
184 "BAAI/bge-multilingual-gemma2": PPTestSettings.fast(),
185 "Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(tp_base=4, trust_remote_code=True), # noqa: E501
186 }
187
186 -MULTIMODAL_MODEL_SETTINGS = {
187 - # [FAST TESTS]
188 +MULTIMODAL_MODELS = {
189 + # [Decoder-only]
190 "Salesforce/blip2-opt-2.7b": PPTestSettings.fast(),
191 "facebook/chameleon-7b": PPTestSettings.fast(),
192 "adept/fuyu-8b": PPTestSettings.fast(),
193 + "THUDM/glm-4v-9b": PPTestSettings.fast(trust_remote_code=True),
194 "OpenGVLab/InternVL2-1B": PPTestSettings.fast(trust_remote_code=True),
195 "llava-hf/llava-1.5-7b-hf": PPTestSettings.fast(),
196 "llava-hf/llava-v1.6-mistral-7b-hf": PPTestSettings.fast(),
197 "llava-hf/LLaVA-NeXT-Video-7B-hf": PPTestSettings.fast(),
198 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf": PPTestSettings.fast(),
199 "openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(trust_remote_code=True),
197 - # TODO: Implement PP
198 - # "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
200 + "allenai/Molmo-7B-D-0924": PPTestSettings.fast(trust_remote_code=True),
201 "microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
202 "mistralai/Pixtral-12B-2409": PPTestSettings.fast(tp_base=2, tokenizer_mode="mistral"), # noqa: E501
203 "Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
204 "Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
205 "Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
206 "fixie-ai/ultravox-v0_3": PPTestSettings.fast(),
205 -}
206 -
207 -CONDITIONAL_GENERATION_MODEL_SETTINGS = { # type: ignore[var-annotated]
208 - # [FAST TESTS]
207 + # [Encoder-decoder]
208 # TODO: Implement PP
210 - # "facebook/bart-base": PPTestSettings.fast(),
209 + # "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
210 }
211 # yapf: enable
212
@@ -323,7 +322,7 @@ def _compare_tp(
322 ("model_name", "parallel_setup", "distributed_backend", "task",
323 "test_options"),
324 [
326 - params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
325 + params for model_name, settings in TEXT_GENERATION_MODELS.items()
326 for params in settings.iter_params(model_name)
327 if model_name in TEST_MODELS
328 ],
@@ -350,7 +349,7 @@ def test_tp_language_generation(
349 ("model_name", "parallel_setup", "distributed_backend", "task",
350 "test_options"),
351 [
353 - params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
352 + params for model_name, settings in EMBEDDING_MODELS.items()
353 for params in settings.iter_params(model_name)
354 if model_name in TEST_MODELS
355 ],
@@ -377,7 +376,7 @@ def test_tp_language_embedding(
376 ("model_name", "parallel_setup", "distributed_backend", "task",
377 "test_options"),
378 [
380 - params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
379 + params for model_name, settings in MULTIMODAL_MODELS.items()
380 for params in settings.iter_params(model_name)
381 if model_name in TEST_MODELS
382 ],
vllm/model_executor/models/chatglm.py
+75 -54
@@ -13,8 +13,9 @@ from torch.nn import LayerNorm
13
14 from vllm.attention import Attention, AttentionMetadata
15 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16 -from vllm.distributed import get_tensor_model_parallel_world_size
17 -from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16 +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
17 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
18 + token_inputs)
19 from vllm.logger import init_logger
20 from vllm.model_executor.layers.activation import SiluAndMul
21 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -22,8 +23,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
23 QKVParallelLinear,
24 RowParallelLinear)
25 from vllm.model_executor.layers.logits_processor import LogitsProcessor
25 -from vllm.model_executor.layers.quantization.base_config import (
26 - QuantizationConfig)
26 +from vllm.model_executor.layers.quantization import QuantizationConfig
27 from vllm.model_executor.layers.rotary_embedding import get_rope
28 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
29 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -39,7 +39,9 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
39 SequenceData)
40 from vllm.transformers_utils.configs import ChatGLMConfig
41
42 -from .interfaces import SupportsLoRA, SupportsMultiModal
42 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
43 +from .utils import (is_pp_missing_parameter,
44 + make_empty_intermediate_tensors_factory, make_layers)
45
46 logger = init_logger(__name__)
47
@@ -150,6 +152,10 @@ def find_all_positions(input_ids: List[int], target: int) -> List[int]:
152
153
154 def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
155 + multi_modal_data = inputs.get("multi_modal_data")
156 + if multi_modal_data is None or "image" not in multi_modal_data:
157 + return inputs
158 +
159 hf_config = ctx.get_hf_config(ChatGLMConfig)
160 vision_config = getattr(hf_config, 'vision_config', None)
161
@@ -161,8 +167,8 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
167 msg = f"Unsupported vision config: {type(vision_config)}"
168 raise NotImplementedError(msg)
169
164 - input_ids = inputs.get("prompt_token_ids")
165 - position_ids = inputs.get("position_ids")
170 + input_ids = inputs["prompt_token_ids"]
171 +
172 tokenizer = cached_get_tokenizer(
173 ctx.model_config.model,
174 trust_remote_code=ctx.model_config.trust_remote_code)
@@ -171,20 +177,19 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
177 raw_batch_data = tokenizer.apply_chat_template(
178 conversation=[{
179 "role": "user",
174 - "image": inputs['multi_modal_data']["image"],
175 - "content": inputs['prompt']
180 + "image": multi_modal_data["image"],
181 + "content": inputs['prompt'],
182 }],
183 add_generation_prompt=True,
184 tokenize=True,
185 return_tensors="pt",
180 - return_dict=True).data
186 + return_dict=True,
187 + ).data
188 except Exception:
189 logger.error("Failed to process content (%s)", inputs['prompt'])
190 raise
191 input_ids = raw_batch_data['input_ids'][0].tolist()
192
186 - if position_ids is None:
187 - position_ids = list(range(len(input_ids)))
193 boi_token_id = hf_config.boi_token_id
194 eoi_token_id = hf_config.eoi_token_id
195 boi_positions = find_all_positions(input_ids, boi_token_id)
@@ -193,7 +198,6 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
198 assert len(boi_positions) == len(eoi_positions)
199
200 new_input_ids = []
196 - new_position_ids = []
201 final_processed_position = 0
202 final_processed_position = 0
203
@@ -201,29 +205,28 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
205 assert boi_position < eoi_position
206 new_input_ids.extend(input_ids[final_processed_position:boi_position +
207 1])
204 - new_position_ids.extend(
205 - list(range(final_processed_position, boi_position + 1)))
208 new_input_ids.extend([input_ids[boi_position + 1]] *
209 image_placeholder_length)
208 - new_position_ids.extend([boi_position + 1] * image_placeholder_length)
210 final_processed_position = eoi_position
211
212 new_input_ids.extend(input_ids[final_processed_position:])
212 - new_position_ids.extend(
213 - list(range(final_processed_position, len(input_ids))))
213
215 - assert len(new_input_ids) == len(new_position_ids)
214 + prompt = inputs.get("prompt")
215 + if prompt is None:
216 + prompt = tokenizer.decode(new_input_ids)
217
217 - inputs["prompt_token_ids"] = new_input_ids
218 - inputs["position_ids"] = new_position_ids
219 - return inputs
218 + return token_inputs(
219 + prompt_token_ids=new_input_ids,
220 + prompt=prompt,
221 + multi_modal_data=multi_modal_data,
222 + )
223
224
225 class GLMAttention(nn.Module):
226
227 def __init__(
228 self,
226 - config,
229 + config: ChatGLMConfig,
230 cache_config: Optional[CacheConfig] = None,
231 quant_config: Optional[QuantizationConfig] = None,
232 ):
@@ -314,7 +317,7 @@ class GLMMLP(nn.Module):
317
318 def __init__(
319 self,
317 - config,
320 + config: ChatGLMConfig,
321 quant_config: Optional[QuantizationConfig] = None,
322 ):
323 super().__init__()
@@ -357,7 +360,7 @@ class GLMBlock(nn.Module):
360
361 def __init__(
362 self,
360 - config,
363 + config: ChatGLMConfig,
364 cache_config: Optional[CacheConfig] = None,
365 quant_config: Optional[QuantizationConfig] = None,
366 ):
@@ -428,9 +431,10 @@ class GLMTransformer(nn.Module):
431
432 def __init__(
433 self,
431 - config,
434 + config: ChatGLMConfig,
435 cache_config: Optional[CacheConfig] = None,
436 quant_config: Optional[QuantizationConfig] = None,
437 + prefix: str = "",
438 ):
439 super().__init__()
440 self.post_layer_norm = config.post_layer_norm
@@ -439,10 +443,11 @@ class GLMTransformer(nn.Module):
443 self.num_layers = config.num_layers
444
445 # Transformer layers.
442 - self.layers = nn.ModuleList([
443 - GLMBlock(config, cache_config, quant_config)
444 - for i in range(self.num_layers)
445 - ])
446 + self.start_layer, self.end_layer, self.layers = make_layers(
447 + self.num_layers,
448 + lambda prefix: GLMBlock(config, cache_config, quant_config),
449 + prefix=f"{prefix}.layers",
450 + )
451
452 if self.post_layer_norm:
453 layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -450,6 +455,10 @@ class GLMTransformer(nn.Module):
455 self.final_layernorm = layer_norm_func(
456 config.hidden_size, eps=config.layernorm_epsilon)
457
458 + self.make_empty_intermediate_tensors = (
459 + make_empty_intermediate_tensors_factory(["hidden_states"],
460 + config.hidden_size))
461 +
462 def forward(
463 self,
464 hidden_states: torch.Tensor,
@@ -457,16 +466,16 @@ class GLMTransformer(nn.Module):
466 kv_caches: List[torch.Tensor],
467 attn_metadata: AttentionMetadata,
468 ) -> torch.Tensor:
460 - for i in range(self.num_layers):
469 + for i in range(self.start_layer, self.end_layer):
470 layer = self.layers[i]
471 hidden_states = layer(
472 hidden_states=hidden_states,
473 position_ids=position_ids,
465 - kv_cache=kv_caches[i],
474 + kv_cache=kv_caches[i - self.start_layer],
475 attn_metadata=attn_metadata,
476 )
477 # Final layer norm.
469 - if self.post_layer_norm:
478 + if get_pp_group().is_last_rank and self.post_layer_norm:
479 hidden_states = self.final_layernorm(hidden_states)
480
481 return hidden_states
@@ -476,7 +485,7 @@ class ChatGLMModel(nn.Module):
485
486 def __init__(
487 self,
479 - config,
488 + config: ChatGLMConfig,
489 cache_config: Optional[CacheConfig] = None,
490 quant_config: Optional[QuantizationConfig] = None,
491 ):
@@ -504,6 +513,9 @@ class ChatGLMModel(nn.Module):
513 else:
514 self.vision = None
515
516 + self.make_empty_intermediate_tensors = (
517 + self.encoder.make_empty_intermediate_tensors)
518 +
519 def _parse_and_validate_image_input(
520 self, **kwargs: object) -> GLMImagePixelInputs:
521
@@ -529,24 +541,26 @@ class ChatGLMModel(nn.Module):
541 intermediate_tensors: Optional[IntermediateTensors] = None,
542 **kwargs: object,
543 ) -> torch.Tensor:
532 -
533 - inputs_embeds = self.embedding(input_ids)
534 - image_input = self._parse_and_validate_image_input(**kwargs)
535 -
536 - if image_input["pixel_values"] is not None:
537 - pixel_values = image_input["pixel_values"].to(
538 - dtype=inputs_embeds.dtype)
539 - image_embeds = self.vision(pixel_values)
540 -
541 - boi_token_id = self.config.boi_token_id
542 - eoi_token_id = self.config.eoi_token_id
543 -
544 - inputs_embeds = merge_glm_vision_embeddings(
545 - input_ids=input_ids,
546 - inputs_embeds=inputs_embeds,
547 - vision_embeddings=image_embeds,
548 - boi_token_id=boi_token_id,
549 - eoi_token_id=eoi_token_id)
544 + if intermediate_tensors is None:
545 + inputs_embeds = self.embedding(input_ids)
546 + image_input = self._parse_and_validate_image_input(**kwargs)
547 +
548 + if image_input["pixel_values"] is not None:
549 + pixel_values = image_input["pixel_values"].to(
550 + dtype=inputs_embeds.dtype)
551 + image_embeds = self.vision(pixel_values)
552 +
553 + boi_token_id = self.config.boi_token_id
554 + eoi_token_id = self.config.eoi_token_id
555 +
556 + inputs_embeds = merge_glm_vision_embeddings(
557 + input_ids=input_ids,
558 + inputs_embeds=inputs_embeds,
559 + vision_embeddings=image_embeds,
560 + boi_token_id=boi_token_id,
561 + eoi_token_id=eoi_token_id)
562 + else:
563 + inputs_embeds = intermediate_tensors["hidden_states"]
564
565 # Run encoder.
566 hidden_states = self.encoder(
@@ -555,6 +569,9 @@ class ChatGLMModel(nn.Module):
569 kv_caches=kv_caches,
570 attn_metadata=attn_metadata,
571 )
572 +
573 + if not get_pp_group().is_last_rank:
574 + return IntermediateTensors({"hidden_states": hidden_states})
575 return hidden_states
576
577
@@ -562,7 +579,8 @@ class ChatGLMModel(nn.Module):
579 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
580 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
581 @INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
565 -class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
582 +class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
583 + SupportsMultiModal):
584 packed_modules_mapping = {
585 "query_key_value": ["query_key_value"],
586 "dense_h_to_4h": ["dense_h_to_4h"]
@@ -610,7 +628,8 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
628 intermediate_tensors: Optional[IntermediateTensors] = None,
629 **kwargs) -> torch.Tensor:
630 hidden_states = self.transformer(input_ids, positions, kv_caches,
613 - attn_metadata, **kwargs)
631 + attn_metadata, intermediate_tensors,
632 + **kwargs)
633 return hidden_states
634
635 def compute_logits(
@@ -656,6 +675,8 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
675 # Skip loading extra bias for GPTQ models.
676 if name.endswith(".bias") and name not in params_dict:
677 continue
678 + if is_pp_missing_parameter(name, self):
679 + continue
680 param = params_dict[name]
681 weight_loader = getattr(param, "weight_loader",
682 default_weight_loader)
vllm/model_executor/models/molmo.py
+46 -27
@@ -30,21 +30,21 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 QKVParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 -from vllm.model_executor.layers.quantization.base_config import (
34 - QuantizationConfig)
33 +from vllm.model_executor.layers.quantization import QuantizationConfig
34 from vllm.model_executor.layers.rotary_embedding import get_rope
35 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36 from vllm.model_executor.layers.vocab_parallel_embedding import (
37 ParallelLMHead, VocabParallelEmbedding)
38 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 -from vllm.model_executor.models.interfaces import SupportsMultiModal
41 -from vllm.model_executor.models.utils import make_layers
39 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
40 +from vllm.multimodal.utils import cached_get_tokenizer
41 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 SequenceData)
43 from vllm.transformers_utils.processor import get_processor
44
47 -from .utils import get_vit_attn_backend
45 +from .interfaces import SupportsMultiModal, SupportsPP
46 +from .utils import (get_vit_attn_backend,
47 + make_empty_intermediate_tensors_factory, make_layers)
48
49 # TODO: hard-coded for now. Consider making it configurable.
50 VIT_LAYERS = [-2, -9]
@@ -744,6 +744,10 @@ class MolmoModel(nn.Module):
744 assert config.layer_norm_type == "rms"
745 self.norm = RMSNorm(config.hidden_size, config.layer_norm_eps)
746
747 + self.make_empty_intermediate_tensors = (
748 + make_empty_intermediate_tensors_factory(
749 + ["hidden_states", "residual"], config.hidden_size))
750 +
751 def forward(
752 self,
753 input_ids: torch.Tensor,
@@ -925,16 +929,19 @@ def pad_images(
929
930
931 def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
928 - prompt = inputs.get("prompt", None)
929 - multi_modal_data = inputs.get("multi_modal_data", None)
930 - if multi_modal_data is not None:
931 - image = multi_modal_data.get("image", None)
932 - else:
933 - image = None
932 + prompt = inputs.get("prompt")
933 + multi_modal_data = inputs.get("multi_modal_data")
934 + image = None if multi_modal_data is None else multi_modal_data.get("image")
935 +
936 processor = cached_get_processor(ctx.model_config.model,
937 trust_remote_code=True,
938 revision=ctx.model_config.code_revision)
939
940 + model_config = ctx.model_config
941 + tokenizer = cached_get_tokenizer(
942 + model_config.tokenizer,
943 + trust_remote_code=model_config.trust_remote_code)
944 +
945 # NOTE: message formatting for raw text prompt is only applied for
946 # offline inference; for online inference, the prompt is always in
947 # instruction format and tokenized.
@@ -997,9 +1004,13 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1004
1005 multi_modal_data = dict(image=image_data)
1006
1007 + prompt = inputs.get("prompt")
1008 + if prompt is None:
1009 + prompt = tokenizer.decode(out["input_ids"])
1010 +
1011 return token_inputs(
1012 prompt_token_ids=out["input_ids"],
1002 - prompt=inputs["prompt"],
1013 + prompt=prompt,
1014 multi_modal_data=multi_modal_data,
1015 )
1016
@@ -1008,7 +1019,7 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1019 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1020 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1021 @INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1011 -class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1022 +class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1023
1024 def __init__(
1025 self,
@@ -1040,6 +1051,9 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1051 or config.vocab_size)
1052 self.sampler = Sampler()
1053
1054 + self.make_empty_intermediate_tensors = (
1055 + self.model.make_empty_intermediate_tensors)
1056 +
1057 def _parse_and_validate_image_input(
1058 self,
1059 **kwargs: object,
@@ -1123,31 +1137,36 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1137 positions: torch.LongTensor,
1138 kv_caches: List[torch.Tensor],
1139 attn_metadata: AttentionMetadata,
1140 + intermediate_tensors: Optional[IntermediateTensors] = None,
1141 **kwargs: object,
1142 ) -> SamplerOutput:
1143 + if intermediate_tensors is not None:
1144 + input_ids = None
1145 + inputs_embeds = None
1146 + else:
1147 + image_input = self._parse_and_validate_image_input(**kwargs)
1148
1129 - image_input = self._parse_and_validate_image_input(**kwargs)
1130 -
1131 - if image_input is not None:
1132 - inputs_embeds = self.model.embed_tokens(input_ids)
1133 - image_features = self._process_image_input(image_input)
1149 + if image_input is not None:
1150 + inputs_embeds = self.model.embed_tokens(input_ids)
1151 + image_features = self._process_image_input(image_input)
1152
1135 - inputs_embeds = self._merge_multimodal_embeddings(
1136 - inputs_embeds,
1137 - image_features,
1138 - image_input["image_input_idx"],
1139 - image_input["seq_len"],
1140 - )
1153 + inputs_embeds = self._merge_multimodal_embeddings(
1154 + inputs_embeds,
1155 + image_features,
1156 + image_input["image_input_idx"],
1157 + image_input["seq_len"],
1158 + )
1159
1142 - input_ids = None
1143 - else:
1144 - inputs_embeds = None
1160 + input_ids = None
1161 + else:
1162 + inputs_embeds = None
1163
1164 hidden_states = self.model(
1165 input_ids=input_ids,
1166 positions=positions,
1167 kv_caches=kv_caches,
1168 attn_metadata=attn_metadata,
1169 + intermediate_tensors=intermediate_tensors,
1170 inputs_embeds=inputs_embeds,
1171 )
1172
vllm/model_executor/models/qwen2_rm.py
+2 -1
@@ -119,5 +119,6 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
119 return self._pooler(hidden_states, pooling_metadata)
120
121 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
122 - loader = AutoWeightsLoader(self)
122 + loader = AutoWeightsLoader(self,
123 + ignore_unexpected_prefixes=["lm_head."])
124 loader.load_weights(weights)
vllm/model_executor/models/qwen2_vl.py
+13 -10
@@ -61,6 +61,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
61 MultiModalInputs)
62 from vllm.multimodal.base import MultiModalData
63 from vllm.multimodal.image import cached_get_image_processor
64 +from vllm.multimodal.utils import cached_get_tokenizer
65 from vllm.sequence import IntermediateTensors, SequenceData
66 from vllm.transformers_utils.config import uses_mrope
67 from vllm.transformers_utils.processor import cached_get_processor
@@ -817,7 +818,7 @@ def input_processor_for_qwen2_vl(
818 min_pixels: Optional[int] = None,
819 max_pixels: Optional[int] = None,
820 ) -> DecoderOnlyInputs:
820 - multi_modal_data = inputs.get("multi_modal_data", None)
821 + multi_modal_data = inputs.get("multi_modal_data")
822 if multi_modal_data is None:
823 return inputs
824
@@ -830,6 +831,7 @@ def input_processor_for_qwen2_vl(
831 min_pixels = min_pixels if min_pixels else image_processor.min_pixels
832 max_pixels = max_pixels if max_pixels else image_processor.max_pixels
833
834 + model_config = ctx.model_config
835 hf_config = ctx.get_hf_config(Qwen2VLConfig)
836
837 # To avoid redundant processing of vision objects (resize, rescale, etc.),
@@ -845,14 +847,11 @@ def input_processor_for_qwen2_vl(
847 # return_tensors="pt")
848 # prompt_token_ids = inputs["input_ids"][0].tolist()
849
848 - prompt_token_ids = inputs.get("prompt_token_ids", None)
849 - if prompt_token_ids is None:
850 - prompt = inputs["prompt"]
851 - prompt_token_ids = processor.tokenizer(
852 - prompt,
853 - padding=True,
854 - return_tensors=None,
855 - )["input_ids"]
850 + tokenizer = cached_get_tokenizer(
851 + model_config.tokenizer,
852 + trust_remote_code=model_config.trust_remote_code)
853 +
854 + prompt_token_ids = inputs["prompt_token_ids"]
855
856 # Expand image pad tokens.
857
@@ -894,9 +893,13 @@ def input_processor_for_qwen2_vl(
893 min_pixels=min_pixels,
894 max_pixels=max_pixels)
895
896 + prompt = inputs.get("prompt")
897 + if prompt is None:
898 + prompt = tokenizer.decode(prompt_token_ids)
899 +
900 return token_inputs(
901 prompt_token_ids=prompt_token_ids,
899 - prompt=inputs["prompt"],
902 + prompt=prompt,
903 multi_modal_data=multi_modal_data,
904 )
905
vllm/model_executor/models/utils.py
+42 -12
@@ -79,6 +79,9 @@ class AutoWeightsLoader:
79
80 Similarly, the weight loading logic for individual parameters can be
81 overridden by defining a ``weight_loader`` method.
82 +
83 + Detailed weight loading information can be viewed by setting the
84 + environment variable ``VLLM_LOGGING_LEVEL=DEBUG``.
85 """
86
87 def __init__(
@@ -136,20 +139,27 @@ class AutoWeightsLoader:
139 weight_qualname = self._get_qualname(base_prefix, weight_name)
140
141 if self._can_skip(weight_qualname):
142 + logger.debug("Skipping weight %s", weight_qualname)
143 +
144 continue
145
146 if weight_name != "":
142 - if not self._can_ignore_unexpected(weight_qualname):
143 - raise ValueError(
144 - f"Attempted to load nested weight '{weight_qualname}' "
145 - f"into a single parameter '{base_prefix}'")
147 + if self._can_ignore_unexpected(weight_qualname):
148 + logger.debug("Ignoring weight %s", weight_qualname)
149
147 - continue
150 + continue
151 +
152 + raise ValueError(
153 + f"Attempted to load nested weight '{weight_qualname}' "
154 + f"into a single parameter '{base_prefix}'")
155
156 weight_loader = getattr(param, "weight_loader",
157 default_weight_loader)
158 weight_loader(param, weight_data)
159
160 + logger.debug("Loaded weight %s with shape %s", weight_qualname,
161 + param.shape)
162 +
163 yield weight_qualname
164
165 def _load_module(
@@ -175,21 +185,41 @@ class AutoWeightsLoader:
185 for child_prefix, child_weights in self._groupby_prefix(weights):
186 prefix = self._get_qualname(base_prefix, child_prefix)
187
178 - if self._can_skip(prefix):
179 - continue
180 -
188 if child_prefix in child_modules:
189 + if self._can_skip(prefix + "."):
190 + logger.debug("Skipping module %s", prefix)
191 +
192 + continue
193 +
194 yield from self._load_module(prefix,
195 child_modules[child_prefix],
196 child_weights)
197 elif child_prefix in child_params:
198 + if self._can_skip(prefix):
199 + logger.debug("Skipping param %s", prefix)
200 +
201 + continue
202 +
203 yield from self._load_param(prefix, child_params[child_prefix],
204 child_weights)
205 else:
189 - if not self._can_ignore_unexpected(prefix):
190 - msg = (f"There is no module or parameter named '{prefix}' "
191 - f"in {type(self.module).__name__}")
192 - raise ValueError(msg)
206 + can_skip_module = self._can_skip(prefix + ".")
207 + can_skip_param = self._can_skip(prefix)
208 + if can_skip_module or can_skip_param:
209 + logger.debug("Skipping missing %s", prefix)
210 +
211 + continue
212 +
213 + can_ignore_module = self._can_ignore_unexpected(prefix + ".")
214 + can_ignore_param = self._can_ignore_unexpected(prefix)
215 + if can_ignore_module or can_ignore_param:
216 + logger.debug("Ignoring missing %s", prefix)
217 +
218 + continue
219 +
220 + msg = (f"There is no module or parameter named '{prefix}' "
221 + f"in {type(self.module).__name__}")
222 + raise ValueError(msg)
223
224 def load_weights(
225 self,