A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix] Fix PP for ChatGLM and Molmo (#9422)
Cyrus Leung committed
Oct 24, 2024 at 14:12 UTC
836e8ef6eeafcd1e24b25c990da6331f48a95fd2
7 files changed
+197
-124
docs/source/models/supported_models.rst
+1
-1
@@ -425,7 +425,7 @@ Text Generation
425
-
426
* - :code:`MolmoForCausalLM`
427
- Molmo
428
- - Image
428
+ - T + I
429
- :code:`allenai/Molmo-7B-D-0924`, :code:`allenai/Molmo-72B-0924`, etc.
430
-
431
- ✅︎
tests/distributed/test_pipeline_parallel.py
+18
-19
@@ -118,11 +118,8 @@ class PPTestSettings:
118
# The values displayed here are only a rough indicator of the size of the model
119
120
# yapf: disable
121
-GENERATION_MODEL_SETTINGS = {
122
- # [DETAILED TESTS]
123
- "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
124
- "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
125
- # [FAST TESTS]
121
+TEXT_GENERATION_MODELS = {
122
+ # [Decoder-only]
123
# Uses Llama
124
# "BAAI/AquilaChat-7B": PPTestSettings.fast(),
125
"Snowflake/snowflake-arctic-instruct": PPTestSettings.fast(tp_base=8, trust_remote_code=True), # noqa: E501
@@ -151,6 +148,7 @@ GENERATION_MODEL_SETTINGS = {
148
"core42/jais-13b-chat": PPTestSettings.fast(),
149
# TODO: Implement PP
150
# "ai21labs/AI21-Jamba-1.5-Mini": PPTestSettings.fast(),
151
+ "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
152
"openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(trust_remote_code=True),
153
"openbmb/MiniCPM3-4B": PPTestSettings.fast(trust_remote_code=True),
154
# Uses Llama
@@ -163,6 +161,7 @@ GENERATION_MODEL_SETTINGS = {
161
"facebook/opt-iml-max-1.3b": PPTestSettings.fast(),
162
"OrionStarAI/Orion-14B-Chat": PPTestSettings.fast(trust_remote_code=True),
163
"microsoft/phi-2": PPTestSettings.fast(),
164
+ "microsoft/Phi-3-mini-4k-instruct": PPTestSettings.detailed(trust_remote_code=True, multi_node_only=True), # noqa: E501
165
"microsoft/Phi-3-small-8k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
166
"microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
167
"adept/persimmon-8b-chat": PPTestSettings.fast(),
@@ -174,40 +173,40 @@ GENERATION_MODEL_SETTINGS = {
173
"upstage/solar-pro-preview-instruct": PPTestSettings.fast(tp_base=2),
174
# FIXME: Cannot load tokenizer in latest transformers version
175
# "xverse/XVERSE-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
176
+ # [Encoder-only]
177
+ # TODO: Implement PP
178
+ # "facebook/bart-base": PPTestSettings.fast(),
179
}
180
179
-EMBEDDING_MODEL_SETTINGS = { # type: ignore[var-annotated]
180
- # [FAST TESTS]
181
+EMBEDDING_MODELS = { # type: ignore[var-annotated]
182
+ # [Text-only]
183
"intfloat/e5-mistral-7b-instruct": PPTestSettings.fast(),
184
"BAAI/bge-multilingual-gemma2": PPTestSettings.fast(),
185
"Qwen/Qwen2.5-Math-RM-72B": PPTestSettings.fast(tp_base=4, trust_remote_code=True), # noqa: E501
186
}
187
186
-MULTIMODAL_MODEL_SETTINGS = {
187
- # [FAST TESTS]
188
+MULTIMODAL_MODELS = {
189
+ # [Decoder-only]
190
"Salesforce/blip2-opt-2.7b": PPTestSettings.fast(),
191
"facebook/chameleon-7b": PPTestSettings.fast(),
192
"adept/fuyu-8b": PPTestSettings.fast(),
193
+ "THUDM/glm-4v-9b": PPTestSettings.fast(trust_remote_code=True),
194
"OpenGVLab/InternVL2-1B": PPTestSettings.fast(trust_remote_code=True),
195
"llava-hf/llava-1.5-7b-hf": PPTestSettings.fast(),
196
"llava-hf/llava-v1.6-mistral-7b-hf": PPTestSettings.fast(),
197
"llava-hf/LLaVA-NeXT-Video-7B-hf": PPTestSettings.fast(),
198
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf": PPTestSettings.fast(),
199
"openbmb/MiniCPM-Llama3-V-2_5": PPTestSettings.fast(trust_remote_code=True),
197
- # TODO: Implement PP
198
- # "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
200
+ "allenai/Molmo-7B-D-0924": PPTestSettings.fast(trust_remote_code=True),
201
"microsoft/Phi-3-vision-128k-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
202
"mistralai/Pixtral-12B-2409": PPTestSettings.fast(tp_base=2, tokenizer_mode="mistral"), # noqa: E501
203
"Qwen/Qwen-VL-Chat": PPTestSettings.fast(trust_remote_code=True),
204
"Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
205
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
206
"fixie-ai/ultravox-v0_3": PPTestSettings.fast(),
205
-}
206
-
207
-CONDITIONAL_GENERATION_MODEL_SETTINGS = { # type: ignore[var-annotated]
208
- # [FAST TESTS]
207
+ # [Encoder-decoder]
208
# TODO: Implement PP
210
- # "facebook/bart-base": PPTestSettings.fast(),
209
+ # "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
210
}
211
# yapf: enable
212
@@ -323,7 +322,7 @@ def _compare_tp(
322
("model_name", "parallel_setup", "distributed_backend", "task",
323
"test_options"),
324
[
326
- params for model_name, settings in GENERATION_MODEL_SETTINGS.items()
325
+ params for model_name, settings in TEXT_GENERATION_MODELS.items()
326
for params in settings.iter_params(model_name)
327
if model_name in TEST_MODELS
328
],
@@ -350,7 +349,7 @@ def test_tp_language_generation(
349
("model_name", "parallel_setup", "distributed_backend", "task",
350
"test_options"),
351
[
353
- params for model_name, settings in EMBEDDING_MODEL_SETTINGS.items()
352
+ params for model_name, settings in EMBEDDING_MODELS.items()
353
for params in settings.iter_params(model_name)
354
if model_name in TEST_MODELS
355
],
@@ -377,7 +376,7 @@ def test_tp_language_embedding(
376
("model_name", "parallel_setup", "distributed_backend", "task",
377
"test_options"),
378
[
380
- params for model_name, settings in MULTIMODAL_MODEL_SETTINGS.items()
379
+ params for model_name, settings in MULTIMODAL_MODELS.items()
380
for params in settings.iter_params(model_name)
381
if model_name in TEST_MODELS
382
],
vllm/model_executor/models/chatglm.py
+75
-54
@@ -13,8 +13,9 @@ from torch.nn import LayerNorm
13
14
from vllm.attention import Attention, AttentionMetadata
15
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16
-from vllm.distributed import get_tensor_model_parallel_world_size
17
-from vllm.inputs import INPUT_REGISTRY, DecoderOnlyInputs, InputContext
16
+from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
17
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
18
+ token_inputs)
19
from vllm.logger import init_logger
20
from vllm.model_executor.layers.activation import SiluAndMul
21
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -22,8 +23,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
23
QKVParallelLinear,
24
RowParallelLinear)
25
from vllm.model_executor.layers.logits_processor import LogitsProcessor
25
-from vllm.model_executor.layers.quantization.base_config import (
26
- QuantizationConfig)
26
+from vllm.model_executor.layers.quantization import QuantizationConfig
27
from vllm.model_executor.layers.rotary_embedding import get_rope
28
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
29
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -39,7 +39,9 @@ from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
39
SequenceData)
40
from vllm.transformers_utils.configs import ChatGLMConfig
41
42
-from .interfaces import SupportsLoRA, SupportsMultiModal
42
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
43
+from .utils import (is_pp_missing_parameter,
44
+ make_empty_intermediate_tensors_factory, make_layers)
45
46
logger = init_logger(__name__)
47
@@ -150,6 +152,10 @@ def find_all_positions(input_ids: List[int], target: int) -> List[int]:
152
153
154
def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
155
+ multi_modal_data = inputs.get("multi_modal_data")
156
+ if multi_modal_data is None or "image" not in multi_modal_data:
157
+ return inputs
158
+
159
hf_config = ctx.get_hf_config(ChatGLMConfig)
160
vision_config = getattr(hf_config, 'vision_config', None)
161
@@ -161,8 +167,8 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
167
msg = f"Unsupported vision config: {type(vision_config)}"
168
raise NotImplementedError(msg)
169
164
- input_ids = inputs.get("prompt_token_ids")
165
- position_ids = inputs.get("position_ids")
170
+ input_ids = inputs["prompt_token_ids"]
171
+
172
tokenizer = cached_get_tokenizer(
173
ctx.model_config.model,
174
trust_remote_code=ctx.model_config.trust_remote_code)
@@ -171,20 +177,19 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
177
raw_batch_data = tokenizer.apply_chat_template(
178
conversation=[{
179
"role": "user",
174
- "image": inputs['multi_modal_data']["image"],
175
- "content": inputs['prompt']
180
+ "image": multi_modal_data["image"],
181
+ "content": inputs['prompt'],
182
}],
183
add_generation_prompt=True,
184
tokenize=True,
185
return_tensors="pt",
180
- return_dict=True).data
186
+ return_dict=True,
187
+ ).data
188
except Exception:
189
logger.error("Failed to process content (%s)", inputs['prompt'])
190
raise
191
input_ids = raw_batch_data['input_ids'][0].tolist()
192
186
- if position_ids is None:
187
- position_ids = list(range(len(input_ids)))
193
boi_token_id = hf_config.boi_token_id
194
eoi_token_id = hf_config.eoi_token_id
195
boi_positions = find_all_positions(input_ids, boi_token_id)
@@ -193,7 +198,6 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
198
assert len(boi_positions) == len(eoi_positions)
199
200
new_input_ids = []
196
- new_position_ids = []
201
final_processed_position = 0
202
final_processed_position = 0
203
@@ -201,29 +205,28 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
205
assert boi_position < eoi_position
206
new_input_ids.extend(input_ids[final_processed_position:boi_position +
207
1])
204
- new_position_ids.extend(
205
- list(range(final_processed_position, boi_position + 1)))
208
new_input_ids.extend([input_ids[boi_position + 1]] *
209
image_placeholder_length)
208
- new_position_ids.extend([boi_position + 1] * image_placeholder_length)
210
final_processed_position = eoi_position
211
212
new_input_ids.extend(input_ids[final_processed_position:])
212
- new_position_ids.extend(
213
- list(range(final_processed_position, len(input_ids))))
213
215
- assert len(new_input_ids) == len(new_position_ids)
214
+ prompt = inputs.get("prompt")
215
+ if prompt is None:
216
+ prompt = tokenizer.decode(new_input_ids)
217
217
- inputs["prompt_token_ids"] = new_input_ids
218
- inputs["position_ids"] = new_position_ids
219
- return inputs
218
+ return token_inputs(
219
+ prompt_token_ids=new_input_ids,
220
+ prompt=prompt,
221
+ multi_modal_data=multi_modal_data,
222
+ )
223
224
225
class GLMAttention(nn.Module):
226
227
def __init__(
228
self,
226
- config,
229
+ config: ChatGLMConfig,
230
cache_config: Optional[CacheConfig] = None,
231
quant_config: Optional[QuantizationConfig] = None,
232
):
@@ -314,7 +317,7 @@ class GLMMLP(nn.Module):
317
318
def __init__(
319
self,
317
- config,
320
+ config: ChatGLMConfig,
321
quant_config: Optional[QuantizationConfig] = None,
322
):
323
super().__init__()
@@ -357,7 +360,7 @@ class GLMBlock(nn.Module):
360
361
def __init__(
362
self,
360
- config,
363
+ config: ChatGLMConfig,
364
cache_config: Optional[CacheConfig] = None,
365
quant_config: Optional[QuantizationConfig] = None,
366
):
@@ -428,9 +431,10 @@ class GLMTransformer(nn.Module):
431
432
def __init__(
433
self,
431
- config,
434
+ config: ChatGLMConfig,
435
cache_config: Optional[CacheConfig] = None,
436
quant_config: Optional[QuantizationConfig] = None,
437
+ prefix: str = "",
438
):
439
super().__init__()
440
self.post_layer_norm = config.post_layer_norm
@@ -439,10 +443,11 @@ class GLMTransformer(nn.Module):
443
self.num_layers = config.num_layers
444
445
# Transformer layers.
442
- self.layers = nn.ModuleList([
443
- GLMBlock(config, cache_config, quant_config)
444
- for i in range(self.num_layers)
445
- ])
446
+ self.start_layer, self.end_layer, self.layers = make_layers(
447
+ self.num_layers,
448
+ lambda prefix: GLMBlock(config, cache_config, quant_config),
449
+ prefix=f"{prefix}.layers",
450
+ )
451
452
if self.post_layer_norm:
453
layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -450,6 +455,10 @@ class GLMTransformer(nn.Module):
455
self.final_layernorm = layer_norm_func(
456
config.hidden_size, eps=config.layernorm_epsilon)
457
458
+ self.make_empty_intermediate_tensors = (
459
+ make_empty_intermediate_tensors_factory(["hidden_states"],
460
+ config.hidden_size))
461
+
462
def forward(
463
self,
464
hidden_states: torch.Tensor,
@@ -457,16 +466,16 @@ class GLMTransformer(nn.Module):
466
kv_caches: List[torch.Tensor],
467
attn_metadata: AttentionMetadata,
468
) -> torch.Tensor:
460
- for i in range(self.num_layers):
469
+ for i in range(self.start_layer, self.end_layer):
470
layer = self.layers[i]
471
hidden_states = layer(
472
hidden_states=hidden_states,
473
position_ids=position_ids,
465
- kv_cache=kv_caches[i],
474
+ kv_cache=kv_caches[i - self.start_layer],
475
attn_metadata=attn_metadata,
476
)
477
# Final layer norm.
469
- if self.post_layer_norm:
478
+ if get_pp_group().is_last_rank and self.post_layer_norm:
479
hidden_states = self.final_layernorm(hidden_states)
480
481
return hidden_states
@@ -476,7 +485,7 @@ class ChatGLMModel(nn.Module):
485
486
def __init__(
487
self,
479
- config,
488
+ config: ChatGLMConfig,
489
cache_config: Optional[CacheConfig] = None,
490
quant_config: Optional[QuantizationConfig] = None,
491
):
@@ -504,6 +513,9 @@ class ChatGLMModel(nn.Module):
513
else:
514
self.vision = None
515
516
+ self.make_empty_intermediate_tensors = (
517
+ self.encoder.make_empty_intermediate_tensors)
518
+
519
def _parse_and_validate_image_input(
520
self, **kwargs: object) -> GLMImagePixelInputs:
521
@@ -529,24 +541,26 @@ class ChatGLMModel(nn.Module):
541
intermediate_tensors: Optional[IntermediateTensors] = None,
542
**kwargs: object,
543
) -> torch.Tensor:
532
-
533
- inputs_embeds = self.embedding(input_ids)
534
- image_input = self._parse_and_validate_image_input(**kwargs)
535
-
536
- if image_input["pixel_values"] is not None:
537
- pixel_values = image_input["pixel_values"].to(
538
- dtype=inputs_embeds.dtype)
539
- image_embeds = self.vision(pixel_values)
540
-
541
- boi_token_id = self.config.boi_token_id
542
- eoi_token_id = self.config.eoi_token_id
543
-
544
- inputs_embeds = merge_glm_vision_embeddings(
545
- input_ids=input_ids,
546
- inputs_embeds=inputs_embeds,
547
- vision_embeddings=image_embeds,
548
- boi_token_id=boi_token_id,
549
- eoi_token_id=eoi_token_id)
544
+ if intermediate_tensors is None:
545
+ inputs_embeds = self.embedding(input_ids)
546
+ image_input = self._parse_and_validate_image_input(**kwargs)
547
+
548
+ if image_input["pixel_values"] is not None:
549
+ pixel_values = image_input["pixel_values"].to(
550
+ dtype=inputs_embeds.dtype)
551
+ image_embeds = self.vision(pixel_values)
552
+
553
+ boi_token_id = self.config.boi_token_id
554
+ eoi_token_id = self.config.eoi_token_id
555
+
556
+ inputs_embeds = merge_glm_vision_embeddings(
557
+ input_ids=input_ids,
558
+ inputs_embeds=inputs_embeds,
559
+ vision_embeddings=image_embeds,
560
+ boi_token_id=boi_token_id,
561
+ eoi_token_id=eoi_token_id)
562
+ else:
563
+ inputs_embeds = intermediate_tensors["hidden_states"]
564
565
# Run encoder.
566
hidden_states = self.encoder(
@@ -555,6 +569,9 @@ class ChatGLMModel(nn.Module):
569
kv_caches=kv_caches,
570
attn_metadata=attn_metadata,
571
)
572
+
573
+ if not get_pp_group().is_last_rank:
574
+ return IntermediateTensors({"hidden_states": hidden_states})
575
return hidden_states
576
577
@@ -562,7 +579,8 @@ class ChatGLMModel(nn.Module):
579
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
580
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
581
@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
565
-class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
582
+class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP,
583
+ SupportsMultiModal):
584
packed_modules_mapping = {
585
"query_key_value": ["query_key_value"],
586
"dense_h_to_4h": ["dense_h_to_4h"]
@@ -610,7 +628,8 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
628
intermediate_tensors: Optional[IntermediateTensors] = None,
629
**kwargs) -> torch.Tensor:
630
hidden_states = self.transformer(input_ids, positions, kv_caches,
613
- attn_metadata, **kwargs)
631
+ attn_metadata, intermediate_tensors,
632
+ **kwargs)
633
return hidden_states
634
635
def compute_logits(
@@ -656,6 +675,8 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
675
# Skip loading extra bias for GPTQ models.
676
if name.endswith(".bias") and name not in params_dict:
677
continue
678
+ if is_pp_missing_parameter(name, self):
679
+ continue
680
param = params_dict[name]
681
weight_loader = getattr(param, "weight_loader",
682
default_weight_loader)
vllm/model_executor/models/molmo.py
+46
-27
@@ -30,21 +30,21 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
QKVParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
-from vllm.model_executor.layers.quantization.base_config import (
34
- QuantizationConfig)
33
+from vllm.model_executor.layers.quantization import QuantizationConfig
34
from vllm.model_executor.layers.rotary_embedding import get_rope
35
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
36
from vllm.model_executor.layers.vocab_parallel_embedding import (
37
ParallelLMHead, VocabParallelEmbedding)
38
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
-from vllm.model_executor.models.interfaces import SupportsMultiModal
41
-from vllm.model_executor.models.utils import make_layers
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
40
+from vllm.multimodal.utils import cached_get_tokenizer
41
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
SequenceData)
43
from vllm.transformers_utils.processor import get_processor
44
47
-from .utils import get_vit_attn_backend
45
+from .interfaces import SupportsMultiModal, SupportsPP
46
+from .utils import (get_vit_attn_backend,
47
+ make_empty_intermediate_tensors_factory, make_layers)
48
49
# TODO: hard-coded for now. Consider making it configurable.
50
VIT_LAYERS = [-2, -9]
@@ -744,6 +744,10 @@ class MolmoModel(nn.Module):
744
assert config.layer_norm_type == "rms"
745
self.norm = RMSNorm(config.hidden_size, config.layer_norm_eps)
746
747
+ self.make_empty_intermediate_tensors = (
748
+ make_empty_intermediate_tensors_factory(
749
+ ["hidden_states", "residual"], config.hidden_size))
750
+
751
def forward(
752
self,
753
input_ids: torch.Tensor,
@@ -925,16 +929,19 @@ def pad_images(
929
930
931
def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
928
- prompt = inputs.get("prompt", None)
929
- multi_modal_data = inputs.get("multi_modal_data", None)
930
- if multi_modal_data is not None:
931
- image = multi_modal_data.get("image", None)
932
- else:
933
- image = None
932
+ prompt = inputs.get("prompt")
933
+ multi_modal_data = inputs.get("multi_modal_data")
934
+ image = None if multi_modal_data is None else multi_modal_data.get("image")
935
+
936
processor = cached_get_processor(ctx.model_config.model,
937
trust_remote_code=True,
938
revision=ctx.model_config.code_revision)
939
940
+ model_config = ctx.model_config
941
+ tokenizer = cached_get_tokenizer(
942
+ model_config.tokenizer,
943
+ trust_remote_code=model_config.trust_remote_code)
944
+
945
# NOTE: message formatting for raw text prompt is only applied for
946
# offline inference; for online inference, the prompt is always in
947
# instruction format and tokenized.
@@ -997,9 +1004,13 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1004
1005
multi_modal_data = dict(image=image_data)
1006
1007
+ prompt = inputs.get("prompt")
1008
+ if prompt is None:
1009
+ prompt = tokenizer.decode(out["input_ids"])
1010
+
1011
return token_inputs(
1012
prompt_token_ids=out["input_ids"],
1002
- prompt=inputs["prompt"],
1013
+ prompt=prompt,
1014
multi_modal_data=multi_modal_data,
1015
)
1016
@@ -1008,7 +1019,7 @@ def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1019
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1020
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1021
@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1011
-class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1022
+class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
1023
1024
def __init__(
1025
self,
@@ -1040,6 +1051,9 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1051
or config.vocab_size)
1052
self.sampler = Sampler()
1053
1054
+ self.make_empty_intermediate_tensors = (
1055
+ self.model.make_empty_intermediate_tensors)
1056
+
1057
def _parse_and_validate_image_input(
1058
self,
1059
**kwargs: object,
@@ -1123,31 +1137,36 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1137
positions: torch.LongTensor,
1138
kv_caches: List[torch.Tensor],
1139
attn_metadata: AttentionMetadata,
1140
+ intermediate_tensors: Optional[IntermediateTensors] = None,
1141
**kwargs: object,
1142
) -> SamplerOutput:
1143
+ if intermediate_tensors is not None:
1144
+ input_ids = None
1145
+ inputs_embeds = None
1146
+ else:
1147
+ image_input = self._parse_and_validate_image_input(**kwargs)
1148
1129
- image_input = self._parse_and_validate_image_input(**kwargs)
1130
-
1131
- if image_input is not None:
1132
- inputs_embeds = self.model.embed_tokens(input_ids)
1133
- image_features = self._process_image_input(image_input)
1149
+ if image_input is not None:
1150
+ inputs_embeds = self.model.embed_tokens(input_ids)
1151
+ image_features = self._process_image_input(image_input)
1152
1135
- inputs_embeds = self._merge_multimodal_embeddings(
1136
- inputs_embeds,
1137
- image_features,
1138
- image_input["image_input_idx"],
1139
- image_input["seq_len"],
1140
- )
1153
+ inputs_embeds = self._merge_multimodal_embeddings(
1154
+ inputs_embeds,
1155
+ image_features,
1156
+ image_input["image_input_idx"],
1157
+ image_input["seq_len"],
1158
+ )
1159
1142
- input_ids = None
1143
- else:
1144
- inputs_embeds = None
1160
+ input_ids = None
1161
+ else:
1162
+ inputs_embeds = None
1163
1164
hidden_states = self.model(
1165
input_ids=input_ids,
1166
positions=positions,
1167
kv_caches=kv_caches,
1168
attn_metadata=attn_metadata,
1169
+ intermediate_tensors=intermediate_tensors,
1170
inputs_embeds=inputs_embeds,
1171
)
1172
vllm/model_executor/models/qwen2_rm.py
+2
-1
@@ -119,5 +119,6 @@ class Qwen2ForRewardModel(nn.Module, SupportsPP):
119
return self._pooler(hidden_states, pooling_metadata)
120
121
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
122
- loader = AutoWeightsLoader(self)
122
+ loader = AutoWeightsLoader(self,
123
+ ignore_unexpected_prefixes=["lm_head."])
124
loader.load_weights(weights)
vllm/model_executor/models/qwen2_vl.py
+13
-10
@@ -61,6 +61,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
61
MultiModalInputs)
62
from vllm.multimodal.base import MultiModalData
63
from vllm.multimodal.image import cached_get_image_processor
64
+from vllm.multimodal.utils import cached_get_tokenizer
65
from vllm.sequence import IntermediateTensors, SequenceData
66
from vllm.transformers_utils.config import uses_mrope
67
from vllm.transformers_utils.processor import cached_get_processor
@@ -817,7 +818,7 @@ def input_processor_for_qwen2_vl(
818
min_pixels: Optional[int] = None,
819
max_pixels: Optional[int] = None,
820
) -> DecoderOnlyInputs:
820
- multi_modal_data = inputs.get("multi_modal_data", None)
821
+ multi_modal_data = inputs.get("multi_modal_data")
822
if multi_modal_data is None:
823
return inputs
824
@@ -830,6 +831,7 @@ def input_processor_for_qwen2_vl(
831
min_pixels = min_pixels if min_pixels else image_processor.min_pixels
832
max_pixels = max_pixels if max_pixels else image_processor.max_pixels
833
834
+ model_config = ctx.model_config
835
hf_config = ctx.get_hf_config(Qwen2VLConfig)
836
837
# To avoid redundant processing of vision objects (resize, rescale, etc.),
@@ -845,14 +847,11 @@ def input_processor_for_qwen2_vl(
847
# return_tensors="pt")
848
# prompt_token_ids = inputs["input_ids"][0].tolist()
849
848
- prompt_token_ids = inputs.get("prompt_token_ids", None)
849
- if prompt_token_ids is None:
850
- prompt = inputs["prompt"]
851
- prompt_token_ids = processor.tokenizer(
852
- prompt,
853
- padding=True,
854
- return_tensors=None,
855
- )["input_ids"]
850
+ tokenizer = cached_get_tokenizer(
851
+ model_config.tokenizer,
852
+ trust_remote_code=model_config.trust_remote_code)
853
+
854
+ prompt_token_ids = inputs["prompt_token_ids"]
855
856
# Expand image pad tokens.
857
@@ -894,9 +893,13 @@ def input_processor_for_qwen2_vl(
893
min_pixels=min_pixels,
894
max_pixels=max_pixels)
895
896
+ prompt = inputs.get("prompt")
897
+ if prompt is None:
898
+ prompt = tokenizer.decode(prompt_token_ids)
899
+
900
return token_inputs(
901
prompt_token_ids=prompt_token_ids,
899
- prompt=inputs["prompt"],
902
+ prompt=prompt,
903
multi_modal_data=multi_modal_data,
904
)
905
vllm/model_executor/models/utils.py
+42
-12
@@ -79,6 +79,9 @@ class AutoWeightsLoader:
79
80
Similarly, the weight loading logic for individual parameters can be
81
overridden by defining a ``weight_loader`` method.
82
+
83
+ Detailed weight loading information can be viewed by setting the
84
+ environment variable ``VLLM_LOGGING_LEVEL=DEBUG``.
85
"""
86
87
def __init__(
@@ -136,20 +139,27 @@ class AutoWeightsLoader:
139
weight_qualname = self._get_qualname(base_prefix, weight_name)
140
141
if self._can_skip(weight_qualname):
142
+ logger.debug("Skipping weight %s", weight_qualname)
143
+
144
continue
145
146
if weight_name != "":
142
- if not self._can_ignore_unexpected(weight_qualname):
143
- raise ValueError(
144
- f"Attempted to load nested weight '{weight_qualname}' "
145
- f"into a single parameter '{base_prefix}'")
147
+ if self._can_ignore_unexpected(weight_qualname):
148
+ logger.debug("Ignoring weight %s", weight_qualname)
149
147
- continue
150
+ continue
151
+
152
+ raise ValueError(
153
+ f"Attempted to load nested weight '{weight_qualname}' "
154
+ f"into a single parameter '{base_prefix}'")
155
156
weight_loader = getattr(param, "weight_loader",
157
default_weight_loader)
158
weight_loader(param, weight_data)
159
160
+ logger.debug("Loaded weight %s with shape %s", weight_qualname,
161
+ param.shape)
162
+
163
yield weight_qualname
164
165
def _load_module(
@@ -175,21 +185,41 @@ class AutoWeightsLoader:
185
for child_prefix, child_weights in self._groupby_prefix(weights):
186
prefix = self._get_qualname(base_prefix, child_prefix)
187
178
- if self._can_skip(prefix):
179
- continue
180
-
188
if child_prefix in child_modules:
189
+ if self._can_skip(prefix + "."):
190
+ logger.debug("Skipping module %s", prefix)
191
+
192
+ continue
193
+
194
yield from self._load_module(prefix,
195
child_modules[child_prefix],
196
child_weights)
197
elif child_prefix in child_params:
198
+ if self._can_skip(prefix):
199
+ logger.debug("Skipping param %s", prefix)
200
+
201
+ continue
202
+
203
yield from self._load_param(prefix, child_params[child_prefix],
204
child_weights)
205
else:
189
- if not self._can_ignore_unexpected(prefix):
190
- msg = (f"There is no module or parameter named '{prefix}' "
191
- f"in {type(self.module).__name__}")
192
- raise ValueError(msg)
206
+ can_skip_module = self._can_skip(prefix + ".")
207
+ can_skip_param = self._can_skip(prefix)
208
+ if can_skip_module or can_skip_param:
209
+ logger.debug("Skipping missing %s", prefix)
210
+
211
+ continue
212
+
213
+ can_ignore_module = self._can_ignore_unexpected(prefix + ".")
214
+ can_ignore_param = self._can_ignore_unexpected(prefix)
215
+ if can_ignore_module or can_ignore_param:
216
+ logger.debug("Ignoring missing %s", prefix)
217
+
218
+ continue
219
+
220
+ msg = (f"There is no module or parameter named '{prefix}' "
221
+ f"in {type(self.module).__name__}")
222
+ raise ValueError(msg)
223
224
def load_weights(
225
self,