A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Composite weight loading for multimodal Qwen2 (#10944)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 7, 2024 at 22:22 UTC
bf0e382e16065edebbbb414f7889d31523a569e1
7 files changed
+147
-205
vllm/config.py
+9
-1
@@ -2472,7 +2472,15 @@ class VllmConfig:
2472
return quant_config
2473
return None
2474
2475
- def with_hf_config(self, hf_config: PretrainedConfig) -> "VllmConfig":
2475
+ def with_hf_config(
2476
+ self,
2477
+ hf_config: PretrainedConfig,
2478
+ architectures: Optional[list[str]] = None,
2479
+ ) -> "VllmConfig":
2480
+ if architectures is not None:
2481
+ hf_config = copy.deepcopy(hf_config)
2482
+ hf_config.architectures = architectures
2483
+
2484
model_config = copy.deepcopy(self.model_config)
2485
model_config.hf_config = hf_config
2486
vllm/model_executor/model_loader/loader.py
+1
-3
@@ -101,12 +101,10 @@ def _initialize_model(
101
vllm_config: VllmConfig,
102
*,
103
prefix: str = "",
104
- architectures: Optional[list[str]] = None,
104
) -> nn.Module:
105
"""Initialize a model with the given configurations."""
106
model_config = vllm_config.model_config
108
- model_class, _ = get_model_architecture(model_config,
109
- architectures=architectures)
107
+ model_class, _ = get_model_architecture(model_config)
108
109
signatures = inspect.signature(model_class.__init__)
110
all_params = [param.name for param in signatures.parameters.values()]
vllm/model_executor/model_loader/utils.py
+3
-7
@@ -1,6 +1,6 @@
1
"""Utilities for selecting and loading models."""
2
import contextlib
3
-from typing import Optional, Tuple, Type
3
+from typing import Tuple, Type
4
5
import torch
6
from torch import nn
@@ -20,12 +20,8 @@ def set_default_torch_dtype(dtype: torch.dtype):
20
21
22
def get_model_architecture(
23
- model_config: ModelConfig,
24
- *,
25
- architectures: Optional[list[str]] = None,
26
-) -> Tuple[Type[nn.Module], str]:
27
- if architectures is None:
28
- architectures = getattr(model_config.hf_config, "architectures", [])
23
+ model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
24
+ architectures = getattr(model_config.hf_config, "architectures", [])
25
26
# Special handling for quantized Mixtral.
27
# FIXME(woosuk): This is a temporary hack.
vllm/model_executor/models/qwen2.py
+10
-7
@@ -444,14 +444,17 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
444
self.model = Qwen2Model(vllm_config=vllm_config,
445
prefix=maybe_prefix(prefix, "model"))
446
447
- if config.tie_word_embeddings:
448
- self.lm_head = self.model.embed_tokens
447
+ if get_pp_group().is_last_rank:
448
+ if config.tie_word_embeddings:
449
+ self.lm_head = self.model.embed_tokens
450
+ else:
451
+ self.lm_head = ParallelLMHead(config.vocab_size,
452
+ config.hidden_size,
453
+ quant_config=quant_config,
454
+ prefix=maybe_prefix(
455
+ prefix, "lm_head"))
456
else:
450
- self.lm_head = ParallelLMHead(config.vocab_size,
451
- config.hidden_size,
452
- quant_config=quant_config,
453
- prefix=maybe_prefix(
454
- prefix, "lm_head"))
457
+ self.lm_head = PPMissingLayer()
458
459
self.logits_processor = LogitsProcessor(config.vocab_size)
460
self.sampler = get_sampler()
vllm/model_executor/models/qwen2_audio.py
+32
-85
@@ -19,7 +19,7 @@
19
# See the License for the specific language governing permissions and
20
# limitations under the License.
21
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22
-from functools import lru_cache
22
+from functools import cached_property, lru_cache
23
from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24
Union)
25
@@ -34,12 +34,7 @@ from vllm.config import VllmConfig
34
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
35
InputContext, token_inputs)
36
from vllm.logger import init_logger
37
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
40
-from vllm.model_executor.model_loader.weight_utils import (
41
- default_weight_loader, maybe_remap_kv_scale_name)
42
-from vllm.model_executor.models.qwen2 import Qwen2Model
38
from vllm.model_executor.sampling_metadata import SamplingMetadata
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40
from vllm.multimodal.inputs import NestedTensors
@@ -47,15 +42,11 @@ from vllm.multimodal.utils import consecutive_placeholder_ranges
42
from vllm.sequence import IntermediateTensors, SequenceData
43
44
from .interfaces import SupportsMultiModal, SupportsPP
50
-from .utils import merge_multimodal_embeddings
45
+from .utils import (AutoWeightsLoader, init_vllm_registered_model,
46
+ maybe_prefix, merge_multimodal_embeddings)
47
48
logger = init_logger(__name__)
49
54
-_KEYS_TO_MODIFY_MAPPING = {
55
- "language_model.lm_head": "lm_head",
56
- "language_model.model": "language_model",
57
-}
58
-
50
51
# # === Audio Inputs === #
52
class Qwen2AudioInputs(TypedDict):
@@ -281,25 +272,23 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
272
273
self.quant_config = quant_config
274
284
- self.language_model = Qwen2Model(
285
- vllm_config=vllm_config.with_hf_config(config.text_config),
286
- prefix=prefix)
287
- self.unpadded_vocab_size = config.text_config.vocab_size
288
- if config.text_config.tie_word_embeddings:
289
- self.lm_head = self.language_model.embed_tokens
290
- else:
291
- self.lm_head = ParallelLMHead(config.text_config.vocab_size,
292
- config.text_config.hidden_size,
293
- quant_config=quant_config)
294
- logit_scale = getattr(config, "logit_scale", 1.0)
295
- self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
296
- config.text_config.vocab_size,
297
- logit_scale)
298
- self.sampler = get_sampler()
275
+ self.language_model = init_vllm_registered_model(
276
+ vllm_config=vllm_config,
277
+ hf_config=config.text_config,
278
+ prefix=maybe_prefix(prefix, "language_model"),
279
+ architectures=["Qwen2ForCausalLM"],
280
+ )
281
282
self.make_empty_intermediate_tensors = (
283
self.language_model.make_empty_intermediate_tensors)
284
285
+ @cached_property
286
+ def sampler(self):
287
+ if hasattr(self.language_model, "sampler"):
288
+ return self.language_model.sampler
289
+
290
+ return get_sampler()
291
+
292
def _validate_and_reshape_mm_tensor(self,
293
mm_input: Union[torch.Tensor,
294
List[torch.Tensor]],
@@ -414,72 +403,30 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
403
multimodal_embeddings)
404
input_ids = None
405
417
- hidden_states = self.language_model(input_ids,
418
- positions,
419
- kv_caches,
420
- attn_metadata,
421
- intermediate_tensors,
422
- inputs_embeds=inputs_embeds)
406
+ hidden_states = self.language_model.model(input_ids,
407
+ positions,
408
+ kv_caches,
409
+ attn_metadata,
410
+ intermediate_tensors,
411
+ inputs_embeds=inputs_embeds)
412
return hidden_states
413
425
- def compute_logits(self, hidden_states: torch.Tensor,
426
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
427
- logits = self.logits_processor(self.lm_head, hidden_states,
428
- sampling_metadata)
429
- return logits
414
+ def compute_logits(
415
+ self,
416
+ hidden_states: torch.Tensor,
417
+ sampling_metadata: SamplingMetadata,
418
+ ) -> Optional[torch.Tensor]:
419
+ return self.language_model.compute_logits(hidden_states,
420
+ sampling_metadata)
421
422
def sample(
423
self,
424
logits: torch.Tensor,
425
sampling_metadata: SamplingMetadata,
426
) -> Optional[SamplerOutput]:
436
- next_tokens = self.sampler(logits, sampling_metadata)
437
- return next_tokens
427
+ return self.language_model.sample(logits, sampling_metadata)
428
429
def load_weights(self, weights: Iterable[Tuple[str,
430
torch.Tensor]]) -> Set[str]:
441
- stacked_params_mapping = [
442
- # (param_name, shard_name, shard_id)
443
- ("qkv_proj", "q_proj", "q"),
444
- ("qkv_proj", "k_proj", "k"),
445
- ("qkv_proj", "v_proj", "v"),
446
- ("gate_up_proj", "gate_proj", 0),
447
- ("gate_up_proj", "up_proj", 1),
448
- ]
449
- params_dict = dict(self.named_parameters(remove_duplicate=False))
450
- loaded_params: Set[str] = set()
451
- for name, loaded_weight in weights:
452
- if "rotary_emb.inv_freq" in name:
453
- continue
454
- if (self.config.text_config.tie_word_embeddings
455
- and "lm_head.weight" in name):
456
- continue
457
- for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
458
- if key_to_modify in name:
459
- name = name.replace(key_to_modify, new_key)
460
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
461
- if weight_name not in name or 'audio' in name:
462
- continue
463
- name = name.replace(weight_name, param_name)
464
- # Skip loading extra bias for GPTQ models.
465
- if name.endswith(".bias") and name not in params_dict:
466
- continue
467
- param = params_dict[name]
468
- weight_loader = param.weight_loader
469
- weight_loader(param, loaded_weight, shard_id)
470
- break
471
- else:
472
- # Skip loading extra bias for GPTQ models.
473
- if name.endswith(".bias") and name not in params_dict:
474
- continue
475
- # Remapping the name of FP8 kv-scale.
476
- name = maybe_remap_kv_scale_name(name, params_dict)
477
- if name is None:
478
- continue
479
-
480
- param = params_dict[name]
481
- weight_loader = getattr(param, "weight_loader",
482
- default_weight_loader)
483
- weight_loader(param, loaded_weight)
484
- loaded_params.add(name)
485
- return loaded_params
431
+ loader = AutoWeightsLoader(self)
432
+ return loader.load_weights(weights)
vllm/model_executor/models/qwen2_vl.py
+83
-96
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24
-from functools import partial
24
+from functools import cached_property, partial
25
from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26
Optional, Set, Tuple, Type, TypedDict, Union)
27
@@ -40,7 +40,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
41
from vllm.attention import AttentionMetadata
42
from vllm.config import VllmConfig
43
-from vllm.distributed import get_pp_group, parallel_state
43
+from vllm.distributed import parallel_state
44
from vllm.distributed import utils as dist_utils
45
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
46
InputContext, token_inputs)
@@ -49,15 +49,12 @@ from vllm.model_executor import SamplingMetadata
49
from vllm.model_executor.layers.activation import QuickGELU
50
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51
RowParallelLinear)
52
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
52
from vllm.model_executor.layers.quantization import QuantizationConfig
53
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
54
from vllm.model_executor.layers.quantization.gptq_marlin import (
55
GPTQMarlinConfig)
56
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
58
-from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
57
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
60
-from vllm.model_executor.models.qwen2 import Qwen2Model
58
from vllm.multimodal import MULTIMODAL_REGISTRY
59
from vllm.multimodal.image import cached_get_image_processor
60
from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
@@ -69,9 +66,8 @@ from vllm.transformers_utils.config import uses_mrope
66
from vllm.transformers_utils.processor import cached_get_processor
67
68
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
72
-from .utils import (PPMissingLayer, get_vit_attn_backend,
73
- is_pp_missing_parameter,
74
- make_empty_intermediate_tensors_factory, maybe_prefix)
69
+from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
70
+ init_vllm_registered_model, maybe_prefix)
71
72
logger = init_logger(__name__)
73
@@ -506,6 +502,8 @@ class Qwen2VisionTransformer(nn.Module):
502
mlp_ratio: float = vision_config.mlp_ratio
503
504
self.spatial_merge_size = spatial_merge_size
505
+ self.num_heads = num_heads
506
+ self.embed_dim = embed_dim
507
508
self.patch_embed = Qwen2VisionPatchEmbed(
509
patch_size=patch_size,
@@ -595,6 +593,53 @@ class Qwen2VisionTransformer(nn.Module):
593
x = self.merger(x)
594
return x
595
596
+ def load_weights(self, weights: Iterable[Tuple[str,
597
+ torch.Tensor]]) -> Set[str]:
598
+ stacked_params_mapping = [
599
+ # (param_name, shard_name, shard_id)
600
+ ("qkv_proj", "q_proj", "q"),
601
+ ("qkv_proj", "k_proj", "k"),
602
+ ("qkv_proj", "v_proj", "v"),
603
+ ]
604
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
605
+ loaded_params: Set[str] = set()
606
+
607
+ for name, loaded_weight in weights:
608
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
609
+ if weight_name not in name:
610
+ continue
611
+ name = name.replace(weight_name, param_name)
612
+
613
+ param = params_dict[name]
614
+ weight_loader = param.weight_loader
615
+ weight_loader(param, loaded_weight, shard_id)
616
+ break
617
+ else:
618
+ if name.endswith("qkv.weight"):
619
+ visual_num_heads = self.num_heads
620
+ visual_embed_dim = self.embed_dim
621
+ head_size = visual_embed_dim // visual_num_heads
622
+ loaded_weight = loaded_weight.view(3, visual_num_heads,
623
+ head_size,
624
+ visual_embed_dim)
625
+ loaded_weight = loaded_weight.transpose(0, 1)
626
+ loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
627
+ elif name.endswith("qkv.bias"):
628
+ visual_num_heads = self.num_heads
629
+ visual_embed_dim = self.embed_dim
630
+ head_size = visual_embed_dim // visual_num_heads
631
+ loaded_weight = loaded_weight.view(3, visual_num_heads,
632
+ head_size)
633
+ loaded_weight = loaded_weight.transpose(0, 1)
634
+ loaded_weight = loaded_weight.reshape(-1)
635
+
636
+ param = params_dict[name]
637
+ weight_loader = getattr(param, "weight_loader",
638
+ default_weight_loader)
639
+ weight_loader(param, loaded_weight)
640
+ loaded_params.add(name)
641
+ return loaded_params
642
+
643
644
# === Vision input helpers === #
645
@@ -1082,27 +1127,21 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1127
prefix=maybe_prefix(prefix, "visual"),
1128
)
1129
1085
- self.model = Qwen2Model(vllm_config=vllm_config,
1086
- prefix=maybe_prefix(prefix, "model"))
1130
+ self.language_model = init_vllm_registered_model(
1131
+ vllm_config=vllm_config,
1132
+ prefix=maybe_prefix(prefix, "language_model"),
1133
+ architectures=["Qwen2ForCausalLM"],
1134
+ )
1135
1088
- if get_pp_group().is_last_rank:
1089
- if config.tie_word_embeddings:
1090
- self.lm_head = self.model.embed_tokens
1091
- else:
1092
- self.lm_head = ParallelLMHead(config.vocab_size,
1093
- config.hidden_size,
1094
- quant_config=quant_config,
1095
- prefix=maybe_prefix(
1096
- prefix, "lm_head"))
1097
- else:
1098
- self.lm_head = PPMissingLayer()
1136
+ self.make_empty_intermediate_tensors = (
1137
+ self.language_model.make_empty_intermediate_tensors)
1138
1100
- self.logits_processor = LogitsProcessor(config.vocab_size)
1101
- self.sampler = get_sampler()
1139
+ @cached_property
1140
+ def sampler(self):
1141
+ if hasattr(self.language_model, "sampler"):
1142
+ return self.language_model.sampler
1143
1103
- self.make_empty_intermediate_tensors = (
1104
- make_empty_intermediate_tensors_factory(
1105
- ["hidden_states", "residual"], config.hidden_size))
1144
+ return get_sampler()
1145
1146
def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1147
# GPTQ configs do not have a list of ignored modules, however AutoGPTQ
@@ -1261,7 +1300,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1300
multimodal_embeddings: Optional[List[Tuple[NestedTensors,
1301
str]]] = None,
1302
) -> torch.Tensor:
1264
- inputs_embeds = self.model.get_input_embeddings(input_ids)
1303
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1304
if multimodal_embeddings is not None:
1305
for embeddings, modality in multimodal_embeddings:
1306
if modality == "image":
@@ -1330,7 +1369,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1369
multimodal_embeddings)
1370
input_ids = None
1371
1333
- hidden_states = self.model(
1372
+ hidden_states = self.language_model.model(
1373
input_ids=input_ids,
1374
positions=positions,
1375
kv_caches=kv_caches,
@@ -1340,80 +1379,28 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1379
)
1380
return hidden_states
1381
1343
- def compute_logits(self, hidden_states: torch.Tensor,
1344
- sampling_metadata: SamplingMetadata) -> torch.Tensor:
1345
- logits = self.logits_processor(self.lm_head, hidden_states,
1346
- sampling_metadata)
1347
- return logits
1382
+ def compute_logits(
1383
+ self,
1384
+ hidden_states: torch.Tensor,
1385
+ sampling_metadata: SamplingMetadata,
1386
+ ) -> Optional[torch.Tensor]:
1387
+ return self.language_model.compute_logits(hidden_states,
1388
+ sampling_metadata)
1389
1390
def sample(
1391
self,
1392
logits: torch.Tensor,
1393
sampling_metadata: SamplingMetadata,
1394
) -> Optional[SamplerOutput]:
1354
- next_tokens = self.sampler(logits, sampling_metadata)
1355
- return next_tokens
1395
+ return self.language_model.sample(logits, sampling_metadata)
1396
1397
def load_weights(self, weights: Iterable[Tuple[str,
1398
torch.Tensor]]) -> Set[str]:
1359
- stacked_params_mapping = [
1360
- # (param_name, shard_name, shard_id)
1361
- ("qkv_proj", "q_proj", "q"),
1362
- ("qkv_proj", "k_proj", "k"),
1363
- ("qkv_proj", "v_proj", "v"),
1364
- ("gate_up_proj", "up_proj", 1),
1365
- ("gate_up_proj", "gate_proj", 0),
1366
- ]
1367
- params_dict = dict(self.named_parameters(remove_duplicate=False))
1368
- loaded_params: Set[str] = set()
1369
- for name, loaded_weight in weights:
1370
- if "rotary_emb.inv_freq" in name:
1371
- continue
1372
- if self.config.tie_word_embeddings and "lm_head.weight" in name:
1373
- continue
1374
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
1375
- if weight_name not in name:
1376
- continue
1377
- name = name.replace(weight_name, param_name)
1378
- # Skip loading extra bias for GPTQ models.
1379
- if name.endswith(".bias") and name not in params_dict:
1380
- continue
1381
- if is_pp_missing_parameter(name, self):
1382
- continue
1383
- param = params_dict[name]
1384
- weight_loader = param.weight_loader
1385
- weight_loader(param, loaded_weight, shard_id)
1386
- break
1387
- else:
1388
- if "visual" in name and name.endswith("qkv.weight"):
1389
- visual_num_heads = self.config.vision_config.num_heads
1390
- visual_embed_dim = self.config.vision_config.embed_dim
1391
- head_size = visual_embed_dim // visual_num_heads
1392
- loaded_weight = loaded_weight.view(3, visual_num_heads,
1393
- head_size,
1394
- visual_embed_dim)
1395
- loaded_weight = loaded_weight.transpose(0, 1)
1396
- loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
1397
- elif "visual" in name and name.endswith("qkv.bias"):
1398
- visual_num_heads = self.config.vision_config.num_heads
1399
- visual_embed_dim = self.config.vision_config.embed_dim
1400
- head_size = visual_embed_dim // visual_num_heads
1401
- loaded_weight = loaded_weight.view(3, visual_num_heads,
1402
- head_size)
1403
- loaded_weight = loaded_weight.transpose(0, 1)
1404
- loaded_weight = loaded_weight.reshape(-1)
1405
- try:
1406
- # Skip loading extra bias for GPTQ models.
1407
- if name.endswith(".bias") and name not in params_dict:
1408
- continue
1409
- if is_pp_missing_parameter(name, self):
1410
- continue
1411
- param = params_dict[name]
1412
- except KeyError:
1413
- raise ValueError(f"Unexpected weight: {name}") from None
1414
-
1415
- weight_loader = getattr(param, "weight_loader",
1416
- default_weight_loader)
1417
- weight_loader(param, loaded_weight)
1418
- loaded_params.add(name)
1419
- return loaded_params
1399
+ hf_to_vllm_mapper = WeightsMapper(
1400
+ orig_to_new_prefix={
1401
+ "lm_head.": "language_model.lm_head.",
1402
+ "model.": "language_model.model.",
1403
+ })
1404
+
1405
+ loader = AutoWeightsLoader(self)
1406
+ return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/utils.py
+9
-6
@@ -17,7 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
18
from vllm.platforms import _Backend, current_platform
19
from vllm.sequence import IntermediateTensors
20
-from vllm.utils import is_pin_memory_available
20
+from vllm.utils import is_pin_memory_available, print_warning_once
21
22
logger = init_logger(__name__)
23
@@ -251,12 +251,15 @@ def init_vllm_registered_model(
251
"""
252
from vllm.model_executor.model_loader.loader import _initialize_model
253
254
+ if hf_config is None and architectures is not None:
255
+ # So that the architectures field is overridden
256
+ hf_config = vllm_config.model_config.hf_config
257
+
258
if hf_config is not None:
255
- vllm_config = vllm_config.with_hf_config(hf_config)
259
+ vllm_config = vllm_config.with_hf_config(hf_config,
260
+ architectures=architectures)
261
257
- return _initialize_model(vllm_config=vllm_config,
258
- prefix=prefix,
259
- architectures=architectures)
262
+ return _initialize_model(vllm_config=vllm_config, prefix=prefix)
263
264
265
@overload
@@ -592,7 +595,7 @@ def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
595
if is_flash_attn_2_available():
596
selected_backend = _Backend.FLASH_ATTN
597
else:
595
- logger.warning(
598
+ print_warning_once(
599
"Current `vllm-flash-attn` has a bug inside vision module, "
600
"so we use xformers backend instead. You can run "
601
"`pip install flash-attn` to use flash-attention backend.")