@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf0e382e16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Composite weight loading for multimodal Qwen2 (#10944)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 7, 2024 at 22:22 UTC bf0e382e16065edebbbb414f7889d31523a569e1
7 files changed +147 -205
vllm/config.py
+9 -1
@@ -2472,7 +2472,15 @@ class VllmConfig:
2472 return quant_config
2473 return None
2474
2475 - def with_hf_config(self, hf_config: PretrainedConfig) -> "VllmConfig":
2475 + def with_hf_config(
2476 + self,
2477 + hf_config: PretrainedConfig,
2478 + architectures: Optional[list[str]] = None,
2479 + ) -> "VllmConfig":
2480 + if architectures is not None:
2481 + hf_config = copy.deepcopy(hf_config)
2482 + hf_config.architectures = architectures
2483 +
2484 model_config = copy.deepcopy(self.model_config)
2485 model_config.hf_config = hf_config
2486
vllm/model_executor/model_loader/loader.py
+1 -3
@@ -101,12 +101,10 @@ def _initialize_model(
101 vllm_config: VllmConfig,
102 *,
103 prefix: str = "",
104 - architectures: Optional[list[str]] = None,
104 ) -> nn.Module:
105 """Initialize a model with the given configurations."""
106 model_config = vllm_config.model_config
108 - model_class, _ = get_model_architecture(model_config,
109 - architectures=architectures)
107 + model_class, _ = get_model_architecture(model_config)
108
109 signatures = inspect.signature(model_class.__init__)
110 all_params = [param.name for param in signatures.parameters.values()]
vllm/model_executor/model_loader/utils.py
+3 -7
@@ -1,6 +1,6 @@
1 """Utilities for selecting and loading models."""
2 import contextlib
3 -from typing import Optional, Tuple, Type
3 +from typing import Tuple, Type
4
5 import torch
6 from torch import nn
@@ -20,12 +20,8 @@ def set_default_torch_dtype(dtype: torch.dtype):
20
21
22 def get_model_architecture(
23 - model_config: ModelConfig,
24 - *,
25 - architectures: Optional[list[str]] = None,
26 -) -> Tuple[Type[nn.Module], str]:
27 - if architectures is None:
28 - architectures = getattr(model_config.hf_config, "architectures", [])
23 + model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
24 + architectures = getattr(model_config.hf_config, "architectures", [])
25
26 # Special handling for quantized Mixtral.
27 # FIXME(woosuk): This is a temporary hack.
vllm/model_executor/models/qwen2.py
+10 -7
@@ -444,14 +444,17 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
444 self.model = Qwen2Model(vllm_config=vllm_config,
445 prefix=maybe_prefix(prefix, "model"))
446
447 - if config.tie_word_embeddings:
448 - self.lm_head = self.model.embed_tokens
447 + if get_pp_group().is_last_rank:
448 + if config.tie_word_embeddings:
449 + self.lm_head = self.model.embed_tokens
450 + else:
451 + self.lm_head = ParallelLMHead(config.vocab_size,
452 + config.hidden_size,
453 + quant_config=quant_config,
454 + prefix=maybe_prefix(
455 + prefix, "lm_head"))
456 else:
450 - self.lm_head = ParallelLMHead(config.vocab_size,
451 - config.hidden_size,
452 - quant_config=quant_config,
453 - prefix=maybe_prefix(
454 - prefix, "lm_head"))
457 + self.lm_head = PPMissingLayer()
458
459 self.logits_processor = LogitsProcessor(config.vocab_size)
460 self.sampler = get_sampler()
vllm/model_executor/models/qwen2_audio.py
+32 -85
@@ -19,7 +19,7 @@
19 # See the License for the specific language governing permissions and
20 # limitations under the License.
21 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22 -from functools import lru_cache
22 +from functools import cached_property, lru_cache
23 from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24 Union)
25
@@ -34,12 +34,7 @@ from vllm.config import VllmConfig
34 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
35 InputContext, token_inputs)
36 from vllm.logger import init_logger
37 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
40 -from vllm.model_executor.model_loader.weight_utils import (
41 - default_weight_loader, maybe_remap_kv_scale_name)
42 -from vllm.model_executor.models.qwen2 import Qwen2Model
38 from vllm.model_executor.sampling_metadata import SamplingMetadata
39 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
40 from vllm.multimodal.inputs import NestedTensors
@@ -47,15 +42,11 @@ from vllm.multimodal.utils import consecutive_placeholder_ranges
42 from vllm.sequence import IntermediateTensors, SequenceData
43
44 from .interfaces import SupportsMultiModal, SupportsPP
50 -from .utils import merge_multimodal_embeddings
45 +from .utils import (AutoWeightsLoader, init_vllm_registered_model,
46 + maybe_prefix, merge_multimodal_embeddings)
47
48 logger = init_logger(__name__)
49
54 -_KEYS_TO_MODIFY_MAPPING = {
55 - "language_model.lm_head": "lm_head",
56 - "language_model.model": "language_model",
57 -}
58 -
50
51 # # === Audio Inputs === #
52 class Qwen2AudioInputs(TypedDict):
@@ -281,25 +272,23 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
272
273 self.quant_config = quant_config
274
284 - self.language_model = Qwen2Model(
285 - vllm_config=vllm_config.with_hf_config(config.text_config),
286 - prefix=prefix)
287 - self.unpadded_vocab_size = config.text_config.vocab_size
288 - if config.text_config.tie_word_embeddings:
289 - self.lm_head = self.language_model.embed_tokens
290 - else:
291 - self.lm_head = ParallelLMHead(config.text_config.vocab_size,
292 - config.text_config.hidden_size,
293 - quant_config=quant_config)
294 - logit_scale = getattr(config, "logit_scale", 1.0)
295 - self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
296 - config.text_config.vocab_size,
297 - logit_scale)
298 - self.sampler = get_sampler()
275 + self.language_model = init_vllm_registered_model(
276 + vllm_config=vllm_config,
277 + hf_config=config.text_config,
278 + prefix=maybe_prefix(prefix, "language_model"),
279 + architectures=["Qwen2ForCausalLM"],
280 + )
281
282 self.make_empty_intermediate_tensors = (
283 self.language_model.make_empty_intermediate_tensors)
284
285 + @cached_property
286 + def sampler(self):
287 + if hasattr(self.language_model, "sampler"):
288 + return self.language_model.sampler
289 +
290 + return get_sampler()
291 +
292 def _validate_and_reshape_mm_tensor(self,
293 mm_input: Union[torch.Tensor,
294 List[torch.Tensor]],
@@ -414,72 +403,30 @@ class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
403 multimodal_embeddings)
404 input_ids = None
405
417 - hidden_states = self.language_model(input_ids,
418 - positions,
419 - kv_caches,
420 - attn_metadata,
421 - intermediate_tensors,
422 - inputs_embeds=inputs_embeds)
406 + hidden_states = self.language_model.model(input_ids,
407 + positions,
408 + kv_caches,
409 + attn_metadata,
410 + intermediate_tensors,
411 + inputs_embeds=inputs_embeds)
412 return hidden_states
413
425 - def compute_logits(self, hidden_states: torch.Tensor,
426 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
427 - logits = self.logits_processor(self.lm_head, hidden_states,
428 - sampling_metadata)
429 - return logits
414 + def compute_logits(
415 + self,
416 + hidden_states: torch.Tensor,
417 + sampling_metadata: SamplingMetadata,
418 + ) -> Optional[torch.Tensor]:
419 + return self.language_model.compute_logits(hidden_states,
420 + sampling_metadata)
421
422 def sample(
423 self,
424 logits: torch.Tensor,
425 sampling_metadata: SamplingMetadata,
426 ) -> Optional[SamplerOutput]:
436 - next_tokens = self.sampler(logits, sampling_metadata)
437 - return next_tokens
427 + return self.language_model.sample(logits, sampling_metadata)
428
429 def load_weights(self, weights: Iterable[Tuple[str,
430 torch.Tensor]]) -> Set[str]:
441 - stacked_params_mapping = [
442 - # (param_name, shard_name, shard_id)
443 - ("qkv_proj", "q_proj", "q"),
444 - ("qkv_proj", "k_proj", "k"),
445 - ("qkv_proj", "v_proj", "v"),
446 - ("gate_up_proj", "gate_proj", 0),
447 - ("gate_up_proj", "up_proj", 1),
448 - ]
449 - params_dict = dict(self.named_parameters(remove_duplicate=False))
450 - loaded_params: Set[str] = set()
451 - for name, loaded_weight in weights:
452 - if "rotary_emb.inv_freq" in name:
453 - continue
454 - if (self.config.text_config.tie_word_embeddings
455 - and "lm_head.weight" in name):
456 - continue
457 - for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
458 - if key_to_modify in name:
459 - name = name.replace(key_to_modify, new_key)
460 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
461 - if weight_name not in name or 'audio' in name:
462 - continue
463 - name = name.replace(weight_name, param_name)
464 - # Skip loading extra bias for GPTQ models.
465 - if name.endswith(".bias") and name not in params_dict:
466 - continue
467 - param = params_dict[name]
468 - weight_loader = param.weight_loader
469 - weight_loader(param, loaded_weight, shard_id)
470 - break
471 - else:
472 - # Skip loading extra bias for GPTQ models.
473 - if name.endswith(".bias") and name not in params_dict:
474 - continue
475 - # Remapping the name of FP8 kv-scale.
476 - name = maybe_remap_kv_scale_name(name, params_dict)
477 - if name is None:
478 - continue
479 -
480 - param = params_dict[name]
481 - weight_loader = getattr(param, "weight_loader",
482 - default_weight_loader)
483 - weight_loader(param, loaded_weight)
484 - loaded_params.add(name)
485 - return loaded_params
431 + loader = AutoWeightsLoader(self)
432 + return loader.load_weights(weights)
vllm/model_executor/models/qwen2_vl.py
+83 -96
@@ -21,7 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24 -from functools import partial
24 +from functools import cached_property, partial
25 from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
26 Optional, Set, Tuple, Type, TypedDict, Union)
27
@@ -40,7 +40,7 @@ from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
41 from vllm.attention import AttentionMetadata
42 from vllm.config import VllmConfig
43 -from vllm.distributed import get_pp_group, parallel_state
43 +from vllm.distributed import parallel_state
44 from vllm.distributed import utils as dist_utils
45 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
46 InputContext, token_inputs)
@@ -49,15 +49,12 @@ from vllm.model_executor import SamplingMetadata
49 from vllm.model_executor.layers.activation import QuickGELU
50 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
51 RowParallelLinear)
52 -from vllm.model_executor.layers.logits_processor import LogitsProcessor
52 from vllm.model_executor.layers.quantization import QuantizationConfig
53 from vllm.model_executor.layers.quantization.gptq import GPTQConfig
54 from vllm.model_executor.layers.quantization.gptq_marlin import (
55 GPTQMarlinConfig)
56 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
58 -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
57 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
60 -from vllm.model_executor.models.qwen2 import Qwen2Model
58 from vllm.multimodal import MULTIMODAL_REGISTRY
59 from vllm.multimodal.image import cached_get_image_processor
60 from vllm.multimodal.inputs import (MultiModalData, MultiModalDataDict,
@@ -69,9 +66,8 @@ from vllm.transformers_utils.config import uses_mrope
66 from vllm.transformers_utils.processor import cached_get_processor
67
68 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
72 -from .utils import (PPMissingLayer, get_vit_attn_backend,
73 - is_pp_missing_parameter,
74 - make_empty_intermediate_tensors_factory, maybe_prefix)
69 +from .utils import (AutoWeightsLoader, WeightsMapper, get_vit_attn_backend,
70 + init_vllm_registered_model, maybe_prefix)
71
72 logger = init_logger(__name__)
73
@@ -506,6 +502,8 @@ class Qwen2VisionTransformer(nn.Module):
502 mlp_ratio: float = vision_config.mlp_ratio
503
504 self.spatial_merge_size = spatial_merge_size
505 + self.num_heads = num_heads
506 + self.embed_dim = embed_dim
507
508 self.patch_embed = Qwen2VisionPatchEmbed(
509 patch_size=patch_size,
@@ -595,6 +593,53 @@ class Qwen2VisionTransformer(nn.Module):
593 x = self.merger(x)
594 return x
595
596 + def load_weights(self, weights: Iterable[Tuple[str,
597 + torch.Tensor]]) -> Set[str]:
598 + stacked_params_mapping = [
599 + # (param_name, shard_name, shard_id)
600 + ("qkv_proj", "q_proj", "q"),
601 + ("qkv_proj", "k_proj", "k"),
602 + ("qkv_proj", "v_proj", "v"),
603 + ]
604 + params_dict = dict(self.named_parameters(remove_duplicate=False))
605 + loaded_params: Set[str] = set()
606 +
607 + for name, loaded_weight in weights:
608 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
609 + if weight_name not in name:
610 + continue
611 + name = name.replace(weight_name, param_name)
612 +
613 + param = params_dict[name]
614 + weight_loader = param.weight_loader
615 + weight_loader(param, loaded_weight, shard_id)
616 + break
617 + else:
618 + if name.endswith("qkv.weight"):
619 + visual_num_heads = self.num_heads
620 + visual_embed_dim = self.embed_dim
621 + head_size = visual_embed_dim // visual_num_heads
622 + loaded_weight = loaded_weight.view(3, visual_num_heads,
623 + head_size,
624 + visual_embed_dim)
625 + loaded_weight = loaded_weight.transpose(0, 1)
626 + loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
627 + elif name.endswith("qkv.bias"):
628 + visual_num_heads = self.num_heads
629 + visual_embed_dim = self.embed_dim
630 + head_size = visual_embed_dim // visual_num_heads
631 + loaded_weight = loaded_weight.view(3, visual_num_heads,
632 + head_size)
633 + loaded_weight = loaded_weight.transpose(0, 1)
634 + loaded_weight = loaded_weight.reshape(-1)
635 +
636 + param = params_dict[name]
637 + weight_loader = getattr(param, "weight_loader",
638 + default_weight_loader)
639 + weight_loader(param, loaded_weight)
640 + loaded_params.add(name)
641 + return loaded_params
642 +
643
644 # === Vision input helpers === #
645
@@ -1082,27 +1127,21 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1127 prefix=maybe_prefix(prefix, "visual"),
1128 )
1129
1085 - self.model = Qwen2Model(vllm_config=vllm_config,
1086 - prefix=maybe_prefix(prefix, "model"))
1130 + self.language_model = init_vllm_registered_model(
1131 + vllm_config=vllm_config,
1132 + prefix=maybe_prefix(prefix, "language_model"),
1133 + architectures=["Qwen2ForCausalLM"],
1134 + )
1135
1088 - if get_pp_group().is_last_rank:
1089 - if config.tie_word_embeddings:
1090 - self.lm_head = self.model.embed_tokens
1091 - else:
1092 - self.lm_head = ParallelLMHead(config.vocab_size,
1093 - config.hidden_size,
1094 - quant_config=quant_config,
1095 - prefix=maybe_prefix(
1096 - prefix, "lm_head"))
1097 - else:
1098 - self.lm_head = PPMissingLayer()
1136 + self.make_empty_intermediate_tensors = (
1137 + self.language_model.make_empty_intermediate_tensors)
1138
1100 - self.logits_processor = LogitsProcessor(config.vocab_size)
1101 - self.sampler = get_sampler()
1139 + @cached_property
1140 + def sampler(self):
1141 + if hasattr(self.language_model, "sampler"):
1142 + return self.language_model.sampler
1143
1103 - self.make_empty_intermediate_tensors = (
1104 - make_empty_intermediate_tensors_factory(
1105 - ["hidden_states", "residual"], config.hidden_size))
1144 + return get_sampler()
1145
1146 def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1147 # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
@@ -1261,7 +1300,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1300 multimodal_embeddings: Optional[List[Tuple[NestedTensors,
1301 str]]] = None,
1302 ) -> torch.Tensor:
1264 - inputs_embeds = self.model.get_input_embeddings(input_ids)
1303 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1304 if multimodal_embeddings is not None:
1305 for embeddings, modality in multimodal_embeddings:
1306 if modality == "image":
@@ -1330,7 +1369,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1369 multimodal_embeddings)
1370 input_ids = None
1371
1333 - hidden_states = self.model(
1372 + hidden_states = self.language_model.model(
1373 input_ids=input_ids,
1374 positions=positions,
1375 kv_caches=kv_caches,
@@ -1340,80 +1379,28 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1379 )
1380 return hidden_states
1381
1343 - def compute_logits(self, hidden_states: torch.Tensor,
1344 - sampling_metadata: SamplingMetadata) -> torch.Tensor:
1345 - logits = self.logits_processor(self.lm_head, hidden_states,
1346 - sampling_metadata)
1347 - return logits
1382 + def compute_logits(
1383 + self,
1384 + hidden_states: torch.Tensor,
1385 + sampling_metadata: SamplingMetadata,
1386 + ) -> Optional[torch.Tensor]:
1387 + return self.language_model.compute_logits(hidden_states,
1388 + sampling_metadata)
1389
1390 def sample(
1391 self,
1392 logits: torch.Tensor,
1393 sampling_metadata: SamplingMetadata,
1394 ) -> Optional[SamplerOutput]:
1354 - next_tokens = self.sampler(logits, sampling_metadata)
1355 - return next_tokens
1395 + return self.language_model.sample(logits, sampling_metadata)
1396
1397 def load_weights(self, weights: Iterable[Tuple[str,
1398 torch.Tensor]]) -> Set[str]:
1359 - stacked_params_mapping = [
1360 - # (param_name, shard_name, shard_id)
1361 - ("qkv_proj", "q_proj", "q"),
1362 - ("qkv_proj", "k_proj", "k"),
1363 - ("qkv_proj", "v_proj", "v"),
1364 - ("gate_up_proj", "up_proj", 1),
1365 - ("gate_up_proj", "gate_proj", 0),
1366 - ]
1367 - params_dict = dict(self.named_parameters(remove_duplicate=False))
1368 - loaded_params: Set[str] = set()
1369 - for name, loaded_weight in weights:
1370 - if "rotary_emb.inv_freq" in name:
1371 - continue
1372 - if self.config.tie_word_embeddings and "lm_head.weight" in name:
1373 - continue
1374 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
1375 - if weight_name not in name:
1376 - continue
1377 - name = name.replace(weight_name, param_name)
1378 - # Skip loading extra bias for GPTQ models.
1379 - if name.endswith(".bias") and name not in params_dict:
1380 - continue
1381 - if is_pp_missing_parameter(name, self):
1382 - continue
1383 - param = params_dict[name]
1384 - weight_loader = param.weight_loader
1385 - weight_loader(param, loaded_weight, shard_id)
1386 - break
1387 - else:
1388 - if "visual" in name and name.endswith("qkv.weight"):
1389 - visual_num_heads = self.config.vision_config.num_heads
1390 - visual_embed_dim = self.config.vision_config.embed_dim
1391 - head_size = visual_embed_dim // visual_num_heads
1392 - loaded_weight = loaded_weight.view(3, visual_num_heads,
1393 - head_size,
1394 - visual_embed_dim)
1395 - loaded_weight = loaded_weight.transpose(0, 1)
1396 - loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
1397 - elif "visual" in name and name.endswith("qkv.bias"):
1398 - visual_num_heads = self.config.vision_config.num_heads
1399 - visual_embed_dim = self.config.vision_config.embed_dim
1400 - head_size = visual_embed_dim // visual_num_heads
1401 - loaded_weight = loaded_weight.view(3, visual_num_heads,
1402 - head_size)
1403 - loaded_weight = loaded_weight.transpose(0, 1)
1404 - loaded_weight = loaded_weight.reshape(-1)
1405 - try:
1406 - # Skip loading extra bias for GPTQ models.
1407 - if name.endswith(".bias") and name not in params_dict:
1408 - continue
1409 - if is_pp_missing_parameter(name, self):
1410 - continue
1411 - param = params_dict[name]
1412 - except KeyError:
1413 - raise ValueError(f"Unexpected weight: {name}") from None
1414 -
1415 - weight_loader = getattr(param, "weight_loader",
1416 - default_weight_loader)
1417 - weight_loader(param, loaded_weight)
1418 - loaded_params.add(name)
1419 - return loaded_params
1399 + hf_to_vllm_mapper = WeightsMapper(
1400 + orig_to_new_prefix={
1401 + "lm_head.": "language_model.lm_head.",
1402 + "model.": "language_model.model.",
1403 + })
1404 +
1405 + loader = AutoWeightsLoader(self)
1406 + return loader.load_weights(weights, mapper=hf_to_vllm_mapper)
vllm/model_executor/models/utils.py
+9 -6
@@ -17,7 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.multimodal import MultiModalPlaceholderMap, NestedTensors
18 from vllm.platforms import _Backend, current_platform
19 from vllm.sequence import IntermediateTensors
20 -from vllm.utils import is_pin_memory_available
20 +from vllm.utils import is_pin_memory_available, print_warning_once
21
22 logger = init_logger(__name__)
23
@@ -251,12 +251,15 @@ def init_vllm_registered_model(
251 """
252 from vllm.model_executor.model_loader.loader import _initialize_model
253
254 + if hf_config is None and architectures is not None:
255 + # So that the architectures field is overridden
256 + hf_config = vllm_config.model_config.hf_config
257 +
258 if hf_config is not None:
255 - vllm_config = vllm_config.with_hf_config(hf_config)
259 + vllm_config = vllm_config.with_hf_config(hf_config,
260 + architectures=architectures)
261
257 - return _initialize_model(vllm_config=vllm_config,
258 - prefix=prefix,
259 - architectures=architectures)
262 + return _initialize_model(vllm_config=vllm_config, prefix=prefix)
263
264
265 @overload
@@ -592,7 +595,7 @@ def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
595 if is_flash_attn_2_available():
596 selected_backend = _Backend.FLASH_ATTN
597 else:
595 - logger.warning(
598 + print_warning_once(
599 "Current `vllm-flash-attn` has a bug inside vision module, "
600 "so we use xformers backend instead. You can run "
601 "`pip install flash-attn` to use flash-attention backend.")