@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ed46f14321
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Support `is_causal` HF config field for Qwen2 model (#10621)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 25, 2024 at 17:51 UTC ed46f143212203b7afcbc8538119b6e8155c643e
5 files changed +51 -13
docs/source/models/supported_models.rst
+10 -3
@@ -342,7 +342,7 @@ Text Embedding
342 - ✅︎
343 * - :code:`Qwen2Model`, :code:`Qwen2ForCausalLM`
344 - Qwen2-based
345 - - :code:`ssmits/Qwen2-7B-Instruct-embed-base`, :code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`, etc.
345 + - :code:`ssmits/Qwen2-7B-Instruct-embed-base`, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc.
346 - ✅︎
347 - ✅︎
348 * - :code:`RobertaModel`, :code:`RobertaForMaskedLM`
@@ -363,6 +363,13 @@ Text Embedding
363 .. tip::
364 You can override the model's pooling method by passing :code:`--override-pooler-config`.
365
366 +.. note::
367 + Unlike base Qwen2, :code:`Alibaba-NLP/gte-Qwen2-7B-instruct` uses bi-directional attention.
368 + You can set `--hf-overrides '{"is_causal": false}'` to change the attention mask accordingly.
369 +
370 + On the other hand, its 1.5B variant (:code:`Alibaba-NLP/gte-Qwen2-1.5B-instruct`) uses causal attention
371 + despite being described otherwise on its model card.
372 +
373 Reward Modeling
374 ---------------
375
@@ -606,10 +613,10 @@ Text Generation
613 | :sup:`+` Multiple items can be inputted per text prompt for this modality.
614
615 .. note::
609 - vLLM currently only supports adding LoRA to the language backbone of multimodal models.
616 + vLLM currently only supports adding LoRA to the language backbone of multimodal models.
617
618 .. note::
612 - For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
619 + The official :code:`openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
620 For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
621
622 Multimodal Embedding
tests/models/embedding/language/test_embedding.py
+10 -2
@@ -21,6 +21,7 @@ from ..utils import check_embeddings_close
21 marks=[pytest.mark.core_model]),
22 pytest.param("ssmits/Qwen2-7B-Instruct-embed-base"),
23 pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
24 + pytest.param("Alibaba-NLP/gte-Qwen2-7B-instruct"),
25 ],
26 )
27 @pytest.mark.parametrize("dtype", ["half"])
@@ -31,6 +32,10 @@ def test_models(
32 model,
33 dtype: str,
34 ) -> None:
35 + vllm_extra_kwargs = {}
36 + if model == "Alibaba-NLP/gte-Qwen2-7B-instruct":
37 + vllm_extra_kwargs["hf_overrides"] = {"is_causal": False}
38 +
39 # The example_prompts has ending "\n", for example:
40 # "Write a short story about a robot that dreams for the first time.\n"
41 # sentence_transformers will strip the input texts, see:
@@ -43,8 +48,11 @@ def test_models(
48 is_sentence_transformer=True) as hf_model:
49 hf_outputs = hf_model.encode(example_prompts)
50
46 - with vllm_runner(model, task="embedding", dtype=dtype,
47 - max_model_len=None) as vllm_model:
51 + with vllm_runner(model,
52 + task="embedding",
53 + dtype=dtype,
54 + max_model_len=None,
55 + **vllm_extra_kwargs) as vllm_model:
56 vllm_outputs = vllm_model.encode(example_prompts)
57 # This test is for verifying whether the model's extra_repr
58 # can be printed correctly.
tests/models/embedding/utils.py
+2 -2
@@ -24,7 +24,7 @@ def check_embeddings_close(
24 dim=0)
25
26 fail_msg = (f"Test{prompt_idx}:"
27 - f"\n{name_0}:\t{embeddings_0!r}"
28 - f"\n{name_1}:\t{embeddings_1!r}")
27 + f"\n{name_0}:\t{embeddings_0[:16]!r}"
28 + f"\n{name_1}:\t{embeddings_1[:16]!r}")
29
30 assert sim >= 1 - tol, fail_msg
vllm/config.py
+11 -4
@@ -27,7 +27,7 @@ from vllm.transformers_utils.config import (
27 get_hf_text_config, get_pooling_config,
28 get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
29 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
30 - identity, print_warning_once, resolve_obj_by_qualname)
30 + print_warning_once, resolve_obj_by_qualname)
31
32 if TYPE_CHECKING:
33 from ray.util.placement_group import PlacementGroup
@@ -183,7 +183,7 @@ class ModelConfig:
183 hf_overrides_fn = hf_overrides
184 else:
185 hf_overrides_kw = hf_overrides
186 - hf_overrides_fn = identity
186 + hf_overrides_fn = None
187
188 if rope_scaling is not None:
189 hf_override: Dict[str, Any] = {"rope_scaling": rope_scaling}
@@ -212,8 +212,15 @@ class ModelConfig:
212 self.skip_tokenizer_init = skip_tokenizer_init
213
214 hf_config = get_config(self.model, trust_remote_code, revision,
215 - code_revision, config_format, **hf_overrides_kw)
216 - hf_config = hf_overrides_fn(hf_config)
215 + code_revision, config_format)
216 +
217 + if hf_overrides_kw:
218 + logger.info("Overriding HF config with %s", hf_overrides_kw)
219 + hf_config.update(hf_overrides_kw)
220 + if hf_overrides_fn:
221 + logger.info("Overriding HF config with %s", hf_overrides_fn)
222 + hf_config = hf_overrides_fn(hf_config)
223 +
224 self.hf_config = hf_config
225
226 self.hf_text_config = get_hf_text_config(self.hf_config)
vllm/model_executor/models/qwen2.py
+18 -2
@@ -27,7 +27,7 @@ import torch
27 from torch import nn
28 from transformers import Qwen2Config
29
30 -from vllm.attention import Attention, AttentionMetadata
30 +from vllm.attention import Attention, AttentionMetadata, AttentionType
31 from vllm.compilation.decorators import support_torch_compile
32 from vllm.config import CacheConfig, VllmConfig
33 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
@@ -164,11 +164,17 @@ class Qwen2Attention(nn.Module):
164 hidden_states: torch.Tensor,
165 kv_cache: torch.Tensor,
166 attn_metadata: AttentionMetadata,
167 + attn_type: str = AttentionType.DECODER,
168 ) -> torch.Tensor:
169 qkv, _ = self.qkv_proj(hidden_states)
170 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
171 q, k = self.rotary_emb(positions, q, k)
171 - attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
172 + attn_output = self.attn(q,
173 + k,
174 + v,
175 + kv_cache,
176 + attn_metadata,
177 + attn_type=attn_type)
178 output, _ = self.o_proj(attn_output)
179 return output
180
@@ -210,6 +216,15 @@ class Qwen2DecoderLayer(nn.Module):
216 self.post_attention_layernorm = RMSNorm(config.hidden_size,
217 eps=config.rms_norm_eps)
218
219 + # By default, Qwen2 uses causal attention as it is a decoder-only model.
220 + # You can override the HF config with `is_causal=False` to enable
221 + # bidirectional attention, which is used in some embedding models
222 + # (e.g. Alibaba-NLP/gte-Qwen2-7B-instruct)
223 + if getattr(config, "is_causal", True):
224 + self._attn_type = AttentionType.DECODER
225 + else:
226 + self._attn_type = AttentionType.ENCODER_ONLY
227 +
228 def forward(
229 self,
230 positions: torch.Tensor,
@@ -230,6 +245,7 @@ class Qwen2DecoderLayer(nn.Module):
245 hidden_states=hidden_states,
246 kv_cache=kv_cache,
247 attn_metadata=attn_metadata,
248 + attn_type=self._attn_type,
249 )
250
251 # Fully Connected