A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Encoder Decoder] Update Mllama to run with both FlashAttention and XFormers (#9982)
Signed-off-by: Sourashis Roy <sroy@roblox.com>
sroy745 committed
Nov 12, 2024 at 10:53 UTC
b41fb9d3b10dcf187ac0501ca80ede96d387617f
5 files changed
+117
-80
tests/encoder_decoder/test_e2e_correctness.py
+8
-1
@@ -7,7 +7,7 @@ from typing import List, Optional, Tuple
7
import pytest
8
from transformers import AutoModelForSeq2SeqLM
9
10
-from vllm.attention.selector import (_Backend,
10
+from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
11
global_force_attn_backend_context_manager)
12
from vllm.platforms import current_platform
13
from vllm.sequence import SampleLogprobs
@@ -34,6 +34,13 @@ def vllm_to_hf_output(
34
return output_ids, hf_output_str, out_logprobs
35
36
37
+@pytest.fixture(autouse=True)
38
+def clear_cache():
39
+ """Fixture to clear backend cache before each test."""
40
+ _cached_get_attn_backend.cache_clear() # Clear the cache
41
+ yield # This allows the test to run
42
+
43
+
44
@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
45
@pytest.mark.parametrize("dtype", ["float"])
46
@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
tests/models/encoder_decoder/vision_language/test_mllama.py
+63
-37
@@ -4,6 +4,8 @@ import pytest
4
from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5
BatchEncoding)
6
7
+from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
8
+ global_force_attn_backend_context_manager)
9
from vllm.multimodal.utils import rescale_image_size
10
from vllm.sequence import SampleLogprobs
11
@@ -14,6 +16,8 @@ from ...utils import check_logprobs_close
16
17
_LIMIT_IMAGE_PER_PROMPT = 3
18
19
+LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
20
+
21
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
22
"stop_sign":
23
"<|image|><|begin_of_text|>The meaning of the image is",
@@ -221,6 +225,13 @@ def _run_test(
225
)
226
227
228
+@pytest.fixture(autouse=True)
229
+def clear_cache():
230
+ """Fixture to clear backend cache before each test."""
231
+ _cached_get_attn_backend.cache_clear() # Clear the cache
232
+ yield # This allows the test to run
233
+
234
+
235
@large_gpu_test(min_gb=48)
236
@pytest.mark.parametrize("model", models)
237
@pytest.mark.parametrize(
@@ -244,20 +255,26 @@ def _run_test(
255
@pytest.mark.parametrize("dtype", ["bfloat16"])
256
@pytest.mark.parametrize("max_tokens", [128])
257
@pytest.mark.parametrize("num_logprobs", [5])
258
+@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
259
def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
260
model, sizes, dtype, max_tokens,
249
- num_logprobs) -> None:
250
- run_test(
251
- hf_runner,
252
- vllm_runner,
253
- image_assets,
254
- model,
255
- sizes=sizes,
256
- dtype=dtype,
257
- max_tokens=max_tokens,
258
- num_logprobs=num_logprobs,
259
- tensor_parallel_size=1,
260
- )
261
+ num_logprobs,
262
+ attn_backend: _Backend) -> None:
263
+ with global_force_attn_backend_context_manager(attn_backend):
264
+ if attn_backend == _Backend.FLASH_ATTN:
265
+ # Flash Attention works only with bfloat16 data-type
266
+ dtype = 'bfloat16'
267
+ run_test(
268
+ hf_runner,
269
+ vllm_runner,
270
+ image_assets,
271
+ model,
272
+ sizes=sizes,
273
+ dtype=dtype,
274
+ max_tokens=max_tokens,
275
+ num_logprobs=num_logprobs,
276
+ tensor_parallel_size=1,
277
+ )
278
279
280
@large_gpu_test(min_gb=48)
@@ -265,9 +282,10 @@ def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
282
@pytest.mark.parametrize("dtype", ["bfloat16"])
283
@pytest.mark.parametrize("max_tokens", [128])
284
@pytest.mark.parametrize("num_logprobs", [5])
285
+@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
286
def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
269
- model, dtype, max_tokens,
270
- num_logprobs) -> None:
287
+ model, dtype, max_tokens, num_logprobs,
288
+ attn_backend: _Backend) -> None:
289
290
stop_sign = image_assets[0].pil_image
291
cherry_blossom = image_assets[1].pil_image
@@ -291,17 +309,20 @@ def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
309
cherry_blossom.resize((512, 1024)),
310
],
311
])]
294
-
295
- _run_test(
296
- hf_runner,
297
- vllm_runner,
298
- inputs,
299
- model,
300
- dtype=dtype,
301
- max_tokens=max_tokens,
302
- num_logprobs=num_logprobs,
303
- tensor_parallel_size=1,
304
- )
312
+ with global_force_attn_backend_context_manager(attn_backend):
313
+ if attn_backend == _Backend.FLASH_ATTN:
314
+ # Flash Attention works only with bfloat16 data-type
315
+ dtype = 'bfloat16'
316
+ _run_test(
317
+ hf_runner,
318
+ vllm_runner,
319
+ inputs,
320
+ model,
321
+ dtype=dtype,
322
+ max_tokens=max_tokens,
323
+ num_logprobs=num_logprobs,
324
+ tensor_parallel_size=1,
325
+ )
326
327
328
@large_gpu_test(min_gb=48)
@@ -309,8 +330,10 @@ def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
330
@pytest.mark.parametrize("dtype", ["bfloat16"])
331
@pytest.mark.parametrize("max_tokens", [128])
332
@pytest.mark.parametrize("num_logprobs", [5])
333
+@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
334
def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
313
- dtype, max_tokens, num_logprobs) -> None:
335
+ dtype, max_tokens, num_logprobs,
336
+ attn_backend: _Backend) -> None:
337
338
stop_sign = image_assets[0].pil_image
339
cherry_blossom = image_assets[1].pil_image
@@ -325,14 +348,17 @@ def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
348
[stop_sign],
349
[stop_sign, cherry_blossom],
350
])]
328
-
329
- _run_test(
330
- hf_runner,
331
- vllm_runner,
332
- inputs,
333
- model,
334
- dtype=dtype,
335
- max_tokens=max_tokens,
336
- num_logprobs=num_logprobs,
337
- tensor_parallel_size=1,
338
- )
351
+ with global_force_attn_backend_context_manager(attn_backend):
352
+ if attn_backend == _Backend.FLASH_ATTN:
353
+ # Flash Attention works only with bfloat16 data-type
354
+ dtype = 'bfloat16'
355
+ _run_test(
356
+ hf_runner,
357
+ vllm_runner,
358
+ inputs,
359
+ model,
360
+ dtype=dtype,
361
+ max_tokens=max_tokens,
362
+ num_logprobs=num_logprobs,
363
+ tensor_parallel_size=1,
364
+ )
tests/test_config.py
+2
@@ -243,6 +243,8 @@ def test_rope_customization():
243
assert longchat_model_config.max_model_len == 4096
244
245
246
+@pytest.mark.skipif(current_platform.is_rocm(),
247
+ reason="Encoder Decoder models not supported on ROCm.")
248
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
249
("facebook/opt-125m", False),
250
("facebook/bart-base", True),
vllm/model_executor/models/mllama.py
+38
-14
@@ -32,6 +32,8 @@ from transformers.models.mllama.processing_mllama import (
32
33
import vllm.distributed.parallel_state as ps
34
from vllm.attention import Attention, AttentionMetadata, AttentionType
35
+from vllm.attention.backends.flash_attn import FlashAttentionMetadata
36
+from vllm.attention.backends.xformers import XFormersMetadata
37
from vllm.attention.ops.paged_attn import PagedAttention
38
from vllm.config import VllmConfig
39
from vllm.distributed import get_tensor_model_parallel_world_size
@@ -799,12 +801,13 @@ class MllamaTextCrossAttention(nn.Module):
801
q = self.q_norm(q)
802
803
if attention_mask is not None:
802
- output = self.attention_with_mask(q, k, v, kv_cache,
803
- attention_mask,
804
- kv_range_for_decode,
805
- attn_metadata)
804
+ output = self._attention_with_mask(q, k, v, kv_cache,
805
+ attention_mask,
806
+ kv_range_for_decode,
807
+ attn_metadata)
808
else:
807
- output = self.attn(q,
809
+ output = self.attn(q.view(-1,
810
+ self.num_local_heads * self.head_dim),
811
k,
812
v,
813
kv_cache,
@@ -813,7 +816,7 @@ class MllamaTextCrossAttention(nn.Module):
816
out, _ = self.o_proj(output)
817
return out
818
816
- def attention_with_mask(
819
+ def _attention_with_mask(
820
self,
821
q: torch.Tensor,
822
k: torch.Tensor,
@@ -824,14 +827,35 @@ class MllamaTextCrossAttention(nn.Module):
827
attn_metadata: AttentionMetadata,
828
) -> torch.Tensor:
829
# Skip writing kv-cache for the initial profiling run.
827
- if len(kv_cache.shape) == 3:
828
- key_cache, value_cache = PagedAttention.split_kv_cache(
829
- kv_cache, self.num_local_key_value_heads, self.head_dim)
830
- cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
831
- cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
832
- PagedAttention.write_to_paged_cache(
833
- cached_k, cached_v, key_cache, value_cache,
834
- attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
830
+ if len(kv_cache.shape) > 1:
831
+ if isinstance(attn_metadata, FlashAttentionMetadata):
832
+ cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
833
+ cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
834
+ torch.ops._C_cache_ops.reshape_and_cache_flash(
835
+ cached_k,
836
+ cached_v,
837
+ kv_cache[0],
838
+ kv_cache[1],
839
+ attn_metadata.
840
+ cross_slot_mapping, # type: ignore[union-attr]
841
+ "auto",
842
+ 1.0,
843
+ 1.0,
844
+ )
845
+ elif isinstance(attn_metadata, XFormersMetadata):
846
+ key_cache, value_cache = PagedAttention.split_kv_cache(
847
+ kv_cache, self.num_local_key_value_heads, self.head_dim)
848
+ cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
849
+ cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
850
+ PagedAttention.write_to_paged_cache(
851
+ cached_k, cached_v, key_cache, value_cache,
852
+ attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
853
+ else:
854
+ raise ValueError(
855
+ f"Unsupported AttentionMetadata {type(attn_metadata)} "
856
+ f"class found. Expected the AttentionMetadata to "
857
+ f"be either XFormersMetadata or FlashAttentionMetadata.")
858
+
859
# We have to call torch.sdpa for prefill when using a
860
# custom cross-attention mask. Because the mask is not a
861
# standard causal mask, neither a block diagonal mask which
vllm/worker/enc_dec_model_runner.py
+6
-28
@@ -9,15 +9,13 @@ from vllm.attention.backends.abstract import (AttentionBackend,
9
AttentionMetadata)
10
from vllm.attention.backends.utils import PAD_SLOT_ID
11
from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
12
- get_global_forced_attn_backend,
13
- global_force_attn_backend)
14
-from vllm.config import ModelConfig, VllmConfig
12
+ get_global_forced_attn_backend)
13
+from vllm.config import VllmConfig
14
from vllm.forward_context import set_forward_context
15
from vllm.inputs import INPUT_REGISTRY, InputRegistry
16
from vllm.logger import init_logger
17
from vllm.model_executor import SamplingMetadata
18
from vllm.model_executor.layers.sampler import SamplerOutput
20
-from vllm.model_executor.model_loader.utils import get_architecture_class_name
19
from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
20
MultiModalRegistry)
21
from vllm.sampling_params import SamplingParams
@@ -35,11 +33,6 @@ from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
33
34
logger = init_logger(__name__)
35
38
-# The Mllama model has PagedAttention specific logic because of which it
39
-# can only be run with the XFORMERS backend
40
-# TODO Make Mllama model work with Flash Attention backend.
41
-_XFORMERS_ONLY_ENCODER_DECODER_ARCHS = ["MllamaForConditionalGeneration"]
42
-
36
37
@dataclasses.dataclass(frozen=True)
38
class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
@@ -97,7 +90,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
90
models) but these arguments are present here for compatibility with
91
the base-class constructor.
92
'''
100
- self._maybe_force_supported_attention_backend(vllm_config.model_config)
93
+ self._maybe_force_supported_attention_backend()
94
95
super().__init__(
96
vllm_config=vllm_config,
@@ -108,12 +101,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
101
# Crash for unsupported encoder/scenarios
102
assert_enc_dec_mr_supported_scenario(self)
103
111
- def _is_xformers_only_encoder_decoder_model(self,
112
- model: ModelConfig) -> bool:
113
- return get_architecture_class_name(
114
- model) in _XFORMERS_ONLY_ENCODER_DECODER_ARCHS
115
-
116
- def _maybe_force_supported_attention_backend(self, model: ModelConfig):
104
+ def _maybe_force_supported_attention_backend(self):
105
'''
106
Force vLLM to use the XFormers attention backend,
107
which is currently the only supported option.
@@ -128,23 +116,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
116
maybe_global_forced_backend = get_global_forced_attn_backend()
117
is_forced_by_global = maybe_global_forced_backend is not None
118
is_forced_by_env_var = maybe_env_var_forced_backend is not None
131
-
132
- if not (is_forced_by_global or is_forced_by_env_var) \
133
- and self._is_xformers_only_encoder_decoder_model(model):
134
- # The user has not already specified an attention backend
135
- # override
136
- logger.info(
137
- "Encoder-Decoder Model Architecture %s requires XFormers "
138
- "backend; overriding backend auto-selection and "
139
- "forcing XFormers.", get_architecture_class_name(model))
140
- global_force_attn_backend(_Backend.XFORMERS)
141
- elif is_forced_by_global:
119
+ if is_forced_by_global: # noqa: SIM102
120
# Backend override enforced by global variable takes
121
# precedence over vLLM backend environment variable.
122
if maybe_global_forced_backend not in\
123
[_Backend.XFORMERS, _Backend.FLASH_ATTN]:
124
raise_backend_err()
147
- elif is_forced_by_env_var:
125
+ elif is_forced_by_env_var: # noqa: SIM102
126
# Backend override enforced by vLLM backend
127
# environment variable
128
if maybe_env_var_forced_backend not in\