@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b41fb9d3b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Encoder Decoder] Update Mllama to run with both FlashAttention and XFormers (#9982)

Signed-off-by: Sourashis Roy <sroy@roblox.com>

sroy745 committed Nov 12, 2024 at 10:53 UTC b41fb9d3b10dcf187ac0501ca80ede96d387617f
5 files changed +117 -80
tests/encoder_decoder/test_e2e_correctness.py
+8 -1
@@ -7,7 +7,7 @@ from typing import List, Optional, Tuple
7 import pytest
8 from transformers import AutoModelForSeq2SeqLM
9
10 -from vllm.attention.selector import (_Backend,
10 +from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
11 global_force_attn_backend_context_manager)
12 from vllm.platforms import current_platform
13 from vllm.sequence import SampleLogprobs
@@ -34,6 +34,13 @@ def vllm_to_hf_output(
34 return output_ids, hf_output_str, out_logprobs
35
36
37 +@pytest.fixture(autouse=True)
38 +def clear_cache():
39 + """Fixture to clear backend cache before each test."""
40 + _cached_get_attn_backend.cache_clear() # Clear the cache
41 + yield # This allows the test to run
42 +
43 +
44 @pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
45 @pytest.mark.parametrize("dtype", ["float"])
46 @pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
tests/models/encoder_decoder/vision_language/test_mllama.py
+63 -37
@@ -4,6 +4,8 @@ import pytest
4 from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5 BatchEncoding)
6
7 +from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
8 + global_force_attn_backend_context_manager)
9 from vllm.multimodal.utils import rescale_image_size
10 from vllm.sequence import SampleLogprobs
11
@@ -14,6 +16,8 @@ from ...utils import check_logprobs_close
16
17 _LIMIT_IMAGE_PER_PROMPT = 3
18
19 +LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
20 +
21 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
22 "stop_sign":
23 "<|image|><|begin_of_text|>The meaning of the image is",
@@ -221,6 +225,13 @@ def _run_test(
225 )
226
227
228 +@pytest.fixture(autouse=True)
229 +def clear_cache():
230 + """Fixture to clear backend cache before each test."""
231 + _cached_get_attn_backend.cache_clear() # Clear the cache
232 + yield # This allows the test to run
233 +
234 +
235 @large_gpu_test(min_gb=48)
236 @pytest.mark.parametrize("model", models)
237 @pytest.mark.parametrize(
@@ -244,20 +255,26 @@ def _run_test(
255 @pytest.mark.parametrize("dtype", ["bfloat16"])
256 @pytest.mark.parametrize("max_tokens", [128])
257 @pytest.mark.parametrize("num_logprobs", [5])
258 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
259 def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
260 model, sizes, dtype, max_tokens,
249 - num_logprobs) -> None:
250 - run_test(
251 - hf_runner,
252 - vllm_runner,
253 - image_assets,
254 - model,
255 - sizes=sizes,
256 - dtype=dtype,
257 - max_tokens=max_tokens,
258 - num_logprobs=num_logprobs,
259 - tensor_parallel_size=1,
260 - )
261 + num_logprobs,
262 + attn_backend: _Backend) -> None:
263 + with global_force_attn_backend_context_manager(attn_backend):
264 + if attn_backend == _Backend.FLASH_ATTN:
265 + # Flash Attention works only with bfloat16 data-type
266 + dtype = 'bfloat16'
267 + run_test(
268 + hf_runner,
269 + vllm_runner,
270 + image_assets,
271 + model,
272 + sizes=sizes,
273 + dtype=dtype,
274 + max_tokens=max_tokens,
275 + num_logprobs=num_logprobs,
276 + tensor_parallel_size=1,
277 + )
278
279
280 @large_gpu_test(min_gb=48)
@@ -265,9 +282,10 @@ def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
282 @pytest.mark.parametrize("dtype", ["bfloat16"])
283 @pytest.mark.parametrize("max_tokens", [128])
284 @pytest.mark.parametrize("num_logprobs", [5])
285 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
286 def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
269 - model, dtype, max_tokens,
270 - num_logprobs) -> None:
287 + model, dtype, max_tokens, num_logprobs,
288 + attn_backend: _Backend) -> None:
289
290 stop_sign = image_assets[0].pil_image
291 cherry_blossom = image_assets[1].pil_image
@@ -291,17 +309,20 @@ def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
309 cherry_blossom.resize((512, 1024)),
310 ],
311 ])]
294 -
295 - _run_test(
296 - hf_runner,
297 - vllm_runner,
298 - inputs,
299 - model,
300 - dtype=dtype,
301 - max_tokens=max_tokens,
302 - num_logprobs=num_logprobs,
303 - tensor_parallel_size=1,
304 - )
312 + with global_force_attn_backend_context_manager(attn_backend):
313 + if attn_backend == _Backend.FLASH_ATTN:
314 + # Flash Attention works only with bfloat16 data-type
315 + dtype = 'bfloat16'
316 + _run_test(
317 + hf_runner,
318 + vllm_runner,
319 + inputs,
320 + model,
321 + dtype=dtype,
322 + max_tokens=max_tokens,
323 + num_logprobs=num_logprobs,
324 + tensor_parallel_size=1,
325 + )
326
327
328 @large_gpu_test(min_gb=48)
@@ -309,8 +330,10 @@ def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
330 @pytest.mark.parametrize("dtype", ["bfloat16"])
331 @pytest.mark.parametrize("max_tokens", [128])
332 @pytest.mark.parametrize("num_logprobs", [5])
333 +@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
334 def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
313 - dtype, max_tokens, num_logprobs) -> None:
335 + dtype, max_tokens, num_logprobs,
336 + attn_backend: _Backend) -> None:
337
338 stop_sign = image_assets[0].pil_image
339 cherry_blossom = image_assets[1].pil_image
@@ -325,14 +348,17 @@ def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
348 [stop_sign],
349 [stop_sign, cherry_blossom],
350 ])]
328 -
329 - _run_test(
330 - hf_runner,
331 - vllm_runner,
332 - inputs,
333 - model,
334 - dtype=dtype,
335 - max_tokens=max_tokens,
336 - num_logprobs=num_logprobs,
337 - tensor_parallel_size=1,
338 - )
351 + with global_force_attn_backend_context_manager(attn_backend):
352 + if attn_backend == _Backend.FLASH_ATTN:
353 + # Flash Attention works only with bfloat16 data-type
354 + dtype = 'bfloat16'
355 + _run_test(
356 + hf_runner,
357 + vllm_runner,
358 + inputs,
359 + model,
360 + dtype=dtype,
361 + max_tokens=max_tokens,
362 + num_logprobs=num_logprobs,
363 + tensor_parallel_size=1,
364 + )
tests/test_config.py
+2
@@ -243,6 +243,8 @@ def test_rope_customization():
243 assert longchat_model_config.max_model_len == 4096
244
245
246 +@pytest.mark.skipif(current_platform.is_rocm(),
247 + reason="Encoder Decoder models not supported on ROCm.")
248 @pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
249 ("facebook/opt-125m", False),
250 ("facebook/bart-base", True),
vllm/model_executor/models/mllama.py
+38 -14
@@ -32,6 +32,8 @@ from transformers.models.mllama.processing_mllama import (
32
33 import vllm.distributed.parallel_state as ps
34 from vllm.attention import Attention, AttentionMetadata, AttentionType
35 +from vllm.attention.backends.flash_attn import FlashAttentionMetadata
36 +from vllm.attention.backends.xformers import XFormersMetadata
37 from vllm.attention.ops.paged_attn import PagedAttention
38 from vllm.config import VllmConfig
39 from vllm.distributed import get_tensor_model_parallel_world_size
@@ -799,12 +801,13 @@ class MllamaTextCrossAttention(nn.Module):
801 q = self.q_norm(q)
802
803 if attention_mask is not None:
802 - output = self.attention_with_mask(q, k, v, kv_cache,
803 - attention_mask,
804 - kv_range_for_decode,
805 - attn_metadata)
804 + output = self._attention_with_mask(q, k, v, kv_cache,
805 + attention_mask,
806 + kv_range_for_decode,
807 + attn_metadata)
808 else:
807 - output = self.attn(q,
809 + output = self.attn(q.view(-1,
810 + self.num_local_heads * self.head_dim),
811 k,
812 v,
813 kv_cache,
@@ -813,7 +816,7 @@ class MllamaTextCrossAttention(nn.Module):
816 out, _ = self.o_proj(output)
817 return out
818
816 - def attention_with_mask(
819 + def _attention_with_mask(
820 self,
821 q: torch.Tensor,
822 k: torch.Tensor,
@@ -824,14 +827,35 @@ class MllamaTextCrossAttention(nn.Module):
827 attn_metadata: AttentionMetadata,
828 ) -> torch.Tensor:
829 # Skip writing kv-cache for the initial profiling run.
827 - if len(kv_cache.shape) == 3:
828 - key_cache, value_cache = PagedAttention.split_kv_cache(
829 - kv_cache, self.num_local_key_value_heads, self.head_dim)
830 - cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
831 - cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
832 - PagedAttention.write_to_paged_cache(
833 - cached_k, cached_v, key_cache, value_cache,
834 - attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
830 + if len(kv_cache.shape) > 1:
831 + if isinstance(attn_metadata, FlashAttentionMetadata):
832 + cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
833 + cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
834 + torch.ops._C_cache_ops.reshape_and_cache_flash(
835 + cached_k,
836 + cached_v,
837 + kv_cache[0],
838 + kv_cache[1],
839 + attn_metadata.
840 + cross_slot_mapping, # type: ignore[union-attr]
841 + "auto",
842 + 1.0,
843 + 1.0,
844 + )
845 + elif isinstance(attn_metadata, XFormersMetadata):
846 + key_cache, value_cache = PagedAttention.split_kv_cache(
847 + kv_cache, self.num_local_key_value_heads, self.head_dim)
848 + cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
849 + cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
850 + PagedAttention.write_to_paged_cache(
851 + cached_k, cached_v, key_cache, value_cache,
852 + attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
853 + else:
854 + raise ValueError(
855 + f"Unsupported AttentionMetadata {type(attn_metadata)} "
856 + f"class found. Expected the AttentionMetadata to "
857 + f"be either XFormersMetadata or FlashAttentionMetadata.")
858 +
859 # We have to call torch.sdpa for prefill when using a
860 # custom cross-attention mask. Because the mask is not a
861 # standard causal mask, neither a block diagonal mask which
vllm/worker/enc_dec_model_runner.py
+6 -28
@@ -9,15 +9,13 @@ from vllm.attention.backends.abstract import (AttentionBackend,
9 AttentionMetadata)
10 from vllm.attention.backends.utils import PAD_SLOT_ID
11 from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
12 - get_global_forced_attn_backend,
13 - global_force_attn_backend)
14 -from vllm.config import ModelConfig, VllmConfig
12 + get_global_forced_attn_backend)
13 +from vllm.config import VllmConfig
14 from vllm.forward_context import set_forward_context
15 from vllm.inputs import INPUT_REGISTRY, InputRegistry
16 from vllm.logger import init_logger
17 from vllm.model_executor import SamplingMetadata
18 from vllm.model_executor.layers.sampler import SamplerOutput
20 -from vllm.model_executor.model_loader.utils import get_architecture_class_name
19 from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
20 MultiModalRegistry)
21 from vllm.sampling_params import SamplingParams
@@ -35,11 +33,6 @@ from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
33
34 logger = init_logger(__name__)
35
38 -# The Mllama model has PagedAttention specific logic because of which it
39 -# can only be run with the XFORMERS backend
40 -# TODO Make Mllama model work with Flash Attention backend.
41 -_XFORMERS_ONLY_ENCODER_DECODER_ARCHS = ["MllamaForConditionalGeneration"]
42 -
36
37 @dataclasses.dataclass(frozen=True)
38 class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
@@ -97,7 +90,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
90 models) but these arguments are present here for compatibility with
91 the base-class constructor.
92 '''
100 - self._maybe_force_supported_attention_backend(vllm_config.model_config)
93 + self._maybe_force_supported_attention_backend()
94
95 super().__init__(
96 vllm_config=vllm_config,
@@ -108,12 +101,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
101 # Crash for unsupported encoder/scenarios
102 assert_enc_dec_mr_supported_scenario(self)
103
111 - def _is_xformers_only_encoder_decoder_model(self,
112 - model: ModelConfig) -> bool:
113 - return get_architecture_class_name(
114 - model) in _XFORMERS_ONLY_ENCODER_DECODER_ARCHS
115 -
116 - def _maybe_force_supported_attention_backend(self, model: ModelConfig):
104 + def _maybe_force_supported_attention_backend(self):
105 '''
106 Force vLLM to use the XFormers attention backend,
107 which is currently the only supported option.
@@ -128,23 +116,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
116 maybe_global_forced_backend = get_global_forced_attn_backend()
117 is_forced_by_global = maybe_global_forced_backend is not None
118 is_forced_by_env_var = maybe_env_var_forced_backend is not None
131 -
132 - if not (is_forced_by_global or is_forced_by_env_var) \
133 - and self._is_xformers_only_encoder_decoder_model(model):
134 - # The user has not already specified an attention backend
135 - # override
136 - logger.info(
137 - "Encoder-Decoder Model Architecture %s requires XFormers "
138 - "backend; overriding backend auto-selection and "
139 - "forcing XFormers.", get_architecture_class_name(model))
140 - global_force_attn_backend(_Backend.XFORMERS)
141 - elif is_forced_by_global:
119 + if is_forced_by_global: # noqa: SIM102
120 # Backend override enforced by global variable takes
121 # precedence over vLLM backend environment variable.
122 if maybe_global_forced_backend not in\
123 [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
124 raise_backend_err()
147 - elif is_forced_by_env_var:
125 + elif is_forced_by_env_var: # noqa: SIM102
126 # Backend override enforced by vLLM backend
127 # environment variable
128 if maybe_env_var_forced_backend not in\