@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4aba6e3d1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[core] gemma2 full context length support (#10584)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 22, 2024 at 20:13 UTC 4aba6e3d1a0cc5cec45efdee0adeaa09278f7518
4 files changed +55 -24
tests/basic_correctness/test_basic_correctness.py
+18 -7
@@ -14,11 +14,12 @@ from vllm import LLM
14 from vllm.platforms import current_platform
15 from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
16
17 +from ..conftest import VllmRunner
18 from ..models.utils import check_outputs_equal
19 from ..utils import multi_gpu_test
20
21 MODELS = [
21 - "facebook/opt-125m",
22 + "google/gemma-2-2b-it",
23 "meta-llama/Llama-3.2-1B",
24 ]
25
@@ -42,8 +43,6 @@ def test_vllm_gc_ed():
43 @pytest.mark.parametrize("enforce_eager", [False, True])
44 def test_models(
45 hf_runner,
45 - vllm_runner,
46 - example_prompts,
46 model: str,
47 backend: str,
48 dtype: str,
@@ -54,15 +53,27 @@ def test_models(
53 if backend == "FLASHINFER" and current_platform.is_rocm():
54 pytest.skip("Flashinfer does not support ROCm/HIP.")
55
56 + if backend == "XFORMERS" and model == "google/gemma-2-2b-it":
57 + pytest.skip(
58 + "XFORMERS does not support gemma2 with full context length.")
59 +
60 os.environ["VLLM_ATTENTION_BACKEND"] = backend
61
62 + # 5042 tokens for gemma2
63 + # gemma2 has alternating sliding window size of 4096
64 + # we need a prompt with more than 4096 tokens to test the sliding window
65 + prompt = "The following numbers of the sequence " + ", ".join(
66 + str(i) for i in range(1024)) + " are:"
67 + example_prompts = [prompt]
68 +
69 with hf_runner(model, dtype=dtype) as hf_model:
70 hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
71
62 - with vllm_runner(model,
63 - dtype=dtype,
64 - enforce_eager=enforce_eager,
65 - gpu_memory_utilization=0.7) as vllm_model:
72 + with VllmRunner(model,
73 + max_model_len=8192,
74 + dtype=dtype,
75 + enforce_eager=enforce_eager,
76 + gpu_memory_utilization=0.7) as vllm_model:
77 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
78
79 check_outputs_equal(
vllm/attention/layer.py
+10 -2
@@ -40,18 +40,26 @@ class Attention(nn.Module):
40 quant_config: Optional[QuantizationConfig] = None,
41 blocksparse_params: Optional[Dict[str, Any]] = None,
42 logits_soft_cap: Optional[float] = None,
43 + per_layer_sliding_window: Optional[int] = None,
44 prefix: str = "",
45 ) -> None:
46 super().__init__()
47 + if per_layer_sliding_window is not None:
48 + # per-layer sliding window
49 + sliding_window = per_layer_sliding_window
50 + elif cache_config is not None:
51 + # model-level sliding window
52 + sliding_window = cache_config.sliding_window
53 + else:
54 + sliding_window = None
55 +
56 if cache_config is not None:
57 kv_cache_dtype = cache_config.cache_dtype
58 block_size = cache_config.block_size
49 - sliding_window = cache_config.sliding_window
59 is_attention_free = cache_config.is_attention_free
60 else:
61 kv_cache_dtype = "auto"
62 block_size = 16
54 - sliding_window = None
63 is_attention_free = False
64 if num_kv_heads is None:
65 num_kv_heads = num_heads
vllm/config.py
+20 -9
@@ -233,15 +233,26 @@ class ModelConfig:
233 (self.hf_text_config.model_type in ["gemma2"]))
234
235 if (not self.disable_sliding_window and has_interleaved_attention):
236 - sliding_window_len_min = get_min_sliding_window(
237 - self.hf_text_config.sliding_window)
238 -
239 - print_warning_once(
240 - f"{self.hf_text_config.model_type} has interleaved attention, "
241 - "which is currently not supported by vLLM. Disabling sliding "
242 - "window and capping the max length to the sliding window size "
243 - f"({sliding_window_len_min}).")
244 - self.disable_sliding_window = True
236 + if envs.VLLM_ATTENTION_BACKEND == "XFORMERS":
237 + sliding_window_len_min = get_min_sliding_window(
238 + self.hf_text_config.sliding_window)
239 +
240 + print_warning_once(
241 + f"{self.hf_text_config.model_type} has interleaved "
242 + "attention, which is currently not supported by the "
243 + "XFORMERS backend. Disabling sliding window and capping "
244 + "the max length to the sliding window size "
245 + f"({sliding_window_len_min}).")
246 + self.disable_sliding_window = True
247 + else:
248 + # for a model with interleaved attention,
249 + # the scheduler and the model treat it as full attention
250 + # (i.e., not dropping any tokens outside the window).
251 + # only the attention layer itself is aware of the sliding
252 + # window, and use the window size to compute the attention.
253 + self.hf_text_config.interleaved_sliding_window = sliding_window
254 + delattr(self.hf_text_config, "sliding_window")
255 + sliding_window = None
256
257 self.max_model_len = _get_and_verify_max_len(
258 hf_config=self.hf_text_config,
vllm/model_executor/models/gemma2.py
+7 -6
@@ -143,12 +143,12 @@ class Gemma2Attention(nn.Module):
143 is_neox_style=True,
144 )
145
146 - # FIXME(woosuk): While Gemma 2 uses sliding window attention for every
147 - # odd layer, vLLM currently ignores it and uses global attention for
148 - # all layers.
149 - use_sliding_window = (layer_idx % 2 == 1
150 - and config.sliding_window is not None)
151 - del use_sliding_window # Unused.
146 + # reference:
147 + # https://github.com/huggingface/transformers/blob/54be2d7ae87e873482b984cc956e165ca4dc0ba3/src/transformers/models/gemma2/modeling_gemma2.py#L312 # noqa
148 + use_sliding_window = (layer_idx % 2 == 0 and
149 + config.interleaved_sliding_window is not None)
150 + sliding_window = config.interleaved_sliding_window if \
151 + use_sliding_window else None
152 self.attn = Attention(self.num_heads,
153 self.head_dim,
154 self.scaling,
@@ -156,6 +156,7 @@ class Gemma2Attention(nn.Module):
156 cache_config=cache_config,
157 quant_config=quant_config,
158 logits_soft_cap=attn_logits_soft_cap,
159 + per_layer_sliding_window=sliding_window,
160 prefix=f"{prefix}.attn")
161
162 def forward(