A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[core] gemma2 full context length support (#10584)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 22, 2024 at 20:13 UTC
4aba6e3d1a0cc5cec45efdee0adeaa09278f7518
4 files changed
+55
-24
tests/basic_correctness/test_basic_correctness.py
+18
-7
@@ -14,11 +14,12 @@ from vllm import LLM
14
from vllm.platforms import current_platform
15
from vllm.worker.model_runner import ModelInputForGPUWithSamplingMetadata
16
17
+from ..conftest import VllmRunner
18
from ..models.utils import check_outputs_equal
19
from ..utils import multi_gpu_test
20
21
MODELS = [
21
- "facebook/opt-125m",
22
+ "google/gemma-2-2b-it",
23
"meta-llama/Llama-3.2-1B",
24
]
25
@@ -42,8 +43,6 @@ def test_vllm_gc_ed():
43
@pytest.mark.parametrize("enforce_eager", [False, True])
44
def test_models(
45
hf_runner,
45
- vllm_runner,
46
- example_prompts,
46
model: str,
47
backend: str,
48
dtype: str,
@@ -54,15 +53,27 @@ def test_models(
53
if backend == "FLASHINFER" and current_platform.is_rocm():
54
pytest.skip("Flashinfer does not support ROCm/HIP.")
55
56
+ if backend == "XFORMERS" and model == "google/gemma-2-2b-it":
57
+ pytest.skip(
58
+ "XFORMERS does not support gemma2 with full context length.")
59
+
60
os.environ["VLLM_ATTENTION_BACKEND"] = backend
61
62
+ # 5042 tokens for gemma2
63
+ # gemma2 has alternating sliding window size of 4096
64
+ # we need a prompt with more than 4096 tokens to test the sliding window
65
+ prompt = "The following numbers of the sequence " + ", ".join(
66
+ str(i) for i in range(1024)) + " are:"
67
+ example_prompts = [prompt]
68
+
69
with hf_runner(model, dtype=dtype) as hf_model:
70
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
71
62
- with vllm_runner(model,
63
- dtype=dtype,
64
- enforce_eager=enforce_eager,
65
- gpu_memory_utilization=0.7) as vllm_model:
72
+ with VllmRunner(model,
73
+ max_model_len=8192,
74
+ dtype=dtype,
75
+ enforce_eager=enforce_eager,
76
+ gpu_memory_utilization=0.7) as vllm_model:
77
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
78
79
check_outputs_equal(
vllm/attention/layer.py
+10
-2
@@ -40,18 +40,26 @@ class Attention(nn.Module):
40
quant_config: Optional[QuantizationConfig] = None,
41
blocksparse_params: Optional[Dict[str, Any]] = None,
42
logits_soft_cap: Optional[float] = None,
43
+ per_layer_sliding_window: Optional[int] = None,
44
prefix: str = "",
45
) -> None:
46
super().__init__()
47
+ if per_layer_sliding_window is not None:
48
+ # per-layer sliding window
49
+ sliding_window = per_layer_sliding_window
50
+ elif cache_config is not None:
51
+ # model-level sliding window
52
+ sliding_window = cache_config.sliding_window
53
+ else:
54
+ sliding_window = None
55
+
56
if cache_config is not None:
57
kv_cache_dtype = cache_config.cache_dtype
58
block_size = cache_config.block_size
49
- sliding_window = cache_config.sliding_window
59
is_attention_free = cache_config.is_attention_free
60
else:
61
kv_cache_dtype = "auto"
62
block_size = 16
54
- sliding_window = None
63
is_attention_free = False
64
if num_kv_heads is None:
65
num_kv_heads = num_heads
vllm/config.py
+20
-9
@@ -233,15 +233,26 @@ class ModelConfig:
233
(self.hf_text_config.model_type in ["gemma2"]))
234
235
if (not self.disable_sliding_window and has_interleaved_attention):
236
- sliding_window_len_min = get_min_sliding_window(
237
- self.hf_text_config.sliding_window)
238
-
239
- print_warning_once(
240
- f"{self.hf_text_config.model_type} has interleaved attention, "
241
- "which is currently not supported by vLLM. Disabling sliding "
242
- "window and capping the max length to the sliding window size "
243
- f"({sliding_window_len_min}).")
244
- self.disable_sliding_window = True
236
+ if envs.VLLM_ATTENTION_BACKEND == "XFORMERS":
237
+ sliding_window_len_min = get_min_sliding_window(
238
+ self.hf_text_config.sliding_window)
239
+
240
+ print_warning_once(
241
+ f"{self.hf_text_config.model_type} has interleaved "
242
+ "attention, which is currently not supported by the "
243
+ "XFORMERS backend. Disabling sliding window and capping "
244
+ "the max length to the sliding window size "
245
+ f"({sliding_window_len_min}).")
246
+ self.disable_sliding_window = True
247
+ else:
248
+ # for a model with interleaved attention,
249
+ # the scheduler and the model treat it as full attention
250
+ # (i.e., not dropping any tokens outside the window).
251
+ # only the attention layer itself is aware of the sliding
252
+ # window, and use the window size to compute the attention.
253
+ self.hf_text_config.interleaved_sliding_window = sliding_window
254
+ delattr(self.hf_text_config, "sliding_window")
255
+ sliding_window = None
256
257
self.max_model_len = _get_and_verify_max_len(
258
hf_config=self.hf_text_config,
vllm/model_executor/models/gemma2.py
+7
-6
@@ -143,12 +143,12 @@ class Gemma2Attention(nn.Module):
143
is_neox_style=True,
144
)
145
146
- # FIXME(woosuk): While Gemma 2 uses sliding window attention for every
147
- # odd layer, vLLM currently ignores it and uses global attention for
148
- # all layers.
149
- use_sliding_window = (layer_idx % 2 == 1
150
- and config.sliding_window is not None)
151
- del use_sliding_window # Unused.
146
+ # reference:
147
+ # https://github.com/huggingface/transformers/blob/54be2d7ae87e873482b984cc956e165ca4dc0ba3/src/transformers/models/gemma2/modeling_gemma2.py#L312 # noqa
148
+ use_sliding_window = (layer_idx % 2 == 0 and
149
+ config.interleaved_sliding_window is not None)
150
+ sliding_window = config.interleaved_sliding_window if \
151
+ use_sliding_window else None
152
self.attn = Attention(self.num_heads,
153
self.head_dim,
154
self.scaling,
@@ -156,6 +156,7 @@ class Gemma2Attention(nn.Module):
156
cache_config=cache_config,
157
quant_config=quant_config,
158
logits_soft_cap=attn_logits_soft_cap,
159
+ per_layer_sliding_window=sliding_window,
160
prefix=f"{prefix}.attn")
161
162
def forward(