A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Remove V0 Encoder-Decoder Support (#24907)
Signed-off-by: Woosuk Kwon <woosuk@thinkingmachines.ai>
Woosuk Kwon committed
Sep 15, 2025 at 21:17 UTC
759ef49b15149daaa0b2ba8900c1983e7e5e4514
47 files changed
+13
-9661
.buildkite/scripts/hardware_ci/run-cpu-test.sh
-1
@@ -66,7 +66,6 @@ function cpu_tests() {
66
67
pytest -x -v -s tests/models/language/pooling -m cpu_model
68
pytest -x -v -s tests/models/multimodal/generation \
69
- --ignore=tests/models/multimodal/generation/test_mllama.py \
69
--ignore=tests/models/multimodal/generation/test_pixtral.py \
70
-m cpu_model"
71
.buildkite/test-pipeline.yaml
-9
@@ -549,15 +549,6 @@ steps:
549
commands: # LMEval+Transcription WER check
550
- pytest -s entrypoints/openai/correctness/
551
552
-- label: Encoder Decoder tests # 12min
553
- timeout_in_minutes: 20
554
- mirror_hardwares: [amdexperimental]
555
- source_file_dependencies:
556
- - vllm/
557
- - tests/encoder_decoder
558
- commands:
559
- - pytest -v -s encoder_decoder
560
-
552
- label: OpenAI-Compatible Tool Use # 23 min
553
timeout_in_minutes: 35
554
mirror_hardwares: [amdexperimental]
docs/contributing/model/multimodal.md
-1
@@ -840,7 +840,6 @@ Some HF processors directly insert feature tokens without replacing anything in
840
Examples:
841
842
- BLIP-2 (insert at start of prompt): <gh-file:vllm/model_executor/models/blip2.py>
843
-- Florence2 (insert at start of prompt): <gh-file:vllm/model_executor/models/florence2.py>
843
- Molmo (insert after `<|endoftext|>` token): <gh-file:vllm/model_executor/models/molmo.py>
844
845
### Handling prompt updates unrelated to multi-modal data
docs/models/supported_models.md
-8
@@ -331,8 +331,6 @@ th {
331
| `BailingMoeV2ForCausalLM` | Ling | `inclusionAI/Ling-mini-2.0`, etc. | ✅︎ | ✅︎ | ✅︎ |
332
| `BambaForCausalLM` | Bamba | `ibm-ai-platform/Bamba-9B-fp8`, `ibm-ai-platform/Bamba-9B` | ✅︎ | ✅︎ | ✅︎ |
333
| `BloomForCausalLM` | BLOOM, BLOOMZ, BLOOMChat | `bigscience/bloom`, `bigscience/bloomz`, etc. | | ✅︎ | ✅︎ |
334
-| `BartForConditionalGeneration` | BART | `facebook/bart-base`, `facebook/bart-large-cnn`, etc. | | | |
335
-| `MBartForConditionalGeneration` | mBART | `facebook/mbart-large-en-ro`, `facebook/mbart-large-50`, etc. | | | |
334
| `ChatGLMModel`, `ChatGLMForConditionalGeneration` | ChatGLM | `zai-org/chatglm2-6b`, `zai-org/chatglm3-6b`, `ShieldLM-6B-chatglm3`, etc. | ✅︎ | ✅︎ | ✅︎ |
335
| `CohereForCausalLM`, `Cohere2ForCausalLM` | Command-R, Command-A | `CohereLabs/c4ai-command-r-v01`, `CohereLabs/c4ai-command-r7b-12-2024`, `CohereLabs/c4ai-command-a-03-2025`, `CohereLabs/command-a-reasoning-08-2025`, etc. | ✅︎ | ✅︎ | ✅︎ |
336
| `DbrxForCausalLM` | DBRX | `databricks/dbrx-base`, `databricks/dbrx-instruct`, etc. | | ✅︎ | ✅︎ |
@@ -426,9 +424,6 @@ Some models are supported only via the [Transformers backend](#transformers). Th
424
!!! note
425
Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
426
429
-!!! note
430
- Some mBART models' config files do not have an `architecture` defined. Therefore, you need to use `--hf-overrides '{"architectures": ["MBartForConditionalGeneration"]}'` to explicitly specify the use of the `MBartForConditionalGeneration` architecture.
431
-
427
### Pooling Models
428
429
See [this page](./pooling_models.md) for more information on how to use pooling models.
@@ -625,9 +620,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
620
| `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ | ✅︎ |
621
| `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ | ✅︎ |
622
| `DeepseekVLV2ForCausalLM`<sup>^</sup> | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ | ✅︎ |
628
-| `DonutForConditionalGeneration`<sup>^</sup> | Donut | T + I | `ByteDance/Dolphin`, `naver-clova-ix/donut-base-finetuned-docvqa`, etc. | | | |
623
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ | ✅︎ |
630
-| `Florence2ForConditionalGeneration` | Florence-2 | T + I | `microsoft/Florence-2-base`, `microsoft/Florence-2-large`, etc. | | | |
624
| `FuyuForCausalLM` | Fuyu | T + I | `adept/fuyu-8b`, etc. | | ✅︎ | ✅︎ |
625
| `Gemma3ForConditionalGeneration` | Gemma 3 | T + I<sup>+</sup> | `google/gemma-3-4b-it`, `google/gemma-3-27b-it`, etc. | ✅︎ | ✅︎ | ⚠️ |
626
| `Gemma3nForConditionalGeneration` | Gemma 3n | T + I + A | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | ✅︎ |
@@ -654,7 +647,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
647
| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, etc. | ✅︎ | | ✅︎ |
648
| `MiniMaxVL01ForConditionalGeneration` | MiniMax-VL | T + I<sup>E+</sup> | `MiniMaxAI/MiniMax-VL-01`, etc. | | ✅︎ | ✅︎ |
649
| `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ | ✅︎ |
657
-| `MllamaForConditionalGeneration` | Llama 3.2 | T + I<sup>+</sup> | `meta-llama/Llama-3.2-90B-Vision-Instruct`, `meta-llama/Llama-3.2-11B-Vision`, etc. | | | |
650
| `MolmoForCausalLM` | Molmo | T + I<sup>+</sup> | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ | ✅︎ |
651
| `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ | ✅︎ |
652
| `Ovis` | Ovis2, Ovis1.6 | T + I<sup>+</sup> | `AIDC-AI/Ovis2-1B`, `AIDC-AI/Ovis1.6-Llama3.2-3B`, etc. | | ✅︎ | ✅︎ |
docs/usage/v1_guide.md
+1
-1
@@ -120,7 +120,7 @@ Please note that prefix caching is not yet supported for any of the above models
120
121
Whisper is supported. Other models requiring cross-attention between separate
122
encoder and decoder (e.g., `BartForConditionalGeneration`,
123
-`MllamaForConditionalGeneration`) are not yet supported.
123
+`MllamaForConditionalGeneration`) are not supported.
124
125
### Features
126
examples/offline_inference/dolphin.py
deleted
-311
@@ -1,311 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import argparse
5
-import copy
6
-import os
7
-from dataclasses import dataclass
8
-
9
-import cv2
10
-import numpy as np
11
-import regex as re
12
-from PIL import Image
13
-from transformers import DonutProcessor
14
-
15
-from vllm import LLM, SamplingParams
16
-from vllm.inputs import ExplicitEncoderDecoderPrompt, TextPrompt, TokensPrompt
17
-from vllm.multimodal.utils import fetch_image
18
-
19
-
20
-# Copied from https://github.com/bytedance/Dolphin/utils/utils.py
21
-@dataclass
22
-class ImageDimensions:
23
- original_w: int
24
- original_h: int
25
- padded_w: int
26
- padded_h: int
27
-
28
-
29
-# Copied from https://github.com/bytedance/Dolphin/utils/utils.py
30
-def map_to_original_coordinates(
31
- x1, y1, x2, y2, dims: ImageDimensions
32
-) -> tuple[int, int, int, int]:
33
- try:
34
- top = (dims.padded_h - dims.original_h) // 2
35
- left = (dims.padded_w - dims.original_w) // 2
36
- orig_x1 = max(0, x1 - left)
37
- orig_y1 = max(0, y1 - top)
38
- orig_x2 = min(dims.original_w, x2 - left)
39
- orig_y2 = min(dims.original_h, y2 - top)
40
- if orig_x2 <= orig_x1:
41
- orig_x2 = min(orig_x1 + 1, dims.original_w)
42
- if orig_y2 <= orig_y1:
43
- orig_y2 = min(orig_y1 + 1, dims.original_h)
44
- return int(orig_x1), int(orig_y1), int(orig_x2), int(orig_y2)
45
- except Exception as e:
46
- print(f"map_to_original_coordinates error: {str(e)}")
47
- return 0, 0, min(100, dims.original_w), min(100, dims.original_h)
48
-
49
-
50
-# Copied from https://github.com/bytedance/Dolphin/utils/utils.py
51
-def adjust_box_edges(image, boxes: list[list[float]], max_pixels=15, threshold=0.2):
52
- if isinstance(image, str):
53
- image = cv2.imread(image)
54
- img_h, img_w = image.shape[:2]
55
- new_boxes = []
56
- for box in boxes:
57
- best_box = copy.deepcopy(box)
58
-
59
- def check_edge(img, current_box, i, is_vertical):
60
- edge = current_box[i]
61
- gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
62
- _, binary = cv2.threshold(
63
- gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU
64
- )
65
- if is_vertical:
66
- line = binary[current_box[1] : current_box[3] + 1, edge]
67
- else:
68
- line = binary[edge, current_box[0] : current_box[2] + 1]
69
- transitions = np.abs(np.diff(line))
70
- return np.sum(transitions) / len(transitions)
71
-
72
- edges = [(0, -1, True), (2, 1, True), (1, -1, False), (3, 1, False)]
73
- current_box = copy.deepcopy(box)
74
- current_box[0] = min(max(current_box[0], 0), img_w - 1)
75
- current_box[1] = min(max(current_box[1], 0), img_h - 1)
76
- current_box[2] = min(max(current_box[2], 0), img_w - 1)
77
- current_box[3] = min(max(current_box[3], 0), img_h - 1)
78
-
79
- for i, direction, is_vertical in edges:
80
- best_score = check_edge(image, current_box, i, is_vertical)
81
- if best_score <= threshold:
82
- continue
83
- for step in range(max_pixels):
84
- current_box[i] += direction
85
- if i == 0 or i == 2:
86
- current_box[i] = min(max(current_box[i], 0), img_w - 1)
87
- else:
88
- current_box[i] = min(max(current_box[i], 0), img_h - 1)
89
- score = check_edge(image, current_box, i, is_vertical)
90
- if score < best_score:
91
- best_score = score
92
- best_box = copy.deepcopy(current_box)
93
- if score <= threshold:
94
- break
95
- new_boxes.append(best_box)
96
- return new_boxes
97
-
98
-
99
-# Copied from https://github.com/bytedance/Dolphin/utils/utils.py
100
-def process_coordinates(coords, padded_image, dims: ImageDimensions, previous_box=None):
101
- try:
102
- x1, y1 = int(coords[0] * dims.padded_w), int(coords[1] * dims.padded_h)
103
- x2, y2 = int(coords[2] * dims.padded_w), int(coords[3] * dims.padded_h)
104
- x1, y1, x2, y2 = (
105
- max(0, min(x1, dims.padded_w - 1)),
106
- max(0, min(y1, dims.padded_h - 1)),
107
- max(0, min(x2, dims.padded_w)),
108
- max(0, min(y2, dims.padded_h)),
109
- )
110
- if x2 <= x1:
111
- x2 = min(x1 + 1, dims.padded_w)
112
- if y2 <= y1:
113
- y2 = min(y1 + 1, dims.padded_h)
114
- new_boxes = adjust_box_edges(padded_image, [[x1, y1, x2, y2]])
115
- x1, y1, x2, y2 = new_boxes[0]
116
- x1, y1, x2, y2 = (
117
- max(0, min(x1, dims.padded_w - 1)),
118
- max(0, min(y1, dims.padded_h - 1)),
119
- max(0, min(x2, dims.padded_w)),
120
- max(0, min(y2, dims.padded_h)),
121
- )
122
- if x2 <= x1:
123
- x2 = min(x1 + 1, dims.padded_w)
124
- if y2 <= y1:
125
- y2 = min(y1 + 1, dims.padded_h)
126
- if previous_box is not None:
127
- prev_x1, prev_y1, prev_x2, prev_y2 = previous_box
128
- if (x1 < prev_x2 and x2 > prev_x1) and (y1 < prev_y2 and y2 > prev_y1):
129
- y1 = prev_y2
130
- y1 = min(y1, dims.padded_h - 1)
131
- if y2 <= y1:
132
- y2 = min(y1 + 1, dims.padded_h)
133
- new_previous_box = [x1, y1, x2, y2]
134
- orig_x1, orig_y1, orig_x2, orig_y2 = map_to_original_coordinates(
135
- x1, y1, x2, y2, dims
136
- )
137
- return x1, y1, x2, y2, orig_x1, orig_y1, orig_x2, orig_y2, new_previous_box
138
- except Exception as e:
139
- print(f"process_coordinates error: {str(e)}")
140
- orig_x1, orig_y1, orig_x2, orig_y2 = (
141
- 0,
142
- 0,
143
- min(100, dims.original_w),
144
- min(100, dims.original_h),
145
- )
146
- return 0, 0, 100, 100, orig_x1, orig_y1, orig_x2, orig_y2, [0, 0, 100, 100]
147
-
148
-
149
-# Copied from https://github.com/bytedance/Dolphin/utils/utils.py
150
-def prepare_image(image) -> tuple[np.ndarray, ImageDimensions]:
151
- try:
152
- image_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
153
- original_h, original_w = image_cv.shape[:2]
154
- max_size = max(original_h, original_w)
155
- top = (max_size - original_h) // 2
156
- bottom = max_size - original_h - top
157
- left = (max_size - original_w) // 2
158
- right = max_size - original_w - left
159
- padded_image = cv2.copyMakeBorder(
160
- image_cv, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(0, 0, 0)
161
- )
162
- padded_h, padded_w = padded_image.shape[:2]
163
- dimensions = ImageDimensions(
164
- original_w=original_w,
165
- original_h=original_h,
166
- padded_w=padded_w,
167
- padded_h=padded_h,
168
- )
169
- return padded_image, dimensions
170
- except Exception as e:
171
- print(f"prepare_image error: {str(e)}")
172
- h, w = image.height, image.width
173
- dimensions = ImageDimensions(original_w=w, original_h=h, padded_w=w, padded_h=h)
174
- return np.zeros((h, w, 3), dtype=np.uint8), dimensions
175
-
176
-
177
-# Copied from https://github.com/bytedance/Dolphin/utils/utils.py
178
-def parse_layout_string(bbox_str):
179
- """Parse layout string using regular expressions"""
180
- pattern = r"\[(\d*\.?\d+),\s*(\d*\.?\d+),\s*(\d*\.?\d+),\s*(\d*\.?\d+)\]\s*(\w+)"
181
- matches = re.finditer(pattern, bbox_str)
182
-
183
- parsed_results = []
184
- for match in matches:
185
- coords = [float(match.group(i)) for i in range(1, 5)]
186
- label = match.group(5).strip()
187
- parsed_results.append((coords, label))
188
-
189
- return parsed_results
190
-
191
-
192
-model_id = "ByteDance/Dolphin"
193
-
194
-# The input image size for Dolphin is 896 x 896,
195
-# and the patch_size is 4 x 4.
196
-# Therefore, the initial number of patches is:
197
-# Height: 896 / 4 = 224 patches
198
-# Width: 896 / 4 = 224 patches
199
-
200
-# The Dolphin model uses a staged downsampling approach,
201
-# defined by the "depths": [2, 2, 14, 2] configuration.
202
-# Before entering stages 2, 3, and 4, a "Patch Merging" operation is performed,
203
-# which halves the feature map's dimensions (dividing both height and width by 2).
204
-# Before Stage 2: The size changes from 224 x 224 to (224/2) x (224/2) = 112 x 112.
205
-# Before Stage 3: The size changes from 112 x 112 to (112/2) x (112/2) = 56 x 56.
206
-# Before Stage 4: The size changes from 56 x 56 to (56/2) x (56/2) = 28 x 28.
207
-
208
-# Because vLLM needs to fill the image features with an encoder_prompt,
209
-# and the encoder_prompt will have `<pad>` tokens added when tokenized,
210
-# we need to construct an encoder_prompt with a length of 28 x 28 - 1 = 783.
211
-encoder_prompt = "".join(["0"] * 783)
212
-sampling_params = SamplingParams(
213
- temperature=0.0,
214
- max_tokens=2048,
215
-)
216
-
217
-processor = DonutProcessor.from_pretrained(model_id)
218
-llm = LLM(
219
- model=model_id,
220
- dtype="float16",
221
- max_num_seqs=8,
222
- hf_overrides={"architectures": ["DonutForConditionalGeneration"]},
223
-)
224
-
225
-parser = argparse.ArgumentParser()
226
-parser.add_argument(
227
- "--image_path", type=str, default=None, help="Path to a local image file."
228
-)
229
-args = parser.parse_args()
230
-
231
-if args.image_path:
232
- if not os.path.exists(args.image_path):
233
- raise FileNotFoundError(f"Error: File not found at {args.image_path}")
234
- image = Image.open(args.image_path).convert("RGB")
235
-else:
236
- image = fetch_image(
237
- "https://huggingface.co/datasets/hf-internal-testing/example-documents/resolve/main/jpeg_images/0.jpg"
238
- )
239
-
240
-
241
-prompt = "Parse the reading order of this document. "
242
-decoder_prompt = f"<s>{prompt}<Answer/>"
243
-decoder_prompt_tokens = TokensPrompt(
244
- prompt_token_ids=processor.tokenizer(decoder_prompt, add_special_tokens=False)[
245
- "input_ids"
246
- ]
247
-)
248
-enc_dec_prompt = ExplicitEncoderDecoderPrompt(
249
- encoder_prompt=TextPrompt(prompt=encoder_prompt, multi_modal_data={"image": image}),
250
- decoder_prompt=decoder_prompt_tokens,
251
-)
252
-layout_outputs = llm.generate(prompts=enc_dec_prompt, sampling_params=sampling_params)
253
-layout_result_str = layout_outputs[0].outputs[0].text
254
-print(f"Layout analysis output:\n{layout_result_str}")
255
-
256
-padded_image, dims = prepare_image(image)
257
-layout_results = parse_layout_string(layout_result_str)
258
-text_table_elements = []
259
-previous_box = None
260
-reading_order = 0
261
-for bbox_coords, label in layout_results:
262
- if label == "fig":
263
- continue
264
- try:
265
- x1, y1, x2, y2, orig_x1, orig_y1, orig_x2, orig_y2, previous_box = (
266
- process_coordinates(bbox_coords, padded_image, dims, previous_box)
267
- )
268
- cropped = padded_image[y1:y2, x1:x2]
269
- if cropped.size > 0 and cropped.shape[0] > 3 and cropped.shape[1] > 3:
270
- pil_crop = Image.fromarray(cv2.cvtColor(cropped, cv2.COLOR_BGR2RGB))
271
- prompt_ocr = (
272
- "Parse the table in the image. "
273
- if label == "tab"
274
- else "Read text in the image. "
275
- )
276
- text_table_elements.append(
277
- {
278
- "crop": pil_crop,
279
- "prompt": prompt_ocr,
280
- "reading_order": reading_order,
281
- }
282
- )
283
- reading_order += 1
284
- except Exception as e:
285
- print(f"Error processing bbox (label: {label}): {str(e)}")
286
- continue
287
-
288
-if text_table_elements:
289
- batch_prompts = []
290
- for elem in text_table_elements:
291
- decoder_prompt_str = f"<s>{elem['prompt']}<Answer/>"
292
- decoder_prompt_tokens = TokensPrompt(
293
- prompt_token_ids=processor.tokenizer(
294
- decoder_prompt_str, add_special_tokens=False
295
- )["input_ids"]
296
- )
297
- enc_dec_prompt = ExplicitEncoderDecoderPrompt(
298
- encoder_prompt=TextPrompt(
299
- prompt=encoder_prompt, multi_modal_data={"image": elem["crop"]}
300
- ),
301
- decoder_prompt=decoder_prompt_tokens,
302
- )
303
- batch_prompts.append(enc_dec_prompt)
304
- batch_outputs = llm.generate(prompts=batch_prompts, sampling_params=sampling_params)
305
- for i, output in enumerate(batch_outputs):
306
- text_table_elements[i]["text"] = output.outputs[0].text.strip()
307
-
308
-print("------" * 8)
309
-text_table_elements.sort(key=lambda x: x["reading_order"])
310
-for elem in text_table_elements:
311
- print(elem.get("text", ""))
examples/offline_inference/encoder_decoder.py
deleted
-195
@@ -1,195 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-Demonstrate prompting of text-to-text
5
-encoder/decoder models, specifically BART and mBART.
6
-
7
-This script is refactored to allow model selection via command-line arguments.
8
-
9
-NOTE: This example is not yet supported in V1.
10
-"""
11
-
12
-import argparse
13
-from typing import NamedTuple, Optional
14
-
15
-from vllm import LLM, SamplingParams
16
-from vllm.inputs import (
17
- ExplicitEncoderDecoderPrompt,
18
- TextPrompt,
19
- TokensPrompt,
20
- zip_enc_dec_prompts,
21
-)
22
-
23
-
24
-class ModelRequestData(NamedTuple):
25
- """
26
- Holds the configuration for a specific model, including its
27
- HuggingFace ID and the prompts to use for the demo.
28
- """
29
-
30
- model_id: str
31
- encoder_prompts: list
32
- decoder_prompts: list
33
- hf_overrides: Optional[dict] = None
34
-
35
-
36
-def get_bart_config() -> ModelRequestData:
37
- """
38
- Returns the configuration for facebook/bart-large-cnn.
39
- This uses the exact test cases from the original script.
40
- """
41
- encoder_prompts = [
42
- "Hello, my name is",
43
- "The president of the United States is",
44
- "The capital of France is",
45
- "An encoder prompt",
46
- ]
47
- decoder_prompts = [
48
- "A decoder prompt",
49
- "Another decoder prompt",
50
- ]
51
- return ModelRequestData(
52
- model_id="facebook/bart-large-cnn",
53
- encoder_prompts=encoder_prompts,
54
- decoder_prompts=decoder_prompts,
55
- )
56
-
57
-
58
-def get_mbart_config() -> ModelRequestData:
59
- """
60
- Returns the configuration for facebook/mbart-large-en-ro.
61
- This uses prompts suitable for an English-to-Romanian translation task.
62
- """
63
- encoder_prompts = [
64
- "The quick brown fox jumps over the lazy dog.",
65
- "How are you today?",
66
- ]
67
- decoder_prompts = ["", ""]
68
- hf_overrides = {"architectures": ["MBartForConditionalGeneration"]}
69
- return ModelRequestData(
70
- model_id="facebook/mbart-large-en-ro",
71
- encoder_prompts=encoder_prompts,
72
- decoder_prompts=decoder_prompts,
73
- hf_overrides=hf_overrides,
74
- )
75
-
76
-
77
-MODEL_GETTERS = {
78
- "bart": get_bart_config,
79
- "mbart": get_mbart_config,
80
-}
81
-
82
-
83
-def create_all_prompt_types(
84
- encoder_prompts_raw: list,
85
- decoder_prompts_raw: list,
86
- tokenizer,
87
-) -> list:
88
- """
89
- Generates a list of diverse prompt types for demonstration.
90
- This function is generic and uses the provided raw prompts
91
- to create various vLLM input objects.
92
- """
93
- text_prompt_raw = encoder_prompts_raw[0]
94
- text_prompt = TextPrompt(prompt=encoder_prompts_raw[1 % len(encoder_prompts_raw)])
95
- tokens_prompt = TokensPrompt(
96
- prompt_token_ids=tokenizer.encode(
97
- encoder_prompts_raw[2 % len(encoder_prompts_raw)]
98
- )
99
- )
100
-
101
- decoder_tokens_prompt = TokensPrompt(
102
- prompt_token_ids=tokenizer.encode(decoder_prompts_raw[0])
103
- )
104
- single_prompt_examples = [
105
- text_prompt_raw,
106
- text_prompt,
107
- tokens_prompt,
108
- ]
109
- explicit_pair_examples = [
110
- ExplicitEncoderDecoderPrompt(
111
- encoder_prompt=text_prompt_raw,
112
- decoder_prompt=decoder_tokens_prompt,
113
- ),
114
- ExplicitEncoderDecoderPrompt(
115
- encoder_prompt=text_prompt,
116
- decoder_prompt=decoder_prompts_raw[1 % len(decoder_prompts_raw)],
117
- ),
118
- ExplicitEncoderDecoderPrompt(
119
- encoder_prompt=tokens_prompt,
120
- decoder_prompt=text_prompt,
121
- ),
122
- ]
123
- zipped_prompt_list = zip_enc_dec_prompts(
124
- encoder_prompts_raw,
125
- decoder_prompts_raw,
126
- )
127
- return single_prompt_examples + explicit_pair_examples + zipped_prompt_list
128
-
129
-
130
-def create_sampling_params() -> SamplingParams:
131
- """Create a sampling params object."""
132
- return SamplingParams(
133
- temperature=0,
134
- top_p=1.0,
135
- min_tokens=0,
136
- max_tokens=30,
137
- )
138
-
139
-
140
-def print_outputs(outputs: list):
141
- """Formats and prints the generation outputs."""
142
- print("-" * 80)
143
- for i, output in enumerate(outputs):
144
- prompt = output.prompt
145
- encoder_prompt = output.encoder_prompt
146
- generated_text = output.outputs[0].text
147
- print(f"Output {i + 1}:")
148
- print(f"Encoder Prompt: {encoder_prompt!r}")
149
- print(f"Decoder Prompt: {prompt!r}")
150
- print(f"Generated Text: {generated_text!r}")
151
- print("-" * 80)
152
-
153
-
154
-def main(args):
155
- """Main execution function."""
156
- model_key = args.model
157
- if model_key not in MODEL_GETTERS:
158
- raise ValueError(
159
- f"Unknown model: {model_key}. "
160
- f"Available models: {list(MODEL_GETTERS.keys())}"
161
- )
162
- config_getter = MODEL_GETTERS[model_key]
163
- model_config = config_getter()
164
-
165
- print(f"🚀 Running demo for model: {model_config.model_id}")
166
- llm = LLM(
167
- model=model_config.model_id,
168
- dtype="float",
169
- hf_overrides=model_config.hf_overrides,
170
- )
171
- tokenizer = llm.llm_engine.get_tokenizer_group()
172
- prompts = create_all_prompt_types(
173
- encoder_prompts_raw=model_config.encoder_prompts,
174
- decoder_prompts_raw=model_config.decoder_prompts,
175
- tokenizer=tokenizer,
176
- )
177
- sampling_params = create_sampling_params()
178
- outputs = llm.generate(prompts, sampling_params)
179
- print_outputs(outputs)
180
-
181
-
182
-if __name__ == "__main__":
183
- parser = argparse.ArgumentParser(
184
- description="A flexible demo for vLLM encoder-decoder models."
185
- )
186
- parser.add_argument(
187
- "--model",
188
- "-m",
189
- type=str,
190
- default="bart",
191
- choices=MODEL_GETTERS.keys(),
192
- help="The short name of the model to run.",
193
- )
194
- args = parser.parse_args()
195
- main(args)
examples/offline_inference/encoder_decoder_multimodal.py
+1
-113
@@ -13,8 +13,6 @@ from typing import NamedTuple
13
14
from vllm import LLM, EngineArgs, PromptType, SamplingParams
15
from vllm.assets.audio import AudioAsset
16
-from vllm.assets.image import ImageAsset
17
-from vllm.multimodal.utils import fetch_image
16
from vllm.utils import FlexibleArgumentParser
17
18
@@ -23,113 +21,6 @@ class ModelRequestData(NamedTuple):
21
prompts: Sequence[PromptType]
22
23
26
-def run_donut():
27
- engine_args = EngineArgs(
28
- model="naver-clova-ix/donut-base-finetuned-docvqa",
29
- max_num_seqs=2,
30
- limit_mm_per_prompt={"image": 1},
31
- dtype="float16",
32
- hf_overrides={"architectures": ["DonutForConditionalGeneration"]},
33
- )
34
-
35
- # The input image size for donut-base-finetuned-docvqa is 2560 x 1920,
36
- # and the patch_size is 4 x 4.
37
- # Therefore, the initial number of patches is:
38
- # Height: 1920 / 4 = 480 patches
39
- # Width: 2560 / 4 = 640 patches
40
- # The Swin model uses a staged downsampling approach,
41
- # defined by the "depths": [2, 2, 14, 2] configuration.
42
- # Before entering stages 2, 3, and 4, a "Patch Merging" operation is performed,
43
- # which halves the feature map's dimensions (dividing both height and width by 2).
44
- # Before Stage 2: The size changes from 480 x 640 to (480/2) x (640/2) = 240 x 320.
45
- # Before Stage 3: The size changes from 240 x 320 to (240/2) x (320/2) = 120 x 160.
46
- # Before Stage 4: The size changes from 120 x 160 to (120/2) x (160/2) = 60 x 80.
47
- # Because vLLM needs to fill the image features with an encoder_prompt,
48
- # and the encoder_prompt will have `<pad>` tokens added when tokenized,
49
- # we need to construct an encoder_prompt with a length of 60 x 80 - 1 = 4799.
50
- prompts = [
51
- {
52
- "encoder_prompt": {
53
- "prompt": "".join(["$"] * 4799),
54
- "multi_modal_data": {
55
- "image": fetch_image(
56
- "https://huggingface.co/datasets/hf-internal-testing/example-documents/resolve/main/jpeg_images/0.jpg"
57
- ) # noqa: E501
58
- },
59
- },
60
- "decoder_prompt": "<s_docvqa><s_question>What time is the coffee break?</s_question><s_answer>", # noqa: E501
61
- },
62
- ]
63
-
64
- return ModelRequestData(
65
- engine_args=engine_args,
66
- prompts=prompts,
67
- )
68
-
69
-
70
-def run_florence2():
71
- engine_args = EngineArgs(
72
- model="microsoft/Florence-2-large",
73
- tokenizer="Isotr0py/Florence-2-tokenizer",
74
- max_num_seqs=8,
75
- trust_remote_code=True,
76
- limit_mm_per_prompt={"image": 1},
77
- dtype="half",
78
- )
79
-
80
- prompts = [
81
- { # implicit prompt with task token
82
- "prompt": "<DETAILED_CAPTION>",
83
- "multi_modal_data": {"image": ImageAsset("stop_sign").pil_image},
84
- },
85
- { # explicit encoder/decoder prompt
86
- "encoder_prompt": {
87
- "prompt": "Describe in detail what is shown in the image.",
88
- "multi_modal_data": {"image": ImageAsset("cherry_blossom").pil_image},
89
- },
90
- "decoder_prompt": "",
91
- },
92
- ]
93
-
94
- return ModelRequestData(
95
- engine_args=engine_args,
96
- prompts=prompts,
97
- )
98
-
99
-
100
-def run_mllama():
101
- engine_args = EngineArgs(
102
- model="meta-llama/Llama-3.2-11B-Vision-Instruct",
103
- max_model_len=8192,
104
- max_num_seqs=2,
105
- limit_mm_per_prompt={"image": 1},
106
- dtype="half",
107
- )
108
-
109
- prompts = [
110
- { # Implicit prompt
111
- "prompt": "<|image|><|begin_of_text|>What is the content of this image?", # noqa: E501
112
- "multi_modal_data": {
113
- "image": ImageAsset("stop_sign").pil_image,
114
- },
115
- },
116
- { # Explicit prompt
117
- "encoder_prompt": {
118
- "prompt": "<|image|>",
119
- "multi_modal_data": {
120
- "image": ImageAsset("stop_sign").pil_image,
121
- },
122
- },
123
- "decoder_prompt": "<|image|><|begin_of_text|>Please describe the image.", # noqa: E501
124
- },
125
- ]
126
-
127
- return ModelRequestData(
128
- engine_args=engine_args,
129
- prompts=prompts,
130
- )
131
-
132
-
24
def run_whisper():
25
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
26
@@ -166,9 +57,6 @@ def run_whisper():
57
58
59
model_example_map = {
169
- "donut": run_donut,
170
- "florence2": run_florence2,
171
- "mllama": run_mllama,
60
"whisper": run_whisper,
61
}
62
@@ -182,7 +70,7 @@ def parse_args():
70
"--model-type",
71
"-m",
72
type=str,
185
- default="mllama",
73
+ default="whisper",
74
choices=model_example_map.keys(),
75
help='Huggingface "model_type".',
76
)
examples/offline_inference/vision_language.py
-62
@@ -204,28 +204,6 @@ def run_ernie45_vl(questions: list[str], modality: str) -> ModelRequestData:
204
)
205
206
207
-# Florence2
208
-def run_florence2(questions: list[str], modality: str) -> ModelRequestData:
209
- assert modality == "image"
210
-
211
- engine_args = EngineArgs(
212
- model="microsoft/Florence-2-large",
213
- tokenizer="Isotr0py/Florence-2-tokenizer",
214
- max_model_len=4096,
215
- max_num_seqs=2,
216
- trust_remote_code=True,
217
- dtype="bfloat16",
218
- limit_mm_per_prompt={modality: 1},
219
- )
220
-
221
- prompts = ["<MORE_DETAILED_CAPTION>" for _ in questions]
222
-
223
- return ModelRequestData(
224
- engine_args=engine_args,
225
- prompts=prompts,
226
- )
227
-
228
-
207
# Fuyu
208
def run_fuyu(questions: list[str], modality: str) -> ModelRequestData:
209
assert modality == "image"
@@ -1008,44 +986,6 @@ def run_mistral3(questions: list[str], modality: str) -> ModelRequestData:
986
)
987
988
1011
-# LLama 3.2
1012
-def run_mllama(questions: list[str], modality: str) -> ModelRequestData:
1013
- assert modality == "image"
1014
-
1015
- model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
1016
-
1017
- # Note: The default setting of max_num_seqs (256) and
1018
- # max_model_len (131072) for this model may cause OOM.
1019
- # You may lower either to run this example on lower-end GPUs.
1020
-
1021
- # The configuration below has been confirmed to launch on a single L40 GPU.
1022
- engine_args = EngineArgs(
1023
- model=model_name,
1024
- max_model_len=8192,
1025
- max_num_seqs=2,
1026
- limit_mm_per_prompt={modality: 1},
1027
- )
1028
-
1029
- tokenizer = AutoTokenizer.from_pretrained(model_name)
1030
- messages = [
1031
- [
1032
- {
1033
- "role": "user",
1034
- "content": [{"type": "image"}, {"type": "text", "text": question}],
1035
- }
1036
- ]
1037
- for question in questions
1038
- ]
1039
- prompts = tokenizer.apply_chat_template(
1040
- messages, add_generation_prompt=True, tokenize=False
1041
- )
1042
-
1043
- return ModelRequestData(
1044
- engine_args=engine_args,
1045
- prompts=prompts,
1046
- )
1047
-
1048
-
989
# Molmo
990
def run_molmo(questions: list[str], modality: str) -> ModelRequestData:
991
assert modality == "image"
@@ -1665,7 +1605,6 @@ model_example_map = {
1605
"command_a_vision": run_command_a_vision,
1606
"deepseek_vl_v2": run_deepseek_vl2,
1607
"ernie45_vl": run_ernie45_vl,
1668
- "florence2": run_florence2,
1608
"fuyu": run_fuyu,
1609
"gemma3": run_gemma3,
1610
"gemma3n": run_gemma3n,
@@ -1691,7 +1630,6 @@ model_example_map = {
1630
"minicpmv": run_minicpmv,
1631
"minimax_vl_01": run_minimax_vl_01,
1632
"mistral3": run_mistral3,
1694
- "mllama": run_mllama,
1633
"molmo": run_molmo,
1634
"nemotron_vl": run_nemotron_vl,
1635
"NVLM_D": run_nvlm_d,
examples/offline_inference/vision_language_multi_image.py
-21
@@ -637,26 +637,6 @@ def load_mistral3(question: str, image_urls: list[str]) -> ModelRequestData:
637
)
638
639
640
-def load_mllama(question: str, image_urls: list[str]) -> ModelRequestData:
641
- model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
642
-
643
- # The configuration below has been confirmed to launch on a single L40 GPU.
644
- engine_args = EngineArgs(
645
- model=model_name,
646
- max_model_len=8192,
647
- max_num_seqs=2,
648
- limit_mm_per_prompt={"image": len(image_urls)},
649
- )
650
-
651
- img_prompt = "Given the first image <|image|> and the second image<|image|>"
652
- prompt = f"<|begin_of_text|>{img_prompt}, {question}?"
653
- return ModelRequestData(
654
- engine_args=engine_args,
655
- prompt=prompt,
656
- image_data=[fetch_image(url) for url in image_urls],
657
- )
658
-
659
-
640
def load_nvlm_d(question: str, image_urls: list[str]) -> ModelRequestData:
641
model_name = "nvidia/NVLM-D-72B"
642
@@ -1253,7 +1233,6 @@ model_example_map = {
1233
"llava-next": load_llava_next,
1234
"llava-onevision": load_llava_onevision,
1235
"mistral3": load_mistral3,
1256
- "mllama": load_mllama,
1236
"NVLM_D": load_nvlm_d,
1237
"ovis": load_ovis,
1238
"ovis2_5": load_ovis2_5,
tests/core/block/test_block_manager.py
+1
-154
@@ -3,15 +3,12 @@
3
4
import pytest
5
6
-from vllm.core.block.utils import (STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE,
7
- STR_NOT_IMPL_ENC_DEC_SWA)
6
from vllm.core.block_manager import SelfAttnBlockSpaceManager
7
from vllm.core.interfaces import AllocStatus
8
from vllm.sequence import Logprob, SequenceStatus
9
from vllm.utils import chunk_list
10
13
-from ..utils import (create_dummy_prompt, create_seq_group,
14
- create_seq_group_encoder_decoder)
11
+from ..utils import create_dummy_prompt, create_seq_group
12
13
14
@pytest.mark.parametrize("block_size", [16])
@@ -58,156 +55,6 @@ def test_can_allocate_seq_group(block_size: int, num_seqs_per_group: int,
55
assert can_allocate_result == AllocStatus.LATER
56
57
61
-@pytest.mark.parametrize("block_size", [16])
62
-@pytest.mark.parametrize("num_gpu_blocks", [16, 80, 160])
63
-@pytest.mark.parametrize("num_seqs_per_group", [1, 4])
64
-@pytest.mark.parametrize("watermark", [0.0, 0.5])
65
-def test_can_allocate_seq_group_encoder_decoder(block_size: int,
66
- num_seqs_per_group: int,
67
- num_gpu_blocks: int,
68
- watermark: float):
69
- block_manager = SelfAttnBlockSpaceManager(
70
- block_size=block_size,
71
- num_gpu_blocks=num_gpu_blocks,
72
- num_cpu_blocks=1024,
73
- watermark=watermark,
74
- )
75
- num_watermark_blocks = int(watermark * num_gpu_blocks)
76
-
77
- num_output_blocks_per_seq = 1
78
-
79
- # NOTE: This should be num_output_blocks_per_seq * num_seqs_per_group, but
80
- # the current implementation assumes all seqs are new prompts / don't have
81
- # different output lens.
82
- num_output_blocks = num_output_blocks_per_seq
83
-
84
- for bdx, num_prompt_blocks in enumerate(
85
- range(1, num_gpu_blocks - num_output_blocks)):
86
- num_cross_blocks_per_seq = num_prompt_blocks
87
-
88
- seq_group = create_seq_group_encoder_decoder(
89
- seq_prompt_len=block_size * num_prompt_blocks,
90
- seq_output_lens=[
91
- block_size * num_output_blocks_per_seq
92
- for _ in range(num_seqs_per_group)
93
- ],
94
- request_id=str(bdx))
95
-
96
- assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
97
-
98
- can_allocate_result = block_manager.can_allocate(seq_group)
99
-
100
- num_required_blocks = num_prompt_blocks + \
101
- num_output_blocks + \
102
- num_cross_blocks_per_seq
103
-
104
- if num_gpu_blocks - num_required_blocks < num_watermark_blocks:
105
- assert can_allocate_result == AllocStatus.NEVER
106
- elif num_gpu_blocks >= num_required_blocks:
107
- assert can_allocate_result == AllocStatus.OK
108
- else:
109
- assert can_allocate_result == AllocStatus.LATER
110
-
111
-
112
-@pytest.mark.parametrize("block_size", [16])
113
-@pytest.mark.parametrize("num_gpu_blocks", [16])
114
-@pytest.mark.parametrize("num_seqs_per_group", [1])
115
-@pytest.mark.parametrize("watermark", [0.0, 0.5])
116
-def test_can_allocate_encoder_decoder_fails_with_swa(block_size: int,
117
- num_seqs_per_group: int,
118
- num_gpu_blocks: int,
119
- watermark: float):
120
- '''
121
- SWA short for Sliding Window Attention.
122
-
123
- At time of writing block manager does not support SWA.
124
-
125
- However even when SWA is implemented for block manager,
126
- there will still most likely be a separate workstream required
127
- to enable SWA for encoder/decoder models.
128
-
129
- Therefore this test enforces that one of the following cases
130
- hold true:
131
- 1. Block manager does not support SWA at all (true at time of writing)
132
- 2. Block manager fails with NotImplementError when SWA is enabled
133
- AND a SequenceGroup with an encoder sequence (i.e. in support of an
134
- encoder/decoder model) is passed into can_allocate() as an argument
135
-
136
- The setup for this test is stripped down version of
137
- test_can_allocate_seq_group_encoder_decoder()
138
- '''
139
-
140
- with pytest.raises((NotImplementedError, AssertionError)) as exc_info:
141
- block_manager = SelfAttnBlockSpaceManager(
142
- block_size=block_size,
143
- num_gpu_blocks=num_gpu_blocks,
144
- num_cpu_blocks=1024,
145
- watermark=watermark,
146
- sliding_window=5 # SWA
147
- )
148
-
149
- num_output_blocks_per_seq = 1
150
- num_prompt_blocks = 1
151
- num_output_blocks = num_output_blocks_per_seq
152
- seq_group = create_seq_group_encoder_decoder(
153
- seq_prompt_len=block_size * num_prompt_blocks,
154
- seq_output_lens=[
155
- block_size * num_output_blocks_per_seq
156
- for _ in range(num_seqs_per_group)
157
- ],
158
- request_id="0")
159
-
160
- assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
161
- block_manager.can_allocate(seq_group)
162
-
163
- # Assert that either
164
- # 1. Block manager constructor fails with assertion that sliding window
165
- # is not yet supported (most likely near-term outcome at time of
166
- # writing), or
167
- # 2. can_allocate() fails with NotImplementedError due to combination of
168
- # encoder/decoder and sliding window attention
169
- if isinstance(exc_info.value, NotImplementedError):
170
- assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_SWA
171
- elif isinstance(exc_info.value, AssertionError):
172
- assert str(exc_info.value) == "Sliding window not yet supported"
173
-
174
-
175
-@pytest.mark.parametrize("block_size", [16])
176
-@pytest.mark.parametrize("num_gpu_blocks", [16])
177
-@pytest.mark.parametrize("num_seqs_per_group", [1])
178
-@pytest.mark.parametrize("watermark", [0.0, 0.5])
179
-def test_can_allocate_encoder_decoder_fails_with_prefix_cache(
180
- block_size: int, num_seqs_per_group: int, num_gpu_blocks: int,
181
- watermark: float):
182
-
183
- block_manager = SelfAttnBlockSpaceManager(
184
- block_size=block_size,
185
- num_gpu_blocks=num_gpu_blocks,
186
- num_cpu_blocks=1024,
187
- watermark=watermark,
188
- enable_caching=True # Prefix cache
189
- )
190
-
191
- num_output_blocks_per_seq = 1
192
- num_prompt_blocks = 1
193
- num_output_blocks = num_output_blocks_per_seq
194
- seq_group = create_seq_group_encoder_decoder(
195
- seq_prompt_len=block_size * num_prompt_blocks,
196
- seq_output_lens=[
197
- block_size * num_output_blocks_per_seq
198
- for _ in range(num_seqs_per_group)
199
- ],
200
- request_id="0")
201
-
202
- assert num_prompt_blocks + num_output_blocks <= num_gpu_blocks
203
-
204
- # Assert that either can_allocate() fails with NotImplementedError
205
- # due to combination of encoder/decoder and prefix cache
206
- with pytest.raises(NotImplementedError) as exc_info:
207
- block_manager.can_allocate(seq_group)
208
- assert str(exc_info.value) == STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE
209
-
210
-
58
@pytest.mark.parametrize("block_size", [1, 8])
59
@pytest.mark.parametrize("prompt_len", [1, 7, 8])
60
@pytest.mark.parametrize("num_slots_to_append", [1, 8, 129])
tests/core/test_scheduler_encoder_decoder.py
deleted
-105
@@ -1,105 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest # noqa
5
-
6
-from vllm.config import CacheConfig, SchedulerConfig
7
-from vllm.core.scheduler import Scheduler
8
-from vllm.sequence import SequenceGroup
9
-
10
-from .utils import (append_new_token, create_dummy_prompt_encoder_decoder,
11
- get_sequence_groups, schedule_and_update_computed_tokens)
12
-
13
-
14
-def test_scheduler_schedule_simple_encoder_decoder():
15
- '''
16
- Test basic scheduler functionality in the context
17
- of an encoder/decoder model. Focus on testing
18
- enc/dec-specific functionality sense tests already
19
- exist for decoder-only functionality
20
-
21
- Test behavior:
22
- * Construct Scheduler
23
- * Construct dummy encoder/decoder sequence groups
24
- * Add dummy seq groups to scheduler backlog
25
- * Schedule the next seq group & validate:
26
- * Cross-attn block tables
27
- * Updated states of seq groups
28
- * Number of batched tokens
29
- * Number of blocks to copy/swap-in/swap-out
30
- * Number of scheduled seq groups
31
- * Repeat for both prefill- and decode-phase
32
- * Abort scheduled seq groups
33
- * Assert that aborted seq groups no longer appear in
34
- cross-attention block table
35
- '''
36
-
37
- block_size = 4
38
- num_seq_group = 4
39
- max_model_len = 16
40
- scheduler_config = SchedulerConfig(
41
- "generate",
42
- max_num_batched_tokens=64,
43
- max_num_seqs=num_seq_group,
44
- max_model_len=max_model_len,
45
- )
46
- cache_config = CacheConfig(block_size, 1.0, 1, "auto")
47
- cache_config.num_cpu_blocks = 16 # enc and dec prompts per seq_group
48
- cache_config.num_gpu_blocks = 16 # enc and dec prompts per seq_group
49
- scheduler = Scheduler(scheduler_config, cache_config, None)
50
- running: list[SequenceGroup] = []
51
-
52
- # Add seq groups to scheduler.
53
- req_id_list = []
54
- for i in range(num_seq_group):
55
- req_id = str(i)
56
- req_id_list.append(req_id)
57
- _, _, seq_group = create_dummy_prompt_encoder_decoder(
58
- req_id, block_size, block_size, block_size)
59
- scheduler.add_seq_group(seq_group)
60
- running.append(seq_group)
61
-
62
- # Schedule seq groups prefill.
63
- num_tokens = block_size * num_seq_group
64
- seq_group_meta_list, out = schedule_and_update_computed_tokens(scheduler)
65
- # - Verify that sequence group cross-attention block tables are
66
- # registered with the block manager
67
- assert all([(req_id in scheduler.block_manager.cross_block_tables)
68
- for req_id in req_id_list])
69
- # - Validate sequence-group status
70
- assert set(get_sequence_groups(out)) == set(running)
71
- # - Validate number of batched tokens
72
- assert out.num_batched_tokens == num_tokens
73
- # - Validate there are no remaining blocks to swap
74
- assert (not out.blocks_to_copy and not out.blocks_to_swap_in
75
- and not out.blocks_to_swap_out)
76
- # - Validate all seq groups were scheduled
77
- assert len(seq_group_meta_list) == num_seq_group
78
- append_new_token(out, 1)
79
-
80
- # Schedule seq groups decode.
81
- seq_group_meta_list, out = schedule_and_update_computed_tokens(scheduler)
82
- # - Verify that sequence group metadata includes encoder attention
83
- # and cross-attention metadata
84
- assert all([
85
- not ((seq_group_meta.encoder_seq_data is None) or
86
- (seq_group_meta.cross_block_table is None))
87
- for seq_group_meta in seq_group_meta_list
88
- ])
89
- # - Validate sequence-group status
90
- assert set(get_sequence_groups(out)) == set(running)
91
- # - Validate there is one batched token per seq group
92
- assert out.num_batched_tokens == num_seq_group
93
- # - Validate there are no remaining blocks to swap
94
- assert (not out.blocks_to_copy and not out.blocks_to_swap_in
95
- and not out.blocks_to_swap_out)
96
- # - Validate that all seq groups were scheduled
97
- assert len(seq_group_meta_list) == num_seq_group
98
- append_new_token(out, 1)
99
-
100
- # Abort sequences
101
- for req_id in req_id_list:
102
- scheduler.abort_seq_group(req_id)
103
- # - Verify that sequence group cross-attention block tables are
104
- # NO LONGER registered with the block manager
105
- assert req_id not in scheduler.block_manager.cross_block_tables
tests/distributed/test_pipeline_parallel.py
-3
@@ -242,9 +242,6 @@ MULTIMODAL_MODELS = {
242
"Qwen/Qwen2-Audio-7B-Instruct": PPTestSettings.fast(),
243
"Qwen/Qwen2-VL-2B-Instruct": PPTestSettings.fast(),
244
"fixie-ai/ultravox-v0_5-llama-3_2-1b": PPTestSettings.fast(),
245
- # [Encoder-decoder]
246
- # TODO: Implement PP
247
- # "meta-llama/Llama-3.2-11B-Vision-Instruct": PPTestSettings.fast(),
245
}
246
# yapf: enable
247
tests/encoder_decoder/__init__.py
tests/encoder_decoder/test_e2e_correctness.py
deleted
-131
@@ -1,131 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""E2E tests to verify the correctness of the encoder-decoder framework
4
-
5
-Run `pytest tests/encoder_decoder/test_e2e_correctness.py`.
6
-"""
7
-from typing import Optional
8
-
9
-import pytest
10
-from transformers import AutoModelForSeq2SeqLM
11
-
12
-from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
13
- global_force_attn_backend_context_manager)
14
-from vllm.platforms import current_platform
15
-from vllm.sequence import SampleLogprobs
16
-
17
-from ..conftest import DecoderPromptType
18
-from ..models.utils import check_logprobs_close
19
-
20
-LIST_ENC_DEC_SUPPORTED_BACKENDS = [
21
- _Backend.XFORMERS, _Backend.FLASH_ATTN, None
22
-]
23
-
24
-
25
-@pytest.fixture(scope="function", autouse=True)
26
-def use_v0_only(monkeypatch):
27
- """
28
- Since this module is V0 only, set VLLM_USE_V1=0 for
29
- all tests in the module.
30
- """
31
- monkeypatch.setenv('VLLM_USE_V1', '0')
32
-
33
-
34
-def vllm_to_hf_output(
35
- vllm_output: tuple[list[int], str, Optional[SampleLogprobs]],
36
- decoder_prompt_type: DecoderPromptType,
37
-):
38
- """Sanitize vllm output to be comparable with hf output."""
39
- output_ids, output_str, out_logprobs = vllm_output
40
-
41
- hf_output_str = output_str + "</s>"
42
- if decoder_prompt_type == DecoderPromptType.NONE:
43
- hf_output_str = "<s>" + hf_output_str
44
-
45
- return output_ids, hf_output_str, out_logprobs
46
-
47
-
48
-@pytest.fixture(autouse=True)
49
-def clear_cache():
50
- """Fixture to clear backend cache before each test."""
51
- _cached_get_attn_backend.cache_clear() # Clear the cache
52
- yield # This allows the test to run
53
-
54
-
55
-@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
56
-@pytest.mark.parametrize("dtype", ["float"])
57
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
58
-@pytest.mark.parametrize("max_tokens", [128])
59
-@pytest.mark.parametrize("num_logprobs", [5])
60
-@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
61
-@pytest.mark.parametrize("enforce_eager", [True, False])
62
-@pytest.mark.skipif(
63
- current_platform.is_cpu(),
64
- reason="CPU backend is not currently supported with encoder/decoder models"
65
-)
66
-@pytest.mark.skip(reason="bart not supported in V1")
67
-def test_encoder_decoder_e2e(
68
- hf_runner,
69
- vllm_runner,
70
- example_encoder_decoder_prompts,
71
- model: str,
72
- dtype: str,
73
- max_tokens: int,
74
- num_logprobs: int,
75
- decoder_prompt_type: DecoderPromptType,
76
- enforce_eager: bool,
77
- attn_backend: _Backend,
78
-) -> None:
79
- '''
80
- End-to-End (E2E) test for the encoder-decoder framework.
81
- This test evaluates the encoder-decoder functionality using the BART
82
- model. We compare the outputs of the Hugging Face and vLLM
83
- implementations to ensure that both implementations produce consistent
84
- and correct results.
85
- '''
86
- with global_force_attn_backend_context_manager(attn_backend):
87
- if attn_backend == _Backend.FLASH_ATTN:
88
- # Flash Attention works only with bfloat16 data-type
89
- dtype = 'bfloat16'
90
- test_case_prompts = example_encoder_decoder_prompts[
91
- decoder_prompt_type]
92
-
93
- # Configuration settings for HF baseline
94
- hf_kwargs = {
95
- "top_k": None,
96
- "num_beams": 1,
97
- "repetition_penalty": 1.0,
98
- "top_p": 1.0,
99
- "length_penalty": 1.0,
100
- "early_stopping": False,
101
- "no_repeat_ngram_size": None,
102
- "min_length": 0
103
- }
104
-
105
- with hf_runner(model, dtype=dtype,
106
- auto_cls=AutoModelForSeq2SeqLM) as hf_model:
107
- hf_outputs = (
108
- hf_model.generate_encoder_decoder_greedy_logprobs_limit(
109
- test_case_prompts,
110
- max_tokens,
111
- num_logprobs,
112
- **hf_kwargs,
113
- ))
114
- with vllm_runner(model, dtype=dtype,
115
- enforce_eager=enforce_eager) as vllm_model:
116
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
117
- test_case_prompts, max_tokens, num_logprobs)
118
-
119
- hf_skip_tokens = (1 if decoder_prompt_type == DecoderPromptType.NONE
120
- else 0)
121
-
122
- check_logprobs_close(
123
- outputs_0_lst=hf_outputs,
124
- outputs_1_lst=[
125
- vllm_to_hf_output(vllm_output, decoder_prompt_type)
126
- for vllm_output in vllm_outputs
127
- ],
128
- name_0="hf",
129
- name_1="vllm",
130
- num_outputs_0_skip_tokens=hf_skip_tokens,
131
- )
tests/entrypoints/openai/test_encoder_decoder.py
deleted
-56
@@ -1,56 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import openai
5
-import pytest
6
-import pytest_asyncio
7
-
8
-from ...utils import RemoteOpenAIServer
9
-
10
-MODEL_NAME = "facebook/bart-base"
11
-
12
-
13
-@pytest.fixture(scope="module")
14
-def server():
15
- args = [
16
- "--dtype",
17
- "bfloat16",
18
- "--enforce-eager",
19
- ]
20
-
21
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
22
- yield remote_server
23
-
24
-
25
-@pytest_asyncio.fixture
26
-async def client(server):
27
- async with server.get_async_client() as async_client:
28
- yield async_client
29
-
30
-
31
-@pytest.mark.asyncio
32
-@pytest.mark.parametrize("model_name", [MODEL_NAME])
33
-@pytest.mark.skip(reason="bart is not yet supported in V1")
34
-async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):
35
- completion = await client.completions.create(model=model_name,
36
- prompt="Hello, my name is",
37
- max_tokens=5,
38
- temperature=0.0)
39
-
40
- assert completion.id is not None
41
- assert completion.choices is not None and len(completion.choices) == 1
42
-
43
- choice = completion.choices[0]
44
- assert len(choice.text) >= 5
45
- assert choice.finish_reason == "length"
46
- assert completion.usage == openai.types.CompletionUsage(
47
- completion_tokens=5, prompt_tokens=2, total_tokens=7)
48
-
49
- # test using token IDs
50
- completion = await client.completions.create(
51
- model=model_name,
52
- prompt=[0, 0, 0, 0, 0],
53
- max_tokens=5,
54
- temperature=0.0,
55
- )
56
- assert len(completion.choices[0].text) >= 1
tests/entrypoints/test_chat_utils.py
-199
@@ -20,7 +20,6 @@ from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,
20
parse_chat_messages_futures,
21
resolve_chat_template_content_format,
22
resolve_hf_chat_template)
23
-from vllm.entrypoints.llm import apply_hf_chat_template
23
from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict
24
from vllm.multimodal.utils import (encode_audio_base64, encode_image_base64,
25
encode_video_base64)
@@ -38,7 +37,6 @@ QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct"
37
QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct"
38
QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct"
39
QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B"
41
-MLLAMA_MODEL_ID = "meta-llama/Llama-3.2-11B-Vision-Instruct"
40
LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B"
41
HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
42
MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
@@ -125,27 +123,6 @@ def qwen25omni_tokenizer():
123
)
124
125
128
-@pytest.fixture(scope="module")
129
-def mllama_model_config():
130
- return ModelConfig(
131
- MLLAMA_MODEL_ID,
132
- runner="generate",
133
- limit_mm_per_prompt={
134
- "image": 2,
135
- },
136
- )
137
-
138
-
139
-@pytest.fixture(scope="module")
140
-def mllama_tokenizer():
141
- return TokenizerGroup(
142
- MLLAMA_MODEL_ID,
143
- enable_lora=False,
144
- max_num_seqs=5,
145
- max_input_length=None,
146
- )
147
-
148
-
126
@pytest.fixture(scope="function")
127
def mistral_model_config():
128
return ModelConfig(
@@ -2249,180 +2226,6 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders(
2226
)
2227
2228
2252
-### Mllama currently wraps images / texts as interleaved dictionaries
2253
-def test_mllama_single_image(
2254
- mllama_model_config,
2255
- mllama_tokenizer,
2256
- image_url,
2257
-):
2258
- """Ensures that a single image is parsed correctly mllama."""
2259
- conversation, mm_data, mm_uuids = parse_chat_messages(
2260
- [{
2261
- "role":
2262
- "user",
2263
- "content": [
2264
- {
2265
- "type": "text",
2266
- "text": "The content of this image is:"
2267
- },
2268
- {
2269
- "image_url": image_url
2270
- },
2271
- ],
2272
- }],
2273
- mllama_model_config,
2274
- mllama_tokenizer,
2275
- content_format="openai",
2276
- )
2277
- _assert_mm_data_is_image_input(mm_data, 1)
2278
- _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None])
2279
- assert conversation == [{
2280
- "role":
2281
- "user",
2282
- "content": [
2283
- {
2284
- "type": "text",
2285
- "text": "The content of this image is:"
2286
- },
2287
- {
2288
- "type": "image"
2289
- },
2290
- ],
2291
- }]
2292
-
2293
-
2294
-def test_mllama_interleaved_images(
2295
- mllama_model_config,
2296
- mllama_tokenizer,
2297
- image_url,
2298
-):
2299
- """Ensures that multiple image are parsed as interleaved dicts."""
2300
- conversation, mm_data, mm_uuids = parse_chat_messages(
2301
- [{
2302
- "role":
2303
- "user",
2304
- "content": [
2305
- {
2306
- "type": "text",
2307
- "text": "The content of the first image is:",
2308
- },
2309
- {
2310
- "image_url": image_url
2311
- },
2312
- {
2313
- "type": "text",
2314
- "text": "The content of the second image is:",
2315
- },
2316
- {
2317
- "image_url": image_url
2318
- },
2319
- ],
2320
- }],
2321
- mllama_model_config,
2322
- mllama_tokenizer,
2323
- content_format="openai",
2324
- )
2325
- _assert_mm_data_is_image_input(mm_data, 2)
2326
- _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None])
2327
- assert conversation == [{
2328
- "role":
2329
- "user",
2330
- "content": [
2331
- {
2332
- "type": "text",
2333
- "text": "The content of the first image is:"
2334
- },
2335
- {
2336
- "type": "image"
2337
- },
2338
- {
2339
- "type": "text",
2340
- "text": "The content of the second image is:"
2341
- },
2342
- {
2343
- "type": "image"
2344
- },
2345
- ],
2346
- }]
2347
-
2348
-
2349
-@pytest.mark.parametrize("model", [MLLAMA_MODEL_ID])
2350
-def test_multimodal_image_parsing_matches_hf(model, image_url):
2351
- """Checks end to end hf alignment for multimodal [image] parsing."""
2352
-
2353
- def get_conversation(is_hf: bool):
2354
- img_part = {"type": "image_url", "image_url": {"url": image_url}}
2355
- if is_hf:
2356
- img_part = {"type": "image"}
2357
- return [{
2358
- "role":
2359
- "user",
2360
- "content": [
2361
- {
2362
- "type": "text",
2363
- "text": "The content of the first image is:",
2364
- },
2365
- img_part,
2366
- {
2367
- "type": "text",
2368
- "text": "The content of the second image is:",
2369
- },
2370
- img_part,
2371
- {
2372
- "type": "text",
2373
- "text": "What animal is in the first image?",
2374
- },
2375
- ],
2376
- }]
2377
-
2378
- # Build a config for the model
2379
- model_config = ModelConfig(
2380
- model,
2381
- runner="generate",
2382
- limit_mm_per_prompt={
2383
- "image": 2,
2384
- },
2385
- )
2386
-
2387
- # Build the tokenizer group and grab the underlying tokenizer
2388
- tokenizer_group = TokenizerGroup(
2389
- model,
2390
- enable_lora=False,
2391
- max_num_seqs=5,
2392
- max_input_length=None,
2393
- trust_remote_code=model_config.trust_remote_code,
2394
- )
2395
- tokenizer = tokenizer_group.tokenizer
2396
-
2397
- # Build and parse a conversation with {"type": "image"} using the tokenizer
2398
- hf_conversation = get_conversation(is_hf=True)
2399
- hf_result = tokenizer.apply_chat_template(
2400
- hf_conversation,
2401
- tokenize=False,
2402
- add_generation_prompt=True,
2403
- )
2404
-
2405
- # Now parse with vLLMs chat utils & apply the template
2406
- vllm_conversation = get_conversation(is_hf=False)
2407
- conversation, _, _ = parse_chat_messages(
2408
- vllm_conversation,
2409
- model_config,
2410
- tokenizer_group,
2411
- content_format="openai",
2412
- )
2413
-
2414
- vllm_result = apply_hf_chat_template(
2415
- tokenizer=tokenizer,
2416
- conversation=conversation,
2417
- chat_template=None,
2418
- model_config=model_config,
2419
- tools=None,
2420
- add_generation_prompt=True,
2421
- )
2422
-
2423
- assert hf_result == vllm_result
2424
-
2425
-
2229
@pytest.mark.parametrize(
2230
"model",
2231
[
@@ -2486,7 +2289,6 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):
2289
(QWEN25VL_MODEL_ID, "openai"),
2290
(ULTRAVOX_MODEL_ID, "string"),
2291
(QWEN2AUDIO_MODEL_ID, "openai"),
2489
- (MLLAMA_MODEL_ID, "openai"),
2292
(LLAMA_GUARD_MODEL_ID, "openai")],
2293
)
2294
# yapf: enable
@@ -2545,7 +2347,6 @@ def test_resolve_content_format_hf_defined(model, expected_format):
2347
[("Salesforce/blip2-opt-2.7b", "string"),
2348
("facebook/chameleon-7b", "string"),
2349
("deepseek-ai/deepseek-vl2-tiny", "string"),
2548
- ("microsoft/Florence-2-base", "string"),
2350
("adept/fuyu-8b", "string"),
2351
("google/paligemma-3b-mix-224", "string"),
2352
("Qwen/Qwen-VL", "string"),
tests/kernels/attention/test_encoder_decoder_attn.py
deleted
-1105
@@ -1,1105 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-Tests:
5
-
6
-* E2E test of Encoder attention + Decoder self-attention +
7
- Encoder/decoder cross-attention (collectively
8
- "encoder/decoder attention")
9
-
10
-"""
11
-
12
-from typing import NamedTuple, Optional
13
-
14
-import pytest
15
-import torch
16
-
17
-from tests.kernels.utils import *
18
-from vllm.attention import Attention, AttentionMetadata, AttentionType
19
-from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
20
-from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
21
- global_force_attn_backend_context_manager)
22
-from vllm.config import VllmConfig, set_current_vllm_config
23
-from vllm.forward_context import set_forward_context
24
-from vllm.platforms import current_platform
25
-
26
-
27
-@pytest.fixture(scope="function", autouse=True)
28
-def use_v0_only(monkeypatch):
29
- """
30
- Encoder-decoder is only supported on V0, so set
31
- VLLM_USE_V1=0 for all tests in the module.
32
- """
33
- monkeypatch.setenv('VLLM_USE_V1', '0')
34
-
35
-
36
-# List of support backends for encoder/decoder models
37
-LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
38
-HEAD_SIZES = [64, 256]
39
-
40
-NUM_HEADS = [1, 16]
41
-
42
-BATCH_SIZES = [1, 16]
43
-BLOCK_SIZES = [16]
44
-CUDA_DEVICE = "cuda:0"
45
-
46
-MAX_DEC_SEQ_LENS = [128]
47
-MAX_ENC_SEQ_LENS = [128]
48
-
49
-# Narrow test-cases for unsupported-scenario
50
-# tests
51
-HEAD_SIZES_FOR_UNSUPP = [HEAD_SIZES[0]]
52
-
53
-
54
-class TestPoint(NamedTuple):
55
- """
56
- Encapsulates the attributes which define a single invocation
57
- of the test_e2e_enc_dec_attn() test
58
-
59
- Attributes:
60
- num_heads: The number of heads in the model.
61
- head_size: Head dimension
62
- backend_name: Name of the backend framework used.
63
- batch_size: Number of samples per batch.
64
- block_size: Size of each block of data processed.
65
- max_dec_seq_len: Maximum sequence length for the decoder.
66
- max_enc_seq_len: Maximum sequence length for the encoder.
67
- num_blocks: Number of blocks in the model.
68
- """
69
-
70
- num_heads: int
71
- head_size: int
72
- backend_name: str
73
- batch_size: int
74
- block_size: int
75
- max_dec_seq_len: int
76
- max_enc_seq_len: int
77
- num_blocks: int
78
- attn_type: AttentionType
79
-
80
-
81
-class TestResources(NamedTuple):
82
- '''
83
- Encapsulates key components for performing an
84
- encoder/decoder attention test
85
-
86
- Note that
87
- (1) attn automatically selects an attention backend
88
- based on platform info & a set of canned
89
- heuristics
90
- (2) attn_backend is thus *not the same backend
91
- instance* used by attn, but rather it is
92
- intended to be a
93
- *different instance* of the *same backend class*;
94
- it is assumed that the user of TestResources
95
- will leverage attn_backend for the purpose of
96
- constructing backend-compatible attention
97
- metadata instances
98
-
99
- Attributes:
100
-
101
- * scale: 1/sqrt(d) scale factor for attn
102
- * attn_backend: implementations of abstraction
103
- attention interface using
104
- a particular kernel library
105
- i.e. XFormers
106
- * attn: Attention layer instance
107
- * kv_cache: shared key/value cache for all attention
108
- '''
109
-
110
- scale: float
111
- attn: Attention
112
- kv_cache: torch.Tensor
113
-
114
-
115
-def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
116
- '''
117
- Build key components for performing encoder/decoder attention test.
118
-
119
- Note that
120
- (1) The Attention instance constructed here, automatically selects
121
- an attention backend class based on platform info & a set of canned
122
- heuristics, so
123
- (2) The attention backend instance constructed here is thus *not
124
- the same backend instance* used by attn, but rather it is
125
- intended to be a *different instance* of the *same backend class*;
126
- therefore,
127
- (3) This function requires that test_pt.backend_name matches the backend
128
- class that Attention will automatically select when it is constructed.
129
-
130
-
131
- Arguments:
132
-
133
- * test_pt: TestPoint data structure; this function relies on the
134
- following fields: num_heads, head_size, num_blocks,
135
- block_size, backend_name
136
-
137
- Returns:
138
-
139
- * TestResources data structure.
140
- '''
141
-
142
- scale = float(1.0 / (test_pt.head_size**0.5))
143
- attn = Attention(
144
- test_pt.num_heads,
145
- test_pt.head_size,
146
- scale=scale,
147
- prefix=f"{test_pt.attn_type}",
148
- attn_type=test_pt.attn_type,
149
- )
150
- if test_pt.num_blocks is None or test_pt.num_heads is None:
151
- # Caller does not require a KV cache
152
- return TestResources(
153
- scale, attn,
154
- torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
155
-
156
- # Construct KV cache
157
- if test_pt.attn_type in (AttentionType.DECODER,
158
- AttentionType.ENCODER_DECODER):
159
- kv_cache = make_kv_cache(test_pt.num_blocks,
160
- test_pt.num_heads,
161
- test_pt.head_size,
162
- test_pt.block_size,
163
- device=CUDA_DEVICE,
164
- backend=test_pt.backend_name)
165
- else:
166
- kv_cache = torch.tensor([])
167
-
168
- attn.kv_cache = [kv_cache]
169
- return TestResources(scale, attn, kv_cache)
170
-
171
-
172
-def _encoder_attn_setup(
173
- test_pt: TestPoint,
174
- test_rsrcs: TestResources,
175
-) -> PhaseTestParameters:
176
- '''
177
- Set up test vectors & data structures for encoder attention test.
178
-
179
- A triplet of synthetic query/key/value tensors are constructed.
180
- Given this is an encoder attention test, the key & value
181
- sequences will have the same length as the corresponding queries.
182
-
183
- The query/key/value tensors are passed to an ideal reference
184
- self-attention implementation to generate an ideal output tensor.
185
-
186
- Encoder inference does not populate the KV cache, therefore
187
- no KV cache memory mapping is constructed
188
-
189
- Arguments:
190
-
191
- * test_pt: TestPoint data structure; this function relies on the
192
- following fields: batch_size, num_heads, head_size,
193
- block_size, max_q_seq_len
194
- * test_rsrcs: TestResources data structure; this function relies on the
195
- scale field
196
-
197
-
198
- Returns:
199
-
200
- * PhaseTestParameters data structure comprising (1) packed query/key/value
201
- tensors, (2) the ideal output of attention computed using a naive
202
- implementation, and (3) KVCache field set to None
203
- '''
204
-
205
- (
206
- num_heads,
207
- head_size,
208
- _,
209
- batch_size,
210
- _,
211
- _,
212
- max_q_seq_len,
213
- _,
214
- _,
215
- ) = test_pt
216
-
217
- scale = test_rsrcs.scale
218
-
219
- max_kv_seq_len = max_q_seq_len
220
-
221
- # Make test tensors
222
-
223
- qkv_in, _, _ = make_qkv(batch_size,
224
- max_q_seq_len,
225
- max_kv_seq_len,
226
- num_heads,
227
- head_size,
228
- attn_type=AttentionType.ENCODER,
229
- device=CUDA_DEVICE)
230
-
231
- # Compute correct answer using naive non-causal attention
232
- # implementation
233
-
234
- ideal_output = ref_masked_attention(qkv_in.query,
235
- qkv_in.key,
236
- qkv_in.value,
237
- scale=scale,
238
- q_seq_lens=qkv_in.q_seq_lens,
239
- kv_seq_lens=qkv_in.kv_seq_lens)
240
-
241
- packed_ideal_output, _ = pack_tensor(ideal_output,
242
- qkv_in.q_seq_lens,
243
- device=CUDA_DEVICE)
244
-
245
- packed_qkv = pack_qkv(qkv_in, device=CUDA_DEVICE)
246
-
247
- return PhaseTestParameters(
248
- PackedQKVO(packed_qkv, packed_ideal_output),
249
- None # No KV cache
250
- )
251
-
252
-
253
-def _decoder_attn_setup(
254
- test_pt: TestPoint,
255
- test_rsrcs: TestResources,
256
- block_base_addr: int = 0,
257
-) -> tuple[QKVInputs, PhaseTestParameters, PhaseTestParameters, int]:
258
- '''
259
- Set up test vectors & data structures for self-attention test.
260
-
261
- A triplet of synthetic query/key/value tensors are constructed ("baseline"
262
- query/key/value). Given this is a self-attention test, the key & value
263
- sequences will have the same length as the corresponding queries.
264
-
265
- "Prefill" query/key/value tensors are derived by masking out the last value
266
- in each baseline query/key/value. These tensors are used to test prefill &
267
- populate KV cache for a subsequent decode test.
268
-
269
- "Decode" query/key/value tensors are derived by extracting *only* the last
270
- value from each baseline query/key/value (i.e. complement of the prefill
271
- tensors.) These tensors are used to test decode, conditional on the kv cache
272
- being populated during the prefill test.
273
-
274
- The baseline query/key/value tensors are passed to an ideal reference
275
- self-attention implementation to generate a "Baseline" ideal output tensor.
276
- This tensor is split into the "Prefill" ideal output tensor (all but the
277
- last element of each output sequence) and the "Decode" ideal output tensor
278
- (*only* the last element of each output sequence); the "Prefill" and
279
- "Decode" ideal output tensors can be used to validate the prefill and decode
280
- test results, respectively.
281
-
282
- This function also constructs the self-attention KV cache memory mapping
283
- (slot mapping and block table), ensuring that the block table starts at
284
- block_base_addr
285
-
286
- Arguments:
287
-
288
- * test_pt: TestPoint data structure; this function relies on the
289
- following fields: batch_size, num_heads, head_size,
290
- block_size, max_q_seq_len
291
- * test_rsrcs: TestResources data structure; this function relies on the
292
- scale field
293
- * block_base_addr: decoder self-attention block-table base address
294
-
295
- Returns:
296
- * qkv: Unpacked (batch_size x padded_seq_len x num_heads x
297
- head_size) query/key/value tensors
298
- * Prefill-phase decoder self-attention PhaseTestParameters data structure,
299
- including (1) packed (number_of_tokens x num_heads x head_size)
300
- query/key/value tensors along with (2) ideal attention output
301
- computed using a naive implementation, and (3) memory-mapping data
302
- structures appropriate for prefill phase.
303
- * Decode-phase decoder self-attention PhaseTestParameters data structure,
304
- including (1) packed (number_of_tokens x num_heads x head_size)
305
- query/key/value tensors along with (2) ideal attention output
306
- computed using a naive implementation, and (3) memory-mapping data
307
- structures appropriate for decode phase.
308
- * max_block_idx: max physical address in decoder self-attention block-table
309
- (intended to be used as the base address for the encoder/
310
- decoder cross-attention block-table, which is not
311
- constructed in this function)
312
- '''
313
-
314
- (
315
- num_heads,
316
- head_size,
317
- _,
318
- batch_size,
319
- block_size,
320
- max_q_seq_len,
321
- _,
322
- _,
323
- _,
324
- ) = test_pt
325
-
326
- scale = test_rsrcs.scale
327
-
328
- max_kv_seq_len = max_q_seq_len
329
-
330
- # Build test tensors
331
-
332
- (
333
- qkv,
334
- prefill_qkv,
335
- decode_qkv,
336
- ) = make_qkv(batch_size,
337
- max_q_seq_len,
338
- max_kv_seq_len,
339
- num_heads,
340
- head_size,
341
- attn_type=AttentionType.DECODER,
342
- device=CUDA_DEVICE)
343
-
344
- # Compute correct answer using naive attention implementation
345
- # with causal attention mask
346
-
347
- causal_mask = make_causal_mask(max_q_seq_len,
348
- max_kv_seq_len).to(CUDA_DEVICE)
349
-
350
- ideal_output = ref_masked_attention(qkv.query,
351
- qkv.key,
352
- qkv.value,
353
- scale=scale,
354
- custom_mask=causal_mask,
355
- q_seq_lens=qkv.q_seq_lens,
356
- kv_seq_lens=qkv.kv_seq_lens)
357
-
358
- # Split out the prefill- & decode-phase ideal answers & pack them
359
-
360
- prefill_ideal_output = torch.zeros_like(ideal_output)
361
- decode_ideal_output = torch.zeros_like(ideal_output[:, 0:1])
362
- for bdx, prefill_q_seq_len in enumerate(prefill_qkv.q_seq_lens):
363
- prefill_ideal_output[bdx, :prefill_q_seq_len] = ideal_output[
364
- bdx, :prefill_q_seq_len]
365
- decode_ideal_output[bdx, :] = ideal_output[bdx, prefill_q_seq_len:(
366
- prefill_q_seq_len + 1)]
367
-
368
- prefill_packed_ideal_output, _ = pack_tensor(prefill_ideal_output,
369
- prefill_qkv.q_seq_lens,
370
- device=CUDA_DEVICE)
371
- decode_packed_ideal_output, _ = pack_tensor(decode_ideal_output,
372
- [1 for _ in range(batch_size)],
373
- device=CUDA_DEVICE)
374
-
375
- # Build prefill- & decode-phase data structures
376
- # for decoder self-attention. Block tables and
377
- # slot mapping must be in a format compatible
378
- # with KV caching & attention kernels
379
- #
380
- # Prefill-phase:
381
- #
382
- # * Empty block-tables tensor
383
- # * Slot-mapping with entries for prompt tokens
384
- #
385
- # Decode-phase:
386
- # * Block-tables tensor with minimum number of blocks
387
- # required by total num. tokens in the entirety of all sequences
388
- # (including both prefill & decode)
389
- # * Slot-mapping with entries for tokens that will be decoded in the
390
- # current decode iteration
391
- #
392
- # Note: the format described above is simply mirroring what ModelRunner
393
- # produces
394
-
395
- prefill_block_tables = make_empty_block_tables_tensor(device=CUDA_DEVICE)
396
-
397
- (
398
- decode_block_tables,
399
- slot_mapping_list,
400
- max_block_idx,
401
- ) = make_block_tables_slot_mapping(block_size,
402
- qkv.q_seq_lens,
403
- device=CUDA_DEVICE,
404
- block_base_addr=block_base_addr)
405
-
406
- (
407
- prefill_slot_mapping,
408
- decode_slot_mapping,
409
- ) = split_slot_mapping(slot_mapping_list,
410
- qkv.q_seq_lens,
411
- device=CUDA_DEVICE)
412
-
413
- prefill_pckd_qkv = pack_qkv(prefill_qkv, device=CUDA_DEVICE)
414
-
415
- decode_pckd_qkv = pack_qkv(decode_qkv, device=CUDA_DEVICE)
416
-
417
- return (
418
- qkv,
419
- PhaseTestParameters( # Prefill test params
420
- PackedQKVO(prefill_pckd_qkv, prefill_packed_ideal_output),
421
- KVMemoryMap(prefill_block_tables, prefill_slot_mapping)),
422
- PhaseTestParameters( # Decode test params
423
- PackedQKVO(decode_pckd_qkv, decode_packed_ideal_output),
424
- KVMemoryMap(decode_block_tables, decode_slot_mapping)),
425
- max_block_idx)
426
-
427
-
428
-def _enc_dec_cross_attn_setup_reuses_query(
429
- decoder_qkv: QKVInputs,
430
- encoder_test_params: PhaseTestParameters,
431
- prefill_decoder_phase_test_params: PhaseTestParameters,
432
- test_pt: TestPoint,
433
- test_rsrcs: TestResources,
434
- block_base_addr: int = 0,
435
-) -> tuple[PhaseTestParameters, PhaseTestParameters]:
436
- '''
437
- Set up test vectors & data structures for cross-attention test.
438
-
439
- A triplet of synthetic cross-attention key/value tensors are constructed
440
- ("baseline" key/value). Given this is a cross-attention test, we assume
441
- query tensors were already synthesized for a prior self-attention test and
442
- will be reused for cross-attention. The key & value sequences generated here
443
- may have a different length than the corresponding queries (as is often
444
- the case for cross-attention between decoder and encoder sequences.)
445
-
446
- Cross attention key & value tensors do not grow during autoregressive
447
- inference; thus this function obtains a single key/value pair suitable for
448
- both prefill and decode.
449
-
450
- The "baseline" query tensor is received as an argument. The "baseline"
451
- query/key/value tensors are passed to an ideal reference cross-attention
452
- implementation to generate a "baseline" ideal output tensor. This tensor is
453
- split into the "Prefill" ideal output tensor (all but the last element of
454
- each output sequence) and the "Decode" ideal output tensor (*only* the last
455
- element of each output sequence); the "Prefill" and "Decode" ideal output
456
- tensors can be used to validate the prefill and decode test results,
457
- respectively.
458
-
459
- This function also constructs the cross-attention KV cache memory mapping
460
- (slot mapping and block table), ensuring that the block table starts at
461
- block_base_addr.
462
-
463
- Arguments:
464
-
465
- * decoder_qkv: pre-existing unpacked (batch_size x padded_seq_len x
466
- num_heads x head_size) decoder self-attention inputs;
467
- this function relies on the query and q_seq_lens
468
- fields
469
- * encoder_test_params: PhaseTestParameters data structure which was
470
- used for encoder inference; KV cache field
471
- is not used by this function
472
- * prefill_decoder_phase_test_params: PhaseTestParameters data structure
473
- used for prefill-phase decoder
474
- self-attention; all fields
475
- including KV cache required
476
- * test_pt: TestPoint data structure; this function relies on the
477
- following fields: batch_size, num_heads, head_size,
478
- block_size, max_q_seq_len
479
- * test_rsrcs: TestResources data structure; this function relies on the
480
- scale field
481
- * block_base_addr: decoder self-attention block-table base address
482
-
483
- Returns:
484
-
485
- * Prefill-phase encoder/decoder cross-attention PhaseTestParameters data
486
- structure, including (1) packed
487
- (number_of_tokens x num_heads x head_size) query/key/value tensors
488
- along with (2) ideal attention output computed using a
489
- naive implementation, and (3) memory-mapping data structures appropriate
490
- for prefill phase.
491
- * Decode-phase encoder/decoder cross-attention PhaseTestParameters data
492
- structure, including (1) packed
493
- (number_of_tokens x num_heads x head_size) query/key/value tensors
494
- along with (2) ideal attention output computed using a
495
- naive implementation, and (3) memory-mapping data structures appropriate
496
- for decode phase.
497
- '''
498
-
499
- assert encoder_test_params.packed_qkvo.packed_qkv is not None
500
- assert prefill_decoder_phase_test_params.packed_qkvo.packed_qkv is not None
501
-
502
- (
503
- num_heads,
504
- head_size,
505
- _,
506
- batch_size,
507
- block_size,
508
- max_decoder_seq_len,
509
- max_encoder_seq_len,
510
- _,
511
- _,
512
- ) = test_pt
513
-
514
- scale = test_rsrcs.scale
515
-
516
- decoder_query = decoder_qkv.query
517
- decoder_seq_lens = decoder_qkv.q_seq_lens
518
- encoder_seq_lens = encoder_test_params.packed_qkvo.packed_qkv.q_seq_lens
519
- prefill_q_seq_lens = (
520
- prefill_decoder_phase_test_params.packed_qkvo.packed_qkv.q_seq_lens)
521
-
522
- assert prefill_q_seq_lens is not None
523
-
524
- (
525
- cross_kv,
526
- _,
527
- _,
528
- ) = make_qkv(batch_size,
529
- max_decoder_seq_len,
530
- max_encoder_seq_len,
531
- num_heads,
532
- head_size,
533
- force_kv_seq_lens=encoder_seq_lens,
534
- attn_type=AttentionType.ENCODER_DECODER,
535
- device=CUDA_DEVICE)
536
-
537
- ideal_output = ref_masked_attention(decoder_query,
538
- cross_kv.key,
539
- cross_kv.value,
540
- scale=scale,
541
- q_seq_lens=decoder_seq_lens,
542
- kv_seq_lens=cross_kv.kv_seq_lens)
543
-
544
- prefill_ideal_output = torch.zeros_like(ideal_output)
545
- decode_ideal_output = torch.zeros_like(ideal_output[:, 0:1])
546
- for bdx, prefill_q_seq_len in enumerate(prefill_q_seq_lens):
547
- prefill_ideal_output[bdx, :prefill_q_seq_len] = ideal_output[
548
- bdx, :prefill_q_seq_len]
549
- decode_ideal_output[bdx, :] = ideal_output[bdx, prefill_q_seq_len:(
550
- prefill_q_seq_len + 1)]
551
-
552
- prefill_packed_ideal_output, _ = pack_tensor(prefill_ideal_output,
553
- prefill_q_seq_lens,
554
- device=CUDA_DEVICE)
555
- decode_packed_ideal_output, _ = pack_tensor(decode_ideal_output,
556
- [1 for _ in range(batch_size)],
557
- device=CUDA_DEVICE)
558
-
559
- # Build prefill- & decode-phase data structures
560
- # for encoder/decoder cross-attention. Block tables and
561
- # slot mapping must be in a format compatible
562
- # with KV caching & attention kernels
563
- #
564
- # Whereas decoder self-attention extracts relationships between
565
- # equal-length Q/K/V sequences, which mutually grow in length
566
- # with each decoded token, cross-attention relates the Q sequence
567
- # - which grows with each new decoded token - to fixed-length
568
- # K and V sequences derived from the encoder hidden states.
569
- #
570
- # Prefill-phase:
571
- #
572
- # * Empty block-tables tensor
573
- # * Slot-mapping with as many entries as there are tokens in the encoder
574
- # prompt.
575
- #
576
- # Decode-phase:
577
- # * Block-tables tensor with minimum number of blocks to
578
- # accommodate K & V tensors which are equal in lnegth
579
- # to the encoder prompt length
580
- # * Empty slot-mapping tensor (since K & V are fixed in size,
581
- # new decoded tokens are not KV-cached and require no slot-
582
- # mapping)
583
- #
584
- # Note: the format above is simply an extension of what ModelRunner
585
- # produces for decoder-only models
586
-
587
- prefill_block_tables = make_empty_block_tables_tensor(device=CUDA_DEVICE)
588
- decode_slot_mapping = make_empty_slot_mapping_tensor(device=CUDA_DEVICE)
589
-
590
- (
591
- decode_block_tables,
592
- prefill_slot_mapping_list,
593
- _,
594
- ) = make_block_tables_slot_mapping(block_size,
595
- cross_kv.kv_seq_lens,
596
- block_base_addr=block_base_addr,
597
- device=CUDA_DEVICE)
598
-
599
- prefill_slot_mapping = maybe_make_long_tensor(prefill_slot_mapping_list,
600
- device=CUDA_DEVICE)
601
-
602
- # Packed key/value (query is already provided)
603
- packed_cross_kv = pack_qkv(cross_kv, device=CUDA_DEVICE)
604
-
605
- return (
606
- PhaseTestParameters( # Prefill-phase test params
607
- PackedQKVO(packed_cross_kv, prefill_packed_ideal_output),
608
- KVMemoryMap(prefill_block_tables, prefill_slot_mapping)),
609
- PhaseTestParameters( # Decode-phase test params
610
- PackedQKVO(None, decode_packed_ideal_output),
611
- KVMemoryMap(decode_block_tables, decode_slot_mapping)))
612
-
613
-
614
-def _run_encoder_attention_test(
615
- attn: Attention,
616
- encoder_test_params: PhaseTestParameters,
617
- attn_metadata: AttentionMetadata,
618
- test_pt: TestPoint,
619
- vllm_config: VllmConfig,
620
-) -> torch.Tensor:
621
- '''
622
- Run encoder attention.
623
-
624
- attn.forward() is passed attn_type=AttentionType.ENCODER in order
625
- to configure the kernel invocation for encoder attention
626
-
627
- Requires attn_metadata.num_decode_tokens == 0
628
- (There is no encoder execution in the decode-phase)
629
-
630
- Arguments:
631
-
632
- * attn: Attention wrapper instance
633
- * encoder_test_params: encoder PhaseTestParameters data structure;
634
- this function relies on the packed
635
- (number_of_tokens x num_heads x head_size)
636
- query/key/value fields
637
- * attn_metadata: attention metadata for encoder/decoder-self attention
638
- * test_pt: The TestPoint object containing test details like number of
639
- model heads, head size, name of the backend being used etc.
640
-
641
- Returns:
642
- * Attention.forward() applied to packed {query,key,value} and
643
- & attn_metadata
644
- '''
645
- assert attn_metadata.num_decode_tokens == 0
646
- packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
647
- assert packed_qkv is not None
648
- with set_forward_context(attn_metadata, vllm_config):
649
- # In the test setup the shape of the query is
650
- # [batch_size, seq_len, num_heads, head_size]. However
651
- # the attention backend expect the shape to be
652
- # [num_tokens, hidden_size]. Hence reshape the query before
653
- # invoking the forward method.
654
- # TODO - Update the way we construct the query so that it
655
- # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
656
- reshaped_query = packed_qkv.query.view(
657
- -1, test_pt.num_heads * test_pt.head_size)
658
- return attn.forward(reshaped_query, packed_qkv.key, packed_qkv.value)
659
-
660
-
661
-def _run_decoder_self_attention_test(
662
- test_rsrcs: TestResources,
663
- decoder_test_params: PhaseTestParameters,
664
- attn_metadata: AttentionMetadata,
665
- test_pt: TestPoint,
666
- vllm_config: VllmConfig,
667
-) -> torch.Tensor:
668
- '''
669
- Run decoder self-attention test.
670
-
671
- attn.forward() is passed attn_type=AttentionType.DECODER
672
- in order to configure the kernel invocation for decoder self-attention.
673
-
674
- Arguments:
675
-
676
- * test_rsrcs: TestResources instance; this function relies on the kv_cache
677
- and attn (Attention wrapper instance) fields
678
- * decoder_test_params: decoder PhaseTestParameters data structure;
679
- this function relies on the packed
680
- (number_of_tokens x num_heads x head_size)
681
- query/key/value fields
682
- * attn_metadata: attention metadata for decoder-self attention
683
- (contains KV cache memory-mapping)
684
- * test_pt: The TestPoint object containing test details like number of
685
- model heads, head size, name of the backend being used etc.
686
-
687
- Returns:
688
- * Attention.forward() applied to packed_{query,key,value}, kv_cache
689
- & attn_metadata
690
- '''
691
- attn = test_rsrcs.attn
692
- packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
693
- assert packed_qkv is not None
694
- with set_forward_context(attn_metadata, vllm_config):
695
- # In the test setup the shape of the query is
696
- # [batch_size, seq_len, num_heads, head_size]. However
697
- # the attention backend expect the shape to be
698
- # [num_tokens, hidden_size]. Hence reshape the query before
699
- # invoking the forward method.
700
- # TODO - Update the way we construct the query so that it
701
- # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
702
- reshaped_query = packed_qkv.query.view(
703
- -1, test_pt.num_heads * test_pt.head_size)
704
- return attn.forward(reshaped_query, packed_qkv.key, packed_qkv.value)
705
-
706
-
707
-def _run_encoder_decoder_cross_attention_test(
708
- test_rsrcs: TestResources,
709
- decoder_test_params: PhaseTestParameters,
710
- cross_test_params: Optional[PhaseTestParameters],
711
- attn_metadata: AttentionMetadata,
712
- test_pt: TestPoint,
713
- vllm_config: VllmConfig,
714
-) -> torch.Tensor:
715
- '''
716
- Run encoder/decoder cross-attention test.
717
-
718
- Via PhaseTestParameters data structures, consumes the same query utilized
719
- for decoder self-attention, plus a key/value specific to cross-attention.
720
-
721
- if cross_test_params is None or cross_test_params.packed_qkvo.packed_qkv
722
- is None, this reflects that in decode-phase cross attention there
723
- is no growth in the key and value tensors.
724
-
725
- attn.forward() is passed attn_type=AttentionType.ENCODER_DECODER
726
- in order to configure the kernel invocation for encoder/decoder cross-
727
- attention.
728
-
729
- Arguments:
730
-
731
- * test_rsrcs: TestResources instance; this function relies on the kv_cache
732
- and attn (Attention wrapper instance) fields
733
- * decoder_test_params: decoder PhaseTestParameters data structure;
734
- this function relies on the packed
735
- (number_of_tokens x num_heads x head_size)
736
- query field
737
- * cross_test_params: encoder/decoder PhaseTestParameters data structure;
738
- this function relies on the packed
739
- (number_of_tokens x num_heads x head_size)
740
- key/value fields
741
- * attn_metadata: attention metadata for encoder/decoder-self attention
742
- * test_pt: The TestPoint object containing test details like number of
743
- model heads, head size, name of the backend being used etc.
744
-
745
- Returns:
746
- * Attention.forward() applied to packed_{query,key,value}, kv_cache
747
- & attn_metadata
748
- '''
749
- assert decoder_test_params.packed_qkvo.packed_qkv is not None
750
-
751
- attn = test_rsrcs.attn
752
- if cross_test_params is None:
753
- key = None
754
- value = None
755
- else:
756
- cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
757
- key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
758
- value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
759
- with set_forward_context(attn_metadata, vllm_config):
760
- # In the test setup the shape of the query is
761
- # [batch_size, seq_len, num_heads, head_size]. However
762
- # the attention backend expect the shape to be
763
- # [num_tokens, hidden_size]. Hence reshape the query before
764
- # invoking the forward method.
765
- # TODO - Update the way we construct the query so that it
766
- # is shaped as [num_tokens, hidden_size] and we can skip the reshape.
767
- reshaped_query = decoder_test_params.packed_qkvo.packed_qkv.query.view(
768
- -1, test_pt.num_heads * test_pt.head_size)
769
- return attn.forward(reshaped_query, key, value)
770
-
771
-
772
-@pytest.fixture(autouse=True)
773
-def set_reset_environment(attn_backend):
774
- # Set the default torch datatype to bfloat16 to enable
775
- # testing of the Flash Attention backend. Also clear the
776
- # cached value of the backend.
777
- default_dtype = torch.get_default_dtype()
778
- if attn_backend.name == 'FLASH_ATTN':
779
- torch.set_default_dtype(torch.bfloat16)
780
- _cached_get_attn_backend.cache_clear()
781
- yield
782
- # Reset the torch datatype to what it was before the test
783
- # so as not to impact the remaining tests.
784
- torch.set_default_dtype(default_dtype)
785
-
786
-
787
-@pytest.mark.skipif(current_platform.is_rocm(),
788
- reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
789
-@pytest.mark.parametrize("num_heads", NUM_HEADS)
790
-@pytest.mark.parametrize("head_size", HEAD_SIZES)
791
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
792
-@pytest.mark.parametrize("batch_size", BATCH_SIZES)
793
-@pytest.mark.parametrize("block_size", BLOCK_SIZES)
794
-@pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
795
-@pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
796
-def test_encoder_only(
797
- num_heads: int,
798
- head_size: int,
799
- attn_backend: _Backend,
800
- batch_size: int,
801
- block_size: int,
802
- max_dec_seq_len: int,
803
- max_enc_seq_len: int,
804
-):
805
- '''
806
- End-to-end encoder-only attention test:
807
-
808
- * Construct fake test vectors for (1) encoder attention
809
- * Construct (1) attention metadata structure with prefill-phase
810
- encoder attention, and (2) an analogous attention metadata
811
- structure but for decode-phase
812
- * Test & validate encoder attention against ideal output
813
-
814
- No KV cache is required for encoder-only attention.
815
-
816
- Note on ROCm/HIP: currently encoder/decoder models are not supported on
817
- AMD GPUs, therefore this test simply is skipped if
818
- current_platform.is_rocm().
819
-
820
- This test globally forces an override of the usual backend
821
- auto-selection process, forcing the specific backend-under-test
822
- to be utilized.
823
-
824
- Arguments:
825
-
826
- * num_heads
827
- * head_size,
828
- * attn_backend: The attention backend to employ for testing
829
- * batch_size
830
- * block_size: KV cache block size
831
- * max_dec_seq_len: max length of decoder input sequences
832
- * max_enc_seq_len: max length of encoder input sequences
833
- '''
834
- # Force Attention wrapper backend
835
- with global_force_attn_backend_context_manager(attn_backend):
836
- # Note: KV cache size of 4096 is arbitrary & chosen intentionally
837
- # to be more than necessary, since exceeding the kv cache size
838
- # is not part of this test
839
- test_pt = TestPoint(num_heads, head_size, attn_backend.name,
840
- batch_size, block_size, max_dec_seq_len,
841
- max_enc_seq_len, 4096, AttentionType.ENCODER)
842
-
843
- # Attention scale factor, attention backend instance, attention wrapper
844
- # instance, KV cache init
845
- vllm_config = VllmConfig()
846
- with set_current_vllm_config(vllm_config):
847
- test_rsrcs = _make_test_resources(test_pt)
848
-
849
- # Construct encoder attention test params (only used
850
- # during prefill)
851
-
852
- enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
853
-
854
- # Shared prefill metadata structure
855
-
856
- prephase_attn_metadata: AttentionMetadata = make_test_metadata(
857
- attn_backend,
858
- True,
859
- None,
860
- decoder_test_params=None,
861
- encoder_test_params=enc_test_params,
862
- cross_test_params=None,
863
- device=CUDA_DEVICE)
864
-
865
- # PREFILL: encoder attention
866
-
867
- enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
868
- test_rsrcs.attn,
869
- enc_test_params,
870
- prephase_attn_metadata,
871
- test_pt=test_pt,
872
- vllm_config=vllm_config))
873
-
874
- # - Is encoder attention result correct?
875
- assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
876
- attn_backend.name)
877
-
878
-
879
-@pytest.mark.skipif(current_platform.is_rocm(),
880
- reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
881
-@pytest.mark.parametrize("num_heads", NUM_HEADS)
882
-@pytest.mark.parametrize("head_size", HEAD_SIZES)
883
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
884
-@pytest.mark.parametrize("batch_size", BATCH_SIZES)
885
-@pytest.mark.parametrize("block_size", BLOCK_SIZES)
886
-@pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
887
-@pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
888
-def test_e2e_enc_dec_attn(
889
- num_heads: int,
890
- head_size: int,
891
- attn_backend: _Backend,
892
- batch_size: int,
893
- block_size: int,
894
- max_dec_seq_len: int,
895
- max_enc_seq_len: int,
896
-) -> None:
897
- '''
898
- End-to-end encoder/decoder test:
899
-
900
- * Construct fake test vectors for (1) encoder attention,
901
- (2) decoder self-attention, and (3) encoder/decoder cross-attention
902
- * Construct (1) attention metadata structure with self- and cross-attention
903
- attributes for prefill-phase, and (2) an analogous attention metadata
904
- structure but for decode-phase
905
- * Test attention steps in the following order
906
-
907
- * Encoder attention
908
- * Prefill self-attention
909
- * Prefill cross-attention
910
- * Decode self-attention
911
- * Decode cross-attention
912
- * Besides being reflective of realistic use-cases, this order would
913
- exacerbate any accidental overlap in the self-/cross-attention
914
- block tables, which one hopes to avoid
915
-
916
-
917
- * Validate output correctness against ideal reference attention
918
- implementation
919
-
920
- Block tables are constructed such that cross-attention KV cache is in a
921
- higher, non-intersecting address-space than self-attention KV cache.
922
-
923
- Self- and cross-attention share the same query tensor but not the K/V
924
- tensors. Self-attention K/Vs must have the same seq len as Q while
925
- cross-attention K/Vs are allowed to differ in seq len, as is often the case
926
- for cross-attention.
927
-
928
- This test globally forces an override of the usual backend
929
- auto-selection process, forcing the specific backend-under-test
930
- to be utilized.
931
-
932
- Note on ROCm/HIP: currently encoder/decoder models are not supported on
933
- AMD GPUs, therefore this test simply is skipped if
934
- current_platform.is_rocm().
935
-
936
- Note on metadata: there is a single attention metadata structure shared by
937
- all prefill-phase attention operations (encoder, decoder, enc/dec cross),
938
- and a single one shared by all decode-phase attention operations
939
- (decoder & enc/dec cross.) This is intended to reflect the behavior
940
- of EncoderDecoderModelRunner, which constructs a single attention metadata
941
- structure for each prefill or decode run. A realistic scenario would rely
942
- on the attention backend to utilize the appropriate attention metadata
943
- fields according to the value of attn_metadata.attention_type. Thus,
944
- this test is organized so as to confirm that the backend-under-test can
945
- handle a shared prefill attention metadata structure & a shared decode\
946
- attention metadata structure.
947
-
948
- Arguments:
949
-
950
- * num_heads
951
- * head_size,
952
- * attn_backend: The attention backend to employ for testing
953
- * batch_size
954
- * block_size: KV cache block size
955
- * max_dec_seq_len: max length of decoder input sequences
956
- * max_enc_seq_len: max length of encoder input sequences
957
- '''
958
- # Force Attention wrapper backend
959
- with global_force_attn_backend_context_manager(attn_backend):
960
- # Note: KV cache size of 4096 is arbitrary & chosen intentionally
961
- # to be more than necessary, since exceeding the kv cache size
962
- # is not part of this test
963
- enc_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
964
- batch_size, block_size, max_dec_seq_len,
965
- max_enc_seq_len, 4096, AttentionType.ENCODER)
966
- enc_dec_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
967
- batch_size, block_size, max_dec_seq_len,
968
- max_enc_seq_len, 4096,
969
- AttentionType.ENCODER_DECODER)
970
- dec_test_pt = TestPoint(num_heads, head_size, attn_backend.name,
971
- batch_size, block_size, max_dec_seq_len,
972
- max_enc_seq_len, 4096, AttentionType.DECODER)
973
-
974
- # Attention scale factor, attention backend instance, attention wrapper
975
- # instance, KV cache init
976
- vllm_config = VllmConfig()
977
- with set_current_vllm_config(vllm_config):
978
- enc_test_rsrcs = _make_test_resources(enc_test_pt)
979
- enc_dec_test_rsrcs = _make_test_resources(enc_dec_test_pt)
980
- dec_test_rsrcs = _make_test_resources(dec_test_pt)
981
-
982
- # Construct encoder attention test params (only used
983
- # during prefill)
984
-
985
- enc_test_params = _encoder_attn_setup(enc_test_pt, enc_test_rsrcs)
986
-
987
- # Construct Decoder self-attention prefill-phase & decode-phase
988
- # test params, including query/key/value tensors, decoder self-attention
989
- # memory-mapping. cross_block_base_addr is the uppermost address in the
990
- # decoder self-attention block-table, i.e. a base address which the
991
- # encoder/decoder cross-attention block-table may build downward toward.
992
-
993
- (
994
- dec_qkv,
995
- prephase_dec_test_params,
996
- decphase_dec_test_params,
997
- cross_block_base_addr,
998
- ) = _decoder_attn_setup(dec_test_pt, dec_test_rsrcs)
999
-
1000
- # Construct encoder/decoder cross-attention prefill-phase
1001
- # & decode-phase test params, including key/value tensors,
1002
- # cross-attention memory-mapping
1003
-
1004
- (
1005
- prephase_cross_test_params,
1006
- decphase_cross_test_params,
1007
- ) = _enc_dec_cross_attn_setup_reuses_query(
1008
- dec_qkv,
1009
- enc_test_params,
1010
- prephase_dec_test_params,
1011
- enc_dec_test_pt,
1012
- enc_dec_test_rsrcs,
1013
- block_base_addr=cross_block_base_addr)
1014
-
1015
- # Shared prefill metadata structure
1016
- assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
1017
- prephase_attn_metadata: AttentionMetadata = make_test_metadata(
1018
- attn_backend,
1019
- True,
1020
- prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
1021
- decoder_test_params=prephase_dec_test_params,
1022
- encoder_test_params=enc_test_params,
1023
- cross_test_params=prephase_cross_test_params,
1024
- device=CUDA_DEVICE)
1025
-
1026
- # PREFILL: encoder attention
1027
-
1028
- enc_pckd_act_out = _run_encoder_attention_test(enc_test_rsrcs.attn,
1029
- enc_test_params,
1030
- prephase_attn_metadata,
1031
- test_pt=enc_test_pt,
1032
- vllm_config=vllm_config)
1033
-
1034
- # - Is encoder attention result correct?
1035
- assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
1036
- attn_backend.name)
1037
-
1038
- # PREFILL: decoder self-attention test
1039
-
1040
- prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
1041
- dec_test_rsrcs,
1042
- prephase_dec_test_params,
1043
- prephase_attn_metadata,
1044
- test_pt=dec_test_pt,
1045
- vllm_config=vllm_config)
1046
-
1047
- # - Is prefill decoder self-attention correct?
1048
- assert_actual_matches_ideal(prephase_dec_test_params,
1049
- prephase_dec_pckd_act_out,
1050
- attn_backend.name)
1051
-
1052
- # PREFILL: encoder/decoder cross-attention test
1053
-
1054
- prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1055
- enc_dec_test_rsrcs,
1056
- prephase_dec_test_params,
1057
- prephase_cross_test_params,
1058
- prephase_attn_metadata,
1059
- test_pt=enc_dec_test_pt,
1060
- vllm_config=vllm_config)
1061
-
1062
- # - Is prefill encoder/decoder cross-attention correct?
1063
- assert_actual_matches_ideal(prephase_cross_test_params,
1064
- prephase_cross_pckd_act_out,
1065
- attn_backend.name)
1066
-
1067
- # DECODE: build decode-phase attention metadata
1068
-
1069
- decphase_attn_metadata: AttentionMetadata = make_test_metadata(
1070
- attn_backend,
1071
- False,
1072
- dec_qkv.q_seq_lens,
1073
- decoder_test_params=decphase_dec_test_params,
1074
- encoder_test_params=enc_test_params,
1075
- cross_test_params=decphase_cross_test_params,
1076
- device=CUDA_DEVICE)
1077
-
1078
- # DECODE: decoder self-attention test
1079
-
1080
- decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
1081
- dec_test_rsrcs,
1082
- decphase_dec_test_params,
1083
- decphase_attn_metadata,
1084
- test_pt=dec_test_pt,
1085
- vllm_config=vllm_config)
1086
-
1087
- # - Is decode-phase decoder self-attention correct?
1088
- assert_actual_matches_ideal(decphase_dec_test_params,
1089
- decphase_dec_pckd_act_out,
1090
- attn_backend.name)
1091
-
1092
- # DECODE: encoder/decoder cross-attention test
1093
-
1094
- decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
1095
- enc_dec_test_rsrcs,
1096
- decphase_dec_test_params,
1097
- None,
1098
- decphase_attn_metadata,
1099
- test_pt=enc_dec_test_pt,
1100
- vllm_config=vllm_config)
1101
-
1102
- # - Is decode-phase encoder/decoder cross-attention correct?
1103
- assert_actual_matches_ideal(decphase_cross_test_params,
1104
- decphase_cross_pckd_act_out,
1105
- attn_backend.name)
tests/models/language/generation/test_bart.py
deleted
-222
@@ -1,222 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from typing import Optional
4
-
5
-import pytest
6
-from transformers import AutoModelForSeq2SeqLM
7
-
8
-from vllm.sequence import SampleLogprobs
9
-
10
-from ....conftest import (DecoderPromptType, ExplicitEncoderDecoderPrompt,
11
- HfRunner, VllmRunner)
12
-from ....utils import multi_gpu_test
13
-from ...utils import check_logprobs_close
14
-
15
-
16
-def vllm_to_hf_output(
17
- vllm_output: tuple[list[int], str, Optional[SampleLogprobs]],
18
- decoder_prompt_type: DecoderPromptType,
19
-):
20
- """Sanitize vllm output to be comparable with hf output."""
21
- output_ids, output_str, out_logprobs = vllm_output
22
-
23
- hf_output_str = output_str + "</s>"
24
- if decoder_prompt_type == DecoderPromptType.NONE:
25
- hf_output_str = "<s>" + hf_output_str
26
-
27
- return output_ids, hf_output_str, out_logprobs
28
-
29
-
30
-def run_test(
31
- hf_runner: type[HfRunner],
32
- vllm_runner: type[VllmRunner],
33
- prompts: list[ExplicitEncoderDecoderPrompt[str, str]],
34
- decoder_prompt_type: DecoderPromptType,
35
- model: str,
36
- *,
37
- dtype: str,
38
- max_tokens: int,
39
- num_logprobs: int,
40
- tensor_parallel_size: int,
41
- distributed_executor_backend: Optional[str] = None,
42
-) -> None:
43
- '''
44
- Test the vLLM BART model for a variety of encoder/decoder input prompts,
45
- by validating it against HuggingFace (HF) BART.
46
-
47
- Arguments:
48
-
49
- * hf_runner: HuggingFace (HF) test model runner
50
- * vllm_runner: vLLM test model runner
51
- * example_encoder_decoder_prompts: test fixture which provides a
52
- dictionary of dummy prompts
53
- * model: the HF ID of the specific BART variant under test
54
- * dtype: the tensor datatype to employ
55
- * max_tokens
56
- * num_logprobs
57
- * decoder_prompt_type: key into the example_encoder_decoder_prompts
58
- dictionary; selects specific encoder/decoder
59
- prompt scenarios to test
60
-
61
- A note on using HF BART as a baseline for validating vLLM BART,
62
- specifically when the decoder prompt is None.
63
-
64
- The HF GenerationMixin's default behavior is to force the first
65
- decoded token to be <BOS> if the prompt does not already contain
66
- <BOS> (this is accomplished using a logit
67
- processor setting.)
68
-
69
- So when we use HF BART as our baseline for comparison, note that
70
- when the user provides a request with a None decoder prompt
71
- (i.e. a singleton encoder prompt, or else an explicit encoder/
72
- decoder prompt with the decoder sub-prompt set to None), HF and
73
- vLLM handle this in different ways:
74
-
75
- * HF will (1) tokenize the None prompt as an empty token-list,
76
- (2) append <decoder-start-token> to the beginning, yielding
77
- [<decoder-start-token>], (3) pass this token list to the model, and
78
- then (4) after computing logits during prefill, override the model
79
- logits & force <BOS> to be the first generated token.
80
-
81
- * vLLM will (1) tokenize the None prompt as [<BOS>], (2) append decoder-
82
- start-token to the beginning, yielding [<decoder-start-token><BOS>],
83
- (3) pass these tokens to the model & proceed with generation.
84
-
85
- The net effect is that compared to vLLM, the list of HF *decoded* tokens
86
- will contain one more initial <BOS> than the vLLM generated tokens,
87
- because vLLM's <BOS> token is injected into the prompt rather than into
88
- the generated output. This is in spite of the fact that overall, the
89
- complete sequences (prompt + decoded tokens) produced by vLLM will match
90
- HF.
91
-
92
- So when we use HF decoded token output to validate vLLM's decoded token
93
- output, the testing process must account for the difference in decoded
94
- token sequences between vLLM and HF specifically in the
95
- decoder-prompt-is-None case.
96
-
97
- One option is to disable the logit processor feature that forces the
98
- <BOS> token to be decoded (forced_bos_token_id = None), eliminating
99
- the problem entirely. However this is not "normal" BART usage.
100
-
101
- The other option is - only in the decoder-prompt-is-None case - to
102
- discard the first decoded token from the HF output before comparing it
103
- to vLLM.
104
-
105
- To that end, when testing the scenario where the decoder prompt is None
106
- (and only in that one scenario), this test skips the first HF decoded
107
- token during the process of validating the vLLM decoded output.
108
- '''
109
-
110
- # NOTE: take care of the order. run vLLM first, and then run HF.
111
- # vLLM needs a fresh new process without cuda initialization.
112
- # if we run HF first, the cuda initialization will be done and it
113
- # will hurt multiprocessing backend with fork method (the default).
114
-
115
- # Note: currently encoder/decoder models are only compatible with
116
- # enforce_eager=True. Normally this is not a problem because
117
- # for encoder/decoder models vLLM will
118
- # default to enforce_eager=True if enforce_eager
119
- # is left unspecified. However, the
120
- # VllmRunner test fixture (which wraps around the LLM class) defaults to
121
- # enforce_eager=False (a behavior which a number of already-existing
122
- # decoder-only unit tests expect), so when testing an encoder/decoder
123
- # model we must explicitly specify enforce_eager=True in the VllmRunner
124
- # constructor.
125
- with vllm_runner(model,
126
- dtype=dtype,
127
- tensor_parallel_size=tensor_parallel_size,
128
- distributed_executor_backend=distributed_executor_backend,
129
- enforce_eager=True) as vllm_model:
130
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
131
- prompts, max_tokens, num_logprobs)
132
-
133
- # Configuration settings for HF baseline
134
- hf_kwargs = {
135
- "top_k": None,
136
- "num_beams": 1,
137
- "repetition_penalty": 1.0,
138
- "top_p": 1.0,
139
- "length_penalty": 1.0,
140
- "early_stopping": False,
141
- "no_repeat_ngram_size": None,
142
- "min_length": 0
143
- }
144
-
145
- with hf_runner(model, dtype=dtype,
146
- auto_cls=AutoModelForSeq2SeqLM) as hf_model:
147
- hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
148
- prompts,
149
- max_tokens,
150
- num_logprobs,
151
- **hf_kwargs,
152
- ))
153
-
154
- hf_skip_tokens = (1
155
- if decoder_prompt_type == DecoderPromptType.NONE else 0)
156
-
157
- check_logprobs_close(
158
- outputs_0_lst=hf_outputs,
159
- outputs_1_lst=[
160
- vllm_to_hf_output(vllm_output, decoder_prompt_type)
161
- for vllm_output in vllm_outputs
162
- ],
163
- name_0="hf",
164
- name_1="vllm",
165
- num_outputs_0_skip_tokens=hf_skip_tokens,
166
- )
167
-
168
-
169
-@pytest.mark.parametrize(
170
- "model",
171
- [
172
- pytest.param("facebook/bart-base",
173
- marks=[pytest.mark.core_model, pytest.mark.cpu_model]),
174
- pytest.param("facebook/bart-large-cnn"),
175
- ],
176
-)
177
-@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
178
-@pytest.mark.parametrize("max_tokens", [64])
179
-@pytest.mark.parametrize("num_logprobs", [5])
180
-@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
181
-@pytest.mark.skip(reason="bart not supported in V1")
182
-def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts, model,
183
- dtype, max_tokens, num_logprobs, decoder_prompt_type) -> None:
184
-
185
- run_test(
186
- hf_runner,
187
- vllm_runner,
188
- example_encoder_decoder_prompts[decoder_prompt_type],
189
- decoder_prompt_type,
190
- model,
191
- dtype=dtype,
192
- max_tokens=max_tokens,
193
- num_logprobs=num_logprobs,
194
- tensor_parallel_size=1,
195
- )
196
-
197
-
198
-@multi_gpu_test(num_gpus=2)
199
-@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
200
-@pytest.mark.parametrize("model", ["facebook/bart-large-cnn"])
201
-@pytest.mark.parametrize("dtype", ["float"])
202
-@pytest.mark.parametrize("max_tokens", [64])
203
-@pytest.mark.parametrize("num_logprobs", [5])
204
-@pytest.mark.parametrize("decoder_prompt_type", [DecoderPromptType.CUSTOM])
205
-@pytest.mark.skip(reason="bart not supported in V1")
206
-def test_models_distributed(hf_runner, vllm_runner,
207
- example_encoder_decoder_prompts,
208
- distributed_executor_backend, model, dtype,
209
- max_tokens, num_logprobs,
210
- decoder_prompt_type) -> None:
211
- run_test(
212
- hf_runner,
213
- vllm_runner,
214
- example_encoder_decoder_prompts[decoder_prompt_type],
215
- decoder_prompt_type,
216
- model,
217
- dtype=dtype,
218
- max_tokens=max_tokens,
219
- num_logprobs=num_logprobs,
220
- tensor_parallel_size=2,
221
- distributed_executor_backend=distributed_executor_backend,
222
- )
tests/models/language/generation/test_mbart.py
deleted
-123
@@ -1,123 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from typing import Optional
4
-
5
-import pytest
6
-from transformers import AutoModelForSeq2SeqLM
7
-
8
-from vllm.sequence import SampleLogprobs
9
-
10
-from ....conftest import DecoderPromptType, HfRunner, VllmRunner
11
-from ...utils import check_logprobs_close
12
-
13
-
14
-def vllm_to_hf_output(
15
- vllm_output: tuple[list[int], str, Optional[SampleLogprobs]],
16
- decoder_prompt_type: DecoderPromptType,
17
-):
18
- """Sanitize vllm output to be comparable with hf output."""
19
- output_ids, output_str, out_logprobs = vllm_output
20
- hf_output_str = output_str + "</s>"
21
- return output_ids, hf_output_str, out_logprobs
22
-
23
-
24
-def run_test(
25
- hf_runner: type[HfRunner],
26
- vllm_runner: type[VllmRunner],
27
- prompts: list[dict[str, str]],
28
- decoder_prompt_type: DecoderPromptType,
29
- model: str,
30
- *,
31
- dtype: str,
32
- max_tokens: int,
33
- num_logprobs: int,
34
- tensor_parallel_size: int,
35
- distributed_executor_backend: Optional[str] = None,
36
-) -> None:
37
- '''
38
- Test the vLLM mBART model by validating it against HuggingFace (HF).
39
- (Docstring content is omitted for brevity)
40
- '''
41
-
42
- vllm_prompts = prompts
43
- if decoder_prompt_type == DecoderPromptType.NONE:
44
- vllm_prompts = [{
45
- "encoder_prompt": p['encoder_prompt'],
46
- "decoder_prompt": ""
47
- } for p in prompts]
48
-
49
- vllm_kwargs = {
50
- "hf_overrides": {
51
- "architectures": ["MBartForConditionalGeneration"]
52
- }
53
- }
54
-
55
- with vllm_runner(model,
56
- dtype=dtype,
57
- tensor_parallel_size=tensor_parallel_size,
58
- distributed_executor_backend=distributed_executor_backend,
59
- enforce_eager=True,
60
- **vllm_kwargs) as vllm_model: # type: ignore
61
- vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
62
- vllm_prompts, max_tokens, num_logprobs)
63
-
64
- hf_kwargs = {
65
- "top_k": None,
66
- "num_beams": 1,
67
- "repetition_penalty": 1.0,
68
- "top_p": 1.0,
69
- "length_penalty": 1.0,
70
- "early_stopping": False,
71
- "no_repeat_ngram_size": None,
72
- "min_length": 0
73
- }
74
-
75
- with hf_runner(model, dtype=dtype,
76
- auto_cls=AutoModelForSeq2SeqLM) as hf_model:
77
- hf_kwargs["decoder_start_token_id"] = (
78
- hf_model.tokenizer.lang_code_to_id["ro_RO"])
79
-
80
- hf_outputs = (
81
- hf_model.generate_encoder_decoder_greedy_logprobs_limit(
82
- prompts, # HF runner still uses the original prompts
83
- max_tokens,
84
- num_logprobs,
85
- **hf_kwargs,
86
- ))
87
-
88
- hf_skip_tokens = 0
89
-
90
- check_logprobs_close(
91
- outputs_0_lst=hf_outputs,
92
- outputs_1_lst=[
93
- vllm_to_hf_output(vllm_output, decoder_prompt_type)
94
- for vllm_output in vllm_outputs
95
- ],
96
- name_0="hf",
97
- name_1="vllm",
98
- num_outputs_0_skip_tokens=hf_skip_tokens,
99
- )
100
-
101
-
102
-@pytest.mark.parametrize(
103
- "model",
104
- [pytest.param("facebook/mbart-large-en-ro")],
105
-)
106
-@pytest.mark.parametrize("dtype", ["float", "bfloat16"])
107
-@pytest.mark.parametrize("max_tokens", [64])
108
-@pytest.mark.parametrize("num_logprobs", [5])
109
-@pytest.mark.parametrize("decoder_prompt_type", list(DecoderPromptType))
110
-def test_models(hf_runner, vllm_runner, example_encoder_decoder_prompts, model,
111
- dtype, max_tokens, num_logprobs, decoder_prompt_type) -> None:
112
-
113
- run_test(
114
- hf_runner,
115
- vllm_runner,
116
- example_encoder_decoder_prompts[decoder_prompt_type],
117
- decoder_prompt_type,
118
- model,
119
- dtype=dtype,
120
- max_tokens=max_tokens,
121
- num_logprobs=num_logprobs,
122
- tensor_parallel_size=1,
123
- )
tests/models/multimodal/generation/test_florence2.py
deleted
-147
@@ -1,147 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Optional
5
-
6
-import pytest
7
-from PIL import Image
8
-
9
-from vllm.inputs.data import ExplicitEncoderDecoderPrompt, TextPrompt
10
-from vllm.multimodal.image import rescale_image_size
11
-from vllm.sequence import SampleLogprobs
12
-
13
-from ....conftest import IMAGE_ASSETS, HfRunner, ImageTestAssets, VllmRunner
14
-from ...utils import check_logprobs_close
15
-
16
-MODELS = ["microsoft/Florence-2-base"]
17
-# Florence-2 model repo's tokenizer config is missing some special tokens.
18
-# Therefore, we use a converted tokenizer from a forked repo
19
-TOKENIZER = "Isotr0py/Florence-2-tokenizer"
20
-HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21
- "stop_sign":
22
- "<OD>", # special task token which will output special tokens
23
- "cherry_blossom":
24
- "Describe in detail what is shown in the image.",
25
-})
26
-
27
-
28
-def get_hf_images_prompts(
29
- prompts_: list[ExplicitEncoderDecoderPrompt[str, TextPrompt]],
30
-) -> tuple[list[ExplicitEncoderDecoderPrompt[str, str]], list[Image.Image]]:
31
- prompts, images = [], []
32
- for prompt in prompts_:
33
- encoder_prompt = prompt["encoder_prompt"]
34
- prompts.append(
35
- ExplicitEncoderDecoderPrompt(
36
- encoder_prompt=encoder_prompt["prompt"],
37
- decoder_prompt=None,
38
- ))
39
- images.append(encoder_prompt["multi_modal_data"]["image"])
40
- return prompts, images
41
-
42
-
43
-def hf_to_vllm_output(hf_output: tuple[list[int], str,
44
- Optional[SampleLogprobs]]):
45
- """Sanitize hf output to be comparable with vllm output."""
46
- output_ids, output_str, out_logprobs = hf_output
47
-
48
- output_str = output_str.replace("</s>", "").replace("<s>", "")
49
-
50
- return output_ids, output_str, out_logprobs
51
-
52
-
53
-def run_test(
54
- hf_runner: type[HfRunner],
55
- vllm_runner: type[VllmRunner],
56
- inputs: list[list[ExplicitEncoderDecoderPrompt]],
57
- model: str,
58
- *,
59
- dtype: str,
60
- max_tokens: int,
61
- num_logprobs: int,
62
- tensor_parallel_size: int,
63
- distributed_executor_backend: Optional[str] = None,
64
-) -> None:
65
- with vllm_runner(model,
66
- max_num_seqs=8,
67
- tokenizer_name=TOKENIZER,
68
- dtype=dtype,
69
- tensor_parallel_size=tensor_parallel_size,
70
- distributed_executor_backend=distributed_executor_backend,
71
- enforce_eager=True) as vllm_model:
72
- vllm_outputs_per_case = [
73
- vllm_model.generate_encoder_decoder_greedy_logprobs(
74
- prompts,
75
- max_tokens,
76
- num_logprobs=num_logprobs,
77
- skip_special_tokens=False,
78
- ) for prompts in inputs
79
- ]
80
-
81
- hf_inputs = [get_hf_images_prompts(prompts) for prompts in inputs]
82
-
83
- with hf_runner(model, dtype=dtype, skip_tokenizer_init=True) as hf_model:
84
- hf_model.model.get_output_embeddings = lambda: \
85
- hf_model.model.language_model.lm_head
86
- hf_outputs_per_case = [
87
- hf_model.generate_encoder_decoder_greedy_logprobs_limit(
88
- prompts, max_tokens, num_logprobs=num_logprobs, images=images)
89
- for prompts, images in hf_inputs
90
- ]
91
-
92
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
93
- vllm_outputs_per_case):
94
- check_logprobs_close(
95
- outputs_0_lst=[hf_to_vllm_output(output) for output in hf_outputs],
96
- outputs_1_lst=vllm_outputs,
97
- name_0="hf",
98
- name_1="vllm",
99
- num_outputs_0_skip_tokens=1,
100
- )
101
-
102
-
103
-# FIXME: https://github.com/huggingface/transformers/issues/38358
104
-@pytest.mark.skip("Model initialization fails")
105
-@pytest.mark.core_model
106
-@pytest.mark.parametrize("model", MODELS)
107
-@pytest.mark.parametrize(
108
- "size_factors",
109
- [
110
- # No image
111
- [],
112
- # Single-scale
113
- [1.0],
114
- # Single-scale, batched
115
- [1.0, 1.0, 1.0],
116
- # Multi-scale
117
- [0.25, 0.5, 1.0],
118
- ],
119
-)
120
-@pytest.mark.parametrize("dtype", ["float"])
121
-@pytest.mark.parametrize("max_tokens", [64])
122
-@pytest.mark.parametrize("num_logprobs", [5])
123
-def test_models(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],
124
- image_assets: ImageTestAssets, model: str,
125
- size_factors: list[int], dtype: str, max_tokens: int,
126
- num_logprobs: int) -> None:
127
- images = [asset.pil_image for asset in image_assets]
128
-
129
- inputs_per_image = [[
130
- ExplicitEncoderDecoderPrompt(
131
- encoder_prompt=TextPrompt(
132
- prompt=prompt,
133
- multi_modal_data={"image": rescale_image_size(image, factor)}),
134
- decoder_prompt=None,
135
- ) for factor in size_factors
136
- ] for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
137
-
138
- run_test(
139
- hf_runner,
140
- vllm_runner,
141
- inputs_per_image,
142
- model,
143
- dtype=dtype,
144
- max_tokens=max_tokens,
145
- num_logprobs=num_logprobs,
146
- tensor_parallel_size=1,
147
- )
tests/models/multimodal/generation/test_mllama.py
deleted
-768
@@ -1,768 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Optional, overload
5
-
6
-import pytest
7
-import torch
8
-from packaging.version import Version
9
-from transformers import AutoConfig, AutoModelForImageTextToText, AutoTokenizer
10
-from transformers import __version__ as TRANSFORMERS_VERSION
11
-
12
-from vllm import LLM, SamplingParams
13
-from vllm.attention.backends.flash_attn import FlashAttentionMetadata
14
-from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
15
- global_force_attn_backend_context_manager)
16
-from vllm.model_executor.models.mllama import MllamaForConditionalGeneration
17
-from vllm.multimodal.image import rescale_image_size
18
-from vllm.sequence import SampleLogprobs
19
-
20
-from ....conftest import (IMAGE_ASSETS, HfRunner, ImageTestAssets,
21
- PromptImageInput, VllmRunner)
22
-from ....quantization.utils import is_quant_method_supported
23
-from ....utils import (create_new_process_for_each_test, large_gpu_test,
24
- multi_gpu_test)
25
-from ...utils import check_logprobs_close
26
-
27
-_LIMIT_IMAGE_PER_PROMPT = 3
28
-MLLAMA_IMAGE_TOKEN_ID = 128256
29
-
30
-LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
31
-
32
-HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
33
- "stop_sign":
34
- "<|image|><|begin_of_text|>The meaning of the image is",
35
- "cherry_blossom":
36
- "<|image|><|begin_of_text|>The city is",
37
-})
38
-
39
-text_only_prompts = [
40
- "The color of the sky is blue but sometimes it can also be",
41
-]
42
-
43
-models = [
44
- "meta-llama/Llama-3.2-11B-Vision-Instruct",
45
-]
46
-
47
-# Indices for inputs
48
-TEXT_ONLY = '0'
49
-IMAGE_AT_BEG = '1'
50
-IMAGE_AT_MIDDLE = '2'
51
-TWO_IMAGES = '3'
52
-
53
-# Input tokenized
54
-prompt_data = {
55
- # Tell me a story
56
- TEXT_ONLY: [41551, 757, 264, 3446],
57
- # <|image|> What's the content of this image
58
- IMAGE_AT_BEG:
59
- [MLLAMA_IMAGE_TOKEN_ID, 3639, 596, 279, 2262, 315, 420, 2217, 220],
60
- # Hello <|image|>What' the content of this image
61
- IMAGE_AT_MIDDLE:
62
- [9906, 220, MLLAMA_IMAGE_TOKEN_ID, 3923, 6, 279, 2262, 315, 420, 2217],
63
- #<|image|>Is there a duck in this image?<|image|>What's the animal in this image? # noqa: E501
64
- TWO_IMAGES: [
65
- MLLAMA_IMAGE_TOKEN_ID, 3957, 1070, 264, 37085, 304, 420, 2217, 30,
66
- MLLAMA_IMAGE_TOKEN_ID, 3923, 596, 279, 10065, 304, 420, 2217, 30
67
- ]
68
-}
69
-
70
-
71
-def vllm_to_hf_output(vllm_output: tuple[list[int], str,
72
- Optional[SampleLogprobs]],
73
- model: str):
74
- """Sanitize vllm output to be comparable with hf output."""
75
- output_ids, output_str, out_logprobs = vllm_output
76
-
77
- config = AutoConfig.from_pretrained(model)
78
- image_token_id = config.image_token_index
79
-
80
- tokenizer = AutoTokenizer.from_pretrained(model)
81
- eos_token_id = tokenizer.eos_token_id
82
-
83
- hf_output_ids = [
84
- token_id for idx, token_id in enumerate(output_ids)
85
- if token_id != image_token_id or output_ids[idx - 1] != image_token_id
86
- ]
87
-
88
- hf_output_str = output_str
89
- if hf_output_ids[-1] == eos_token_id:
90
- hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
91
-
92
- return hf_output_ids, hf_output_str, out_logprobs
93
-
94
-
95
-def _get_inputs(
96
- image_assets: ImageTestAssets,
97
- *,
98
- size_factors: Optional[list[float]] = None,
99
- sizes: Optional[list[tuple[int, int]]] = None,
100
-) -> list[tuple[list[str], PromptImageInput]]:
101
- images = [asset.pil_image for asset in image_assets]
102
-
103
- if size_factors is not None:
104
- inputs_per_image = [(
105
- [prompt for _ in size_factors],
106
- [rescale_image_size(image, factor) for factor in size_factors],
107
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
108
- elif sizes is not None:
109
- inputs_per_image = [(
110
- [
111
- prompt if size is not None else text_only_prompts[0]
112
- for size in sizes
113
- ],
114
- [
115
- image.resize(size) if size is not None else None
116
- for size in sizes
117
- ],
118
- ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
119
- if len(sizes) == 0:
120
- inputs_per_image.append(
121
- (text_only_prompts, [None] * len(text_only_prompts)))
122
- else:
123
- raise ValueError("You must provide either `size_factors` or `sizes`")
124
-
125
- return inputs_per_image
126
-
127
-
128
-@overload
129
-def run_test(
130
- hf_runner: type[HfRunner],
131
- vllm_runner: type[VllmRunner],
132
- image_assets: ImageTestAssets,
133
- model: str,
134
- *,
135
- size_factors: list[float],
136
- dtype: str,
137
- max_tokens: int,
138
- num_logprobs: int,
139
- tensor_parallel_size: int,
140
- distributed_executor_backend: Optional[str] = None,
141
-):
142
- ...
143
-
144
-
145
-@overload
146
-def run_test(
147
- hf_runner: type[HfRunner],
148
- vllm_runner: type[VllmRunner],
149
- image_assets: ImageTestAssets,
150
- model: str,
151
- *,
152
- sizes: list[tuple[int, int]],
153
- dtype: str,
154
- max_tokens: int,
155
- num_logprobs: int,
156
- tensor_parallel_size: int,
157
- distributed_executor_backend: Optional[str] = None,
158
-):
159
- ...
160
-
161
-
162
-def run_test(
163
- hf_runner: type[HfRunner],
164
- vllm_runner: type[VllmRunner],
165
- image_assets: ImageTestAssets,
166
- model: str,
167
- *,
168
- size_factors: Optional[list[float]] = None,
169
- sizes: Optional[list[tuple[int, int]]] = None,
170
- dtype: str,
171
- max_tokens: int,
172
- num_logprobs: int,
173
- tensor_parallel_size: int,
174
- distributed_executor_backend: Optional[str] = None,
175
-):
176
- _run_test(
177
- hf_runner,
178
- vllm_runner,
179
- _get_inputs(image_assets, size_factors=size_factors, sizes=sizes),
180
- model,
181
- dtype=dtype,
182
- max_tokens=max_tokens,
183
- num_logprobs=num_logprobs,
184
- tensor_parallel_size=tensor_parallel_size,
185
- distributed_executor_backend=distributed_executor_backend,
186
- )
187
-
188
-
189
-def _run_test(
190
- hf_runner: type[HfRunner],
191
- vllm_runner: type[VllmRunner],
192
- inputs: list[tuple[list[str], PromptImageInput]],
193
- model: str,
194
- *,
195
- dtype: str,
196
- max_tokens: int,
197
- num_logprobs: int,
198
- tensor_parallel_size: int,
199
- distributed_executor_backend: Optional[str] = None,
200
-):
201
- """Inference result should be the same between hf and vllm.
202
-
203
- All the image fixtures for the test are from IMAGE_ASSETS.
204
- For huggingface runner, we provide the PIL images as input.
205
- For vllm runner, we provide MultiModalDataDict objects
206
- and corresponding MultiModalConfig as input.
207
- Note, the text input is also adjusted to abide by vllm contract.
208
- The text output is sanitized to be able to compare with hf.
209
- """
210
- # NOTE: take care of the order. run vLLM first, and then run HF.
211
- # vLLM needs a fresh new process without cuda initialization.
212
- # if we run HF first, the cuda initialization will be done and it
213
- # will hurt multiprocessing backend with fork method (the default method).
214
-
215
- # max_model_len should be greater than image_feature_size
216
- with vllm_runner(
217
- model,
218
- dtype=dtype,
219
- max_model_len=19212, # 3 max size images
220
- max_num_seqs=3,
221
- tensor_parallel_size=tensor_parallel_size,
222
- distributed_executor_backend=distributed_executor_backend,
223
- limit_mm_per_prompt={"image":
224
- _LIMIT_IMAGE_PER_PROMPT}) as vllm_model:
225
- vllm_outputs_per_image = [
226
- vllm_model.generate_greedy_logprobs(prompts,
227
- max_tokens,
228
- num_logprobs=num_logprobs,
229
- images=images)
230
- for prompts, images in inputs
231
- ]
232
-
233
- with hf_runner(model,
234
- dtype=dtype,
235
- model_kwargs={"device_map": "auto"},
236
- auto_cls=AutoModelForImageTextToText) as hf_model:
237
- hf_outputs_per_image = [
238
- hf_model.generate_greedy_logprobs_limit(prompts,
239
- max_tokens,
240
- num_logprobs=num_logprobs,
241
- images=images)
242
- for prompts, images in inputs
243
- ]
244
-
245
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
246
- vllm_outputs_per_image):
247
- check_logprobs_close(
248
- outputs_0_lst=hf_outputs,
249
- outputs_1_lst=[
250
- vllm_to_hf_output(vllm_output, model)
251
- for vllm_output in vllm_outputs
252
- ],
253
- name_0="hf",
254
- name_1="vllm",
255
- )
256
-
257
-
258
-@pytest.fixture(autouse=True)
259
-def clear_cache():
260
- """Fixture to clear backend cache before each test."""
261
- _cached_get_attn_backend.cache_clear() # Clear the cache
262
- yield # This allows the test to run
263
-
264
-
265
-@large_gpu_test(min_gb=48)
266
-@pytest.mark.core_model
267
-@pytest.mark.parametrize("model", models)
268
-@pytest.mark.parametrize(
269
- "sizes",
270
- [
271
- # Text only
272
- [],
273
- # Single-size
274
- [(512, 512)],
275
- # Single-size, batched
276
- [(512, 512), (512, 512), (512, 512)],
277
- # Multi-size, batched
278
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
279
- (1024, 1024), (512, 1536), (512, 2028)],
280
- # Multi-size, batched, including text only
281
- [(512, 512), (1024, 512), (1536, 512), (2048, 512), (512, 1024),
282
- (1024, 1024), (512, 1536), (512, 2028), None],
283
- # mllama has 8 possible aspect ratios, carefully set the sizes
284
- # to cover all of them
285
- ])
286
-@pytest.mark.parametrize("dtype", ["bfloat16"])
287
-@pytest.mark.parametrize("max_tokens", [128])
288
-@pytest.mark.parametrize("num_logprobs", [5])
289
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
290
-@pytest.mark.skipif(
291
- Version(TRANSFORMERS_VERSION) <= Version("4.55.2"),
292
- reason="Transformers v4.55 has a regression issue on mllama, "
293
- "see: https://github.com/huggingface/transformers/pull/40083")
294
-def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
295
- model, sizes, dtype, max_tokens,
296
- num_logprobs,
297
- attn_backend: _Backend) -> None:
298
- with global_force_attn_backend_context_manager(attn_backend):
299
- if attn_backend == _Backend.FLASH_ATTN:
300
- # Flash Attention works only with bfloat16 data-type
301
- dtype = 'bfloat16'
302
- run_test(
303
- hf_runner,
304
- vllm_runner,
305
- image_assets,
306
- model,
307
- sizes=sizes,
308
- dtype=dtype,
309
- max_tokens=max_tokens,
310
- num_logprobs=num_logprobs,
311
- tensor_parallel_size=1,
312
- )
313
-
314
-
315
-@large_gpu_test(min_gb=48)
316
-@pytest.mark.core_model
317
-@pytest.mark.parametrize("model", models)
318
-@pytest.mark.parametrize("dtype", ["bfloat16"])
319
-@pytest.mark.parametrize("max_tokens", [128])
320
-@pytest.mark.parametrize("num_logprobs", [5])
321
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
322
-@pytest.mark.skipif(
323
- Version(TRANSFORMERS_VERSION) <= Version("4.55.2"),
324
- reason="Transformers v4.55 has a regression issue on mllama, "
325
- "see: https://github.com/huggingface/transformers/pull/40083")
326
-def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
327
- model, dtype, max_tokens, num_logprobs,
328
- attn_backend: _Backend) -> None:
329
-
330
- stop_sign = image_assets[0].pil_image
331
- cherry_blossom = image_assets[1].pil_image
332
-
333
- inputs = [(
334
- [
335
- "<|image|><|image|><|begin_of_text|>Describe 2 images.", # noqa: E501
336
- "<|image|><|image|><|begin_of_text|>Describe 2 images.", # noqa: E501
337
- "<|image|><|image|><|image|><|begin_of_text|>Describe 3 images.", # noqa: E501
338
- ],
339
- [
340
- [stop_sign, cherry_blossom],
341
- # Images with different sizes.
342
- [
343
- stop_sign.resize((512, 512)),
344
- stop_sign,
345
- ],
346
- [
347
- stop_sign,
348
- stop_sign.resize((512, 1536)),
349
- cherry_blossom.resize((512, 1024)),
350
- ],
351
- ])]
352
- with global_force_attn_backend_context_manager(attn_backend):
353
- if attn_backend == _Backend.FLASH_ATTN:
354
- # Flash Attention works only with bfloat16 data-type
355
- dtype = 'bfloat16'
356
- _run_test(
357
- hf_runner,
358
- vllm_runner,
359
- inputs,
360
- model,
361
- dtype=dtype,
362
- max_tokens=max_tokens,
363
- num_logprobs=num_logprobs,
364
- tensor_parallel_size=1,
365
- )
366
-
367
-
368
-@large_gpu_test(min_gb=48)
369
-@pytest.mark.core_model
370
-@pytest.mark.parametrize("model", models)
371
-@pytest.mark.parametrize("dtype", ["bfloat16"])
372
-@pytest.mark.parametrize("max_tokens", [128])
373
-@pytest.mark.parametrize("num_logprobs", [5])
374
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
375
-@pytest.mark.skipif(
376
- Version(TRANSFORMERS_VERSION) <= Version("4.55.2"),
377
- reason="Transformers v4.55 has a regression issue on mllama, "
378
- "see: https://github.com/huggingface/transformers/pull/40083")
379
-def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
380
- dtype, max_tokens, num_logprobs,
381
- attn_backend: _Backend) -> None:
382
-
383
- stop_sign = image_assets[0].pil_image
384
- cherry_blossom = image_assets[1].pil_image
385
-
386
- inputs = [(
387
- [
388
- "<|begin_of_text|>The content of the image <|image|> is", # noqa: E501
389
- "<|begin_of_text|>Between the first image <|image|> and the second image<|image|>, " # noqa: E501
390
- "which is a stop sign and which is a cherry blossom?", # noqa: E501
391
- ],
392
- [
393
- [stop_sign],
394
- [stop_sign, cherry_blossom],
395
- ])]
396
- with global_force_attn_backend_context_manager(attn_backend):
397
- if attn_backend == _Backend.FLASH_ATTN:
398
- # Flash Attention works only with bfloat16 data-type
399
- dtype = 'bfloat16'
400
- _run_test(
401
- hf_runner,
402
- vllm_runner,
403
- inputs,
404
- model,
405
- dtype=dtype,
406
- max_tokens=max_tokens,
407
- num_logprobs=num_logprobs,
408
- tensor_parallel_size=1,
409
- )
410
-
411
-
412
-@create_new_process_for_each_test()
413
-@multi_gpu_test(num_gpus=2)
414
-@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
415
-@pytest.mark.parametrize("model", models)
416
-@pytest.mark.parametrize("dtype", ["bfloat16"])
417
-@pytest.mark.parametrize("max_tokens", [64])
418
-@pytest.mark.parametrize("num_logprobs", [5])
419
-@pytest.mark.skipif(
420
- Version(TRANSFORMERS_VERSION) <= Version("4.55.2"),
421
- reason="Transformers v4.55 has a regression issue on mllama, "
422
- "see: https://github.com/huggingface/transformers/pull/40083")
423
-def test_models_distributed(
424
- hf_runner,
425
- vllm_runner,
426
- image_assets,
427
- distributed_executor_backend,
428
- model,
429
- dtype,
430
- max_tokens,
431
- num_logprobs,
432
-) -> None:
433
- run_test(
434
- hf_runner,
435
- vllm_runner,
436
- image_assets,
437
- model=model,
438
- size_factors=[0.25, 0.5, 1.0],
439
- dtype=dtype,
440
- max_tokens=max_tokens,
441
- num_logprobs=num_logprobs,
442
- tensor_parallel_size=2,
443
- distributed_executor_backend=distributed_executor_backend,
444
- )
445
-
446
-
447
-@large_gpu_test(min_gb=48)
448
-@pytest.mark.core_model
449
-@pytest.mark.parametrize("model", models)
450
-@pytest.mark.parametrize("dtype", ["float16"])
451
-@pytest.mark.parametrize("max_tokens", [32])
452
-@pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),
453
- reason='bitsandbytes is not supported on this GPU type.')
454
-def test_bnb_regression(
455
- image_assets: ImageTestAssets,
456
- model: str,
457
- dtype: str,
458
- max_tokens: int,
459
-):
460
- stop_sign = image_assets[0].pil_image
461
- prompts = [
462
- {
463
- "prompt": "<|begin_of_text|>The content of the image <|image|> is",
464
- "multi_modal_data": {
465
- "image": stop_sign
466
- },
467
- },
468
- {
469
- "prompt":
470
- "The color of the sky is blue but sometimes it can also be",
471
- },
472
- ]
473
- # Test regression about QKVCrossParallelLinear
474
- llm = LLM(
475
- model=model,
476
- dtype=dtype,
477
- max_model_len=8192,
478
- max_num_seqs=2,
479
- quantization="bitsandbytes",
480
- )
481
- sampling_params = SamplingParams(
482
- temperature=0,
483
- max_tokens=max_tokens,
484
- )
485
- outputs = llm.generate(prompts, sampling_params)
486
- assert outputs
487
-
488
-
489
-@large_gpu_test(min_gb=48)
490
-@pytest.mark.core_model
491
-@pytest.mark.parametrize("model", models)
492
-@pytest.mark.parametrize("dtype", ["bfloat16"])
493
-@pytest.mark.parametrize("max_tokens", [32])
494
-def test_explicit_implicit_prompt(
495
- image_assets: ImageTestAssets,
496
- model: str,
497
- dtype: str,
498
- max_tokens: int,
499
-):
500
- stop_sign = image_assets[0].pil_image
501
- # yapf: disable
502
- prompts = [
503
- # explicit prompt
504
- {
505
- "encoder_prompt": {
506
- "prompt": "<|image|>",
507
- "multi_modal_data": {"image": stop_sign},
508
- },
509
- "decoder_prompt": {
510
- "prompt_token_ids": [128000, 791, 2262, 315, 279, 2217, 220, 128256, 374], # noqa: E501
511
- }
512
- },
513
- {
514
- "encoder_prompt": "Not <|image|>",
515
- "decoder_prompt": "The color of the sky is blue but sometimes it can also be", # noqa: E501
516
- },
517
- # implicit prompt
518
- {
519
- "prompt": "<|begin_of_text|>The content of the image <|image|> is", # noqa: E501
520
- "multi_modal_data": {"image": stop_sign},
521
- },
522
- {
523
- "prompt": "The color of the sky is blue but sometimes it can also be", # noqa: E501
524
- },
525
- ]
526
- # yapf: enable
527
- llm = LLM(
528
- model=model,
529
- dtype=dtype,
530
- max_model_len=8192,
531
- max_num_seqs=2,
532
- tensor_parallel_size=1,
533
- )
534
- sampling_params = SamplingParams(
535
- temperature=0,
536
- max_tokens=max_tokens,
537
- )
538
- outputs = llm.generate(prompts, sampling_params)
539
- n_prompts = len(prompts)
540
- explicit_outputs = outputs[:n_prompts // 2]
541
- implicit_outputs = outputs[n_prompts // 2:]
542
- for exp_output, imp_output in zip(explicit_outputs, implicit_outputs):
543
- assert exp_output.outputs[0].text == imp_output.outputs[0].text
544
-
545
-
546
-@large_gpu_test(min_gb=48)
547
-@pytest.mark.core_model
548
-@pytest.mark.parametrize("model", models)
549
-@pytest.mark.parametrize("dtype", ["bfloat16"])
550
-@pytest.mark.parametrize("max_tokens", [128])
551
-@pytest.mark.parametrize("num_logprobs", [5])
552
-@pytest.mark.parametrize("attn_backend", LIST_ENC_DEC_SUPPORTED_BACKENDS)
553
-def test_regression(vllm_runner, image_assets, model, dtype, max_tokens,
554
- num_logprobs, attn_backend: _Backend) -> None:
555
-
556
- stop_sign = image_assets[0].pil_image
557
-
558
- with global_force_attn_backend_context_manager(attn_backend), vllm_runner(
559
- model,
560
- dtype=dtype,
561
- max_model_len=8192,
562
- max_num_seqs=4,
563
- tensor_parallel_size=1,
564
- limit_mm_per_prompt={"image":
565
- _LIMIT_IMAGE_PER_PROMPT}) as vllm_model:
566
-
567
- # Regression tests for https://github.com/vllm-project/vllm/issues/10648
568
-
569
- # Number of groups of image tokens is greater than the number of images
570
- # provided (the whitespace between the tags is necessary)
571
- prompt = "<|begin_of_text|><|image|> <|image|> Compare the two images" # noqa: E501
572
- image = stop_sign
573
- with pytest.raises(ValueError):
574
- vllm_model.generate_greedy_logprobs([prompt],
575
- max_tokens,
576
- num_logprobs,
577
- images=[image])
578
-
579
- # Batch of a text-only and image request that requires cross-attention
580
- prompts = [
581
- "What is the capital of spain?",
582
- "Text before the image...<|image|>What is in the image?", # noqa: E501
583
- ]
584
- images = [
585
- None,
586
- [stop_sign],
587
- ]
588
- vllm_model.generate_greedy_logprobs(prompts,
589
- max_tokens,
590
- num_logprobs,
591
- images=images)
592
-
593
- # Test the reverse order too for good measure
594
- prompts = [
595
- "<|begin_of_text|>Text before the image...<|image|>What is in the image?", # noqa: E501
596
- "<|begin_of_text|>Hello!",
597
- ]
598
- images = [
599
- [stop_sign],
600
- None,
601
- ]
602
- vllm_model.generate_greedy_logprobs(prompts,
603
- max_tokens,
604
- num_logprobs,
605
- images=images)
606
-
607
- # Mixed batch with text and images with different numbers of tiles
608
- prompts = [
609
- "<|begin_of_text|>Hello!",
610
- "<|begin_of_text|>Some text before.<|image|>What is in the image?", # noqa: E501
611
- "<|begin_of_text|>Some text before.<|image|>What is in the image?", # noqa: E501
612
- ]
613
- images = [
614
- None,
615
- [stop_sign],
616
- # smaller image must be 2nd for the repro
617
- [stop_sign.resize((448, 448))],
618
- ]
619
- vllm_model.generate_greedy_logprobs(prompts,
620
- max_tokens,
621
- num_logprobs,
622
- images=images)
623
-
624
-
625
-class DummyModel:
626
- image_token_id = MLLAMA_IMAGE_TOKEN_ID
627
-
628
-
629
-@pytest.mark.core_model
630
-@pytest.mark.parametrize(
631
- "input_indices_and_output",
632
- # inputs, (cross_attention_mask, kv_range_for_decode)
633
- [([TEXT_ONLY], (None, None)), ([IMAGE_AT_BEG], (None, None)),
634
- ([TEXT_ONLY, IMAGE_AT_BEG], (None, None)),
635
- ([IMAGE_AT_MIDDLE], ((10, 12), [[0, 6]])),
636
- ([TEXT_ONLY, IMAGE_AT_MIDDLE], ((14, 12), [[0, 6]])),
637
- ([TEXT_ONLY, IMAGE_AT_BEG, IMAGE_AT_MIDDLE],
638
- ((23, 24), [[0, 6], [6, 12]])),
639
- ([IMAGE_AT_MIDDLE, TEXT_ONLY], ((14, 12), [[0, 6]])),
640
- ([TWO_IMAGES], ((18, 12), [[6, 12]])),
641
- ([TEXT_ONLY, TWO_IMAGES], ((22, 12), [[6, 12]]))])
642
-def test_get_cross_attention_mask(input_indices_and_output) -> None:
643
-
644
- input_indices, expected_output = input_indices_and_output
645
-
646
- sequences = [torch.tensor(prompt_data[i]) for i in input_indices]
647
- num_tiles = [[2, 2] if i != TEXT_ONLY else [] for i in input_indices
648
- if i != TEXT_ONLY]
649
- input = torch.cat(sequences)
650
-
651
- seq_lens = [len(s) for s in sequences]
652
-
653
- attn_data = FlashAttentionMetadata(
654
- seq_lens=seq_lens,
655
- # Dummy values
656
- enable_kv_scales_calculation=False,
657
- num_prefills=0,
658
- num_prefill_tokens=0,
659
- num_decode_tokens=0,
660
- slot_mapping=0,
661
- multi_modal_placeholder_index_maps=None,
662
- seq_lens_tensor=0,
663
- max_prefill_seq_len=0,
664
- max_decode_seq_len=0,
665
- context_lens_tensor=None,
666
- block_tables=None,
667
- use_cuda_graph=False,
668
- )
669
-
670
- dummy = DummyModel()
671
-
672
- cross_attention_mask, kv_range_for_decode = MllamaForConditionalGeneration\
673
- .get_cross_attention_mask(dummy,
674
- input,
675
- attn_data,
676
- num_tiles=num_tiles,
677
- num_tokens_per_tile=3,
678
- dtype=torch.bfloat16)
679
-
680
- expected_cross_attention_mask, expected_kv_range_for_decode = \
681
- expected_output
682
-
683
- assert kv_range_for_decode == expected_kv_range_for_decode
684
- if expected_cross_attention_mask is not None:
685
- assert cross_attention_mask is not None
686
- assert cross_attention_mask.shape == expected_cross_attention_mask
687
- else:
688
- assert cross_attention_mask is None
689
-
690
-
691
-@pytest.mark.core_model
692
-@pytest.mark.parametrize(
693
- "input_indices",
694
- [[TEXT_ONLY], [IMAGE_AT_BEG], [TEXT_ONLY, IMAGE_AT_BEG], [IMAGE_AT_MIDDLE],
695
- [TEXT_ONLY, IMAGE_AT_MIDDLE], [TEXT_ONLY, IMAGE_AT_BEG, IMAGE_AT_MIDDLE],
696
- [IMAGE_AT_MIDDLE, TEXT_ONLY], [TWO_IMAGES], [TEXT_ONLY, TWO_IMAGES]])
697
-def test_get_full_text_row_masked_out_mask(input_indices) -> None:
698
-
699
- sequences = [torch.tensor(prompt_data[i]) for i in input_indices]
700
-
701
- seq_lens = [len(s) for s in sequences]
702
-
703
- num_prefill_tokens = sum(seq_lens)
704
-
705
- # TEXT_ONLY is zero, so it will be masked out,
706
- # other instances should not be.
707
- encoder_seq_lens = [int(i) for i in input_indices]
708
-
709
- attn_data = FlashAttentionMetadata(
710
- seq_lens=seq_lens,
711
- encoder_seq_lens=encoder_seq_lens,
712
- num_prefill_tokens=num_prefill_tokens,
713
- # Dummy values
714
- enable_kv_scales_calculation=False,
715
- num_prefills=0,
716
- num_decode_tokens=0,
717
- slot_mapping=0,
718
- multi_modal_placeholder_index_maps=None,
719
- seq_lens_tensor=0,
720
- max_prefill_seq_len=0,
721
- max_decode_seq_len=0,
722
- context_lens_tensor=None,
723
- block_tables=None,
724
- use_cuda_graph=False,
725
- )
726
-
727
- dummy = DummyModel()
728
-
729
- full_text_row_masked_out_mask = MllamaForConditionalGeneration\
730
- .get_full_text_row_masked_out_mask(dummy,
731
- attn_data,
732
- torch.get_default_device())
733
-
734
- full_text_row_masked_out_mask = full_text_row_masked_out_mask.squeeze()
735
- full_text_row_masked_out_mask = full_text_row_masked_out_mask.tolist()
736
-
737
- idx = 0
738
- assert len(full_text_row_masked_out_mask) == num_prefill_tokens
739
- for i, seq_len in enumerate(seq_lens):
740
- must_be_masked = input_indices[i] != TEXT_ONLY
741
- for _ in range(seq_len):
742
- assert full_text_row_masked_out_mask[idx] == must_be_masked, \
743
- f"full_text_row_masked_out_mask[{idx}] must be " \
744
- f"'{must_be_masked}' "
745
- idx += 1
746
-
747
-
748
-@pytest.mark.core_model
749
-@pytest.mark.parametrize("encoder_seq_lens, num_tiles, expected", [
750
- ([6404], [[4]], [6404]),
751
- ([0, 6404], [[4]], [6404]),
752
- ([0, 1601, 8005], [[1], [4, 1]], [1601, 8005]),
753
- ([0, 19212, 0, 3202], [[4, 4, 4], [2]], [19212, 3202]),
754
-])
755
-def test_parse_and_validate_encoder_lens(encoder_seq_lens, num_tiles,
756
- expected) -> None:
757
-
758
- dummy = DummyModel()
759
- num_tokens_per_tile = 1601
760
- actual_encoder_seq_lens = MllamaForConditionalGeneration \
761
- ._get_and_validate_encoder_lens(
762
- dummy,
763
- encoder_seq_lens,
764
- num_tiles,
765
- num_tokens_per_tile,
766
- )
767
- assert actual_encoder_seq_lens == expected, \
768
- f"Expected {expected} but got {actual_encoder_seq_lens}"
tests/models/multimodal/processing/test_common.py
-5
@@ -167,8 +167,6 @@ def _test_processing_correctness(
167
# incorrect token ids. So we need use `add_special_tokens=False` here
168
# to leave bos_token to be added by the processor.
169
_ADD_SPECIAL_TOKENS_OVERRIDES = {
170
- "donut": False,
171
- "mllama": False,
170
"ovis": False,
171
"ovis2_5": False,
172
"paligemma": False,
@@ -278,9 +276,7 @@ def _test_processing_correctness_one(
276
"facebook/chameleon-7b",
277
"CohereLabs/command-a-vision-07-2025",
278
"deepseek-ai/deepseek-vl2-tiny",
281
- "naver-clova-ix/donut-base-finetuned-docvqa",
279
"baidu/ERNIE-4.5-VL-28B-A3B-PT",
283
- "microsoft/Florence-2-base",
280
"adept/fuyu-8b",
281
"google/gemma-3-4b-it",
282
"google/gemma-3n-E2B-it",
@@ -305,7 +301,6 @@ def _test_processing_correctness_one(
301
"llava-hf/llava-v1.6-mistral-7b-hf",
302
"llava-hf/LLaVA-NeXT-Video-7B-hf",
303
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
308
- "meta-llama/Llama-3.2-11B-Vision-Instruct",
304
"TIGER-Lab/Mantis-8B-siglip-llama3",
305
"mispeech/midashenglm-7b",
306
"openbmb/MiniCPM-Llama3-V-2_5",
tests/models/multimodal/processing/test_mllama.py
deleted
-72
@@ -1,72 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Tests for mllama's multimodal preprocessing and profiling."""
4
-import pytest
5
-from transformers import MllamaConfig
6
-
7
-from vllm.multimodal import MULTIMODAL_REGISTRY
8
-from vllm.multimodal.profiling import MultiModalProfiler
9
-
10
-from ...utils import build_model_context
11
-
12
-
13
-@pytest.mark.parametrize("model_id",
14
- ["meta-llama/Llama-3.2-11B-Vision-Instruct"])
15
-@pytest.mark.parametrize("max_model_len", [4096, 8192, 25600, 131072])
16
-@pytest.mark.parametrize("max_num_seqs", [1, 2, 8])
17
-def test_profiling(
18
- model_id: str,
19
- max_model_len: int,
20
- max_num_seqs: int,
21
-):
22
- # regression test for https://github.com/vllm-project/vllm/issues/13929
23
- from vllm.model_executor.models.mllama import calc_token_per_chunk
24
-
25
- model_config_kwargs = {
26
- "max_model_len": max_model_len,
27
- }
28
- ctx = build_model_context(
29
- model_id,
30
- model_config_kwargs=model_config_kwargs,
31
- limit_mm_per_prompt={"image": 1},
32
- )
33
-
34
- mm_config = ctx.get_mm_config()
35
- processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
36
- profiler = MultiModalProfiler(processor)
37
-
38
- dummy_encoder_data = profiler.get_encoder_dummy_data(
39
- max_model_len,
40
- mm_counts=mm_config.limit_per_prompt,
41
- )
42
- dummy_mm_data = processor.dummy_inputs.get_dummy_processor_inputs(
43
- max_model_len,
44
- mm_counts=mm_config.limit_per_prompt,
45
- )
46
-
47
- hf_config = ctx.get_hf_config(MllamaConfig)
48
- image_size = hf_config.vision_config.image_size
49
- encoder_seq_lens = [len(dummy_encoder_data.prompt_token_ids)
50
- ] * max_num_seqs
51
-
52
- mm_data = processor.apply(
53
- prompt=dummy_mm_data.prompt,
54
- mm_data=dummy_mm_data.mm_data,
55
- hf_processor_mm_kwargs=dict(),
56
- )["mm_kwargs"].get_data()
57
-
58
- # Get the actual number of encoder tokens for each sample.
59
- # Because attn_metadata.encoder_seq_lens only counts the last
60
- # group of images for each sample, which is used to cheat the
61
- # block manager to allocate blocks for those images only.
62
- # See MllamaMultiModalProcessor for more details.
63
- num_tiles = [[t] for t in mm_data.pop("num_tiles")]
64
- num_tokens_per_tile = calc_token_per_chunk(image_size)
65
- actual_encoder_seq_lens = [
66
- sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
67
- ]
68
-
69
- # simulate mllama image-present prefill.
70
- for actual_len, last_group_len in zip(actual_encoder_seq_lens,
71
- encoder_seq_lens):
72
- assert actual_len >= last_group_len
tests/models/multimodal/processing/test_tensor_schema.py
-1
@@ -31,7 +31,6 @@ from ...utils import dummy_hf_overrides
31
32
ARCH_TO_SKIP = {
33
"MolmoForCausalLM": "incompatible requirements",
34
- "Florence2ForConditionalGeneration": "not supported in V1",
34
}
35
ARCH_NEEDS_EXTRAS = [
36
"InternVLChatModel",
tests/models/registry.py
+1
-15
@@ -354,11 +354,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
354
"MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL",
355
trust_remote_code=True),
356
"Dots1ForCausalLM": _HfExamplesInfo("rednote-hilab/dots.llm1.inst"),
357
- # [Encoder-decoder]
358
- "BartModel": _HfExamplesInfo("facebook/bart-base"),
359
- "BartForConditionalGeneration": _HfExamplesInfo("facebook/bart-large-cnn"),
360
- "MBartForConditionalGeneration": _HfExamplesInfo("facebook/mbart-large-en-ro", # noqa: E501
361
- hf_overrides={"architectures": ["MBartForConditionalGeneration"]}), # noqa: E501
357
}
358
359
_EMBEDDING_EXAMPLE_MODELS = {
@@ -496,7 +491,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
491
trust_remote_code=True),
492
"Llama4ForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-4-Scout-17B-16E-Instruct", # noqa: E501
493
max_model_len=10240,
499
- extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"}, # noqa: E501
494
+ extras={"llama-guard-4": "meta-llama/Llama-Guard-4-12B"}, # noqa: E501
495
),
496
"LlavaForConditionalGeneration": _HfExamplesInfo("llava-hf/llava-1.5-7b-hf",
497
extras={"mistral": "mistral-community/pixtral-12b", # noqa: E501
@@ -583,15 +578,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
578
is_available_online=False,
579
),
580
# [Encoder-decoder]
586
- "DonutForConditionalGeneration": _HfExamplesInfo("naver-clova-ix/donut-base-finetuned-docvqa", # noqa: E501
587
- hf_overrides={"architectures": ["DonutForConditionalGeneration"], "model_type": "donut"}, # noqa: E501
588
- extras={"dolphin": "ByteDance/Dolphin"}), # noqa: E501
589
- # Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
590
- # Therefore, we borrow the BartTokenizer from the original Bart model
591
- "Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
592
- tokenizer="Isotr0py/Florence-2-tokenizer", # noqa: E501
593
- trust_remote_code=True), # noqa: E501
594
- "MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
581
"WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501
582
# [Cross-encoder]
583
"JinaVLForRanking": _HfExamplesInfo("jinaai/jina-reranker-m0"), # noqa: E501
tests/models/test_initialization.py
-4
@@ -92,10 +92,6 @@ def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
92
# has cc==8.9 which hasn't supported FA3 yet. Remove this hack when
93
# L4 supports FA3.
94
m.setenv("VLLM_ATTENTION_BACKEND", "TRITON_ATTN_VLLM_V1")
95
- if model_arch == "Florence2ForConditionalGeneration":
96
- # An encoder-decoder model that's V0-only. Just skip it
97
- # since V0 is about to be removed.
98
- pytest.skip("Skipping Florence2ForConditionalGeneration")
95
if model_arch == "WhisperForConditionalGeneration":
96
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
97
LLM(
tests/models/test_registry.py
-1
@@ -50,7 +50,6 @@ def test_registry_imports(model_arch):
50
@create_new_process_for_each_test()
51
@pytest.mark.parametrize("model_arch,is_mm,init_cuda,is_ce", [
52
("LlamaForCausalLM", False, False, False),
53
- ("MllamaForConditionalGeneration", True, False, False),
53
("LlavaForConditionalGeneration", True, True, False),
54
("BertForSequenceClassification", False, False, True),
55
("RobertaForSequenceClassification", False, False, True),
tests/test_config.py
+1
-2
@@ -299,9 +299,8 @@ def test_rope_customization():
299
reason="Encoder Decoder models not supported on ROCm.")
300
@pytest.mark.parametrize(("model_id", "is_encoder_decoder"), [
301
("facebook/opt-125m", False),
302
- ("facebook/bart-base", True),
302
+ ("openai/whisper-tiny", True),
303
("meta-llama/Llama-3.2-1B-Instruct", False),
304
- ("meta-llama/Llama-3.2-11B-Vision", True),
304
])
305
def test_is_encoder_decoder(model_id, is_encoder_decoder):
306
config = ModelConfig(model_id)
tests/utils_/test_utils.py
-28
@@ -501,34 +501,6 @@ def test_bind_kv_cache_non_attention():
501
assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[1]
502
503
504
-def test_bind_kv_cache_encoder_decoder(monkeypatch: pytest.MonkeyPatch):
505
- # V1 TESTS: ENCODER_DECODER is not supported on V1 yet.
506
- with monkeypatch.context() as m:
507
- m.setenv("VLLM_USE_V1", "0")
508
-
509
- from vllm.attention import Attention, AttentionType
510
-
511
- # example from bart
512
- ctx = {
513
- 'encoder.layers.0.self_attn.attn':
514
- Attention(32, 128, 0.1, attn_type=AttentionType.ENCODER),
515
- 'decoder.layers.0.encoder_attn.attn':
516
- Attention(32, 128, 0.1, attn_type=AttentionType.ENCODER_DECODER),
517
- 'decoder.layers.0.self_attn.attn':
518
- Attention(32, 128, 0.1, attn_type=AttentionType.DECODER),
519
- }
520
-
521
- kv_cache = [
522
- torch.zeros((1, )),
523
- ]
524
- encoder_kv_cache = ctx['encoder.layers.0.self_attn.attn'].kv_cache
525
-
526
- bind_kv_cache(ctx, [kv_cache])
527
- assert ctx['encoder.layers.0.self_attn.attn'].kv_cache is encoder_kv_cache
528
- assert ctx['decoder.layers.0.encoder_attn.attn'].kv_cache[0] is kv_cache[0]
529
- assert ctx['decoder.layers.0.self_attn.attn'].kv_cache[0] is kv_cache[0]
530
-
531
-
504
def test_bind_kv_cache_pp():
505
with patch("vllm.utils.cuda_device_count_stateless", lambda: 2):
506
# this test runs with 1 GPU, but we simulate 2 GPUs
tests/v1/test_oracle.py
-21
@@ -9,24 +9,9 @@ from vllm import LLM
9
from vllm.engine.arg_utils import AsyncEngineArgs
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
12
-UNSUPPORTED_MODELS_V1 = [
13
- "facebook/bart-large-cnn", # encoder decoder
14
-]
15
-
12
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
13
14
19
-@pytest.mark.parametrize("model", UNSUPPORTED_MODELS_V1)
20
-def test_reject_unsupported_models(monkeypatch, model):
21
- with monkeypatch.context() as m:
22
- m.setenv("VLLM_USE_V1", "1")
23
- args = AsyncEngineArgs(model=model)
24
-
25
- with pytest.raises(NotImplementedError):
26
- _ = args.create_engine_config()
27
- m.delenv("VLLM_USE_V1")
28
-
29
-
15
def test_reject_bad_config(monkeypatch):
16
with monkeypatch.context() as m:
17
m.setenv("VLLM_USE_V1", "0")
@@ -77,12 +62,6 @@ def test_enable_by_default_fallback(monkeypatch):
62
assert envs.VLLM_USE_V1
63
m.delenv("VLLM_USE_V1")
64
80
- # Should fall back to V0 for supported model.
81
- _ = AsyncEngineArgs(
82
- model=UNSUPPORTED_MODELS_V1[0]).create_engine_config()
83
- assert not envs.VLLM_USE_V1
84
- m.delenv("VLLM_USE_V1")
85
-
65
66
def test_v1_llm_by_default(monkeypatch):
67
with monkeypatch.context() as m:
tests/worker/test_encoder_decoder_model_runner.py
deleted
-648
@@ -1,648 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import itertools
5
-
6
-import pytest
7
-import torch
8
-
9
-from vllm.engine.arg_utils import EngineArgs
10
-from vllm.platforms import current_platform
11
-from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
-from vllm.utils import make_tensor_with_pad
13
-from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
14
-
15
-BATCH_SIZES = [1, 4, 16, 64, 256]
16
-
17
-
18
-def _create_model_runner(model: str, *args,
19
- **kwargs) -> EncoderDecoderModelRunner:
20
- engine_args = EngineArgs(model, *args, **kwargs)
21
- engine_config = engine_args.create_engine_config()
22
- model_runner = EncoderDecoderModelRunner(
23
- vllm_config=engine_config,
24
- is_driver_worker=True,
25
- )
26
- return model_runner
27
-
28
-
29
-@pytest.mark.skipif(condition=current_platform.is_cpu(),
30
- reason="CPU backend is currently "
31
- "unsupported for encoder/ "
32
- "decoder models")
33
-def test_empty_seq_group():
34
- """Verify prepare prompt and decode returns empty output
35
- for empty seq group list"""
36
-
37
- model_runner = _create_model_runner(
38
- "facebook/bart-base",
39
- seed=0,
40
- dtype="float16",
41
- max_num_batched_tokens=100000,
42
- max_num_seqs=100000,
43
- enable_chunked_prefill=False,
44
- enforce_eager=True,
45
- )
46
- seq_group_metadata_list: list[SequenceGroupMetadata] = []
47
- model_input = model_runner._prepare_model_input_tensors(
48
- seq_group_metadata_list)
49
- (
50
- input_tokens,
51
- input_positions,
52
- encoder_input_tokens,
53
- encoder_input_positions,
54
- attn_metadata,
55
- return_seq_lens,
56
- ) = (
57
- model_input.input_tokens,
58
- model_input.input_positions,
59
- model_input.encoder_input_tokens,
60
- model_input.encoder_input_positions,
61
- model_input.attn_metadata,
62
- model_input.seq_lens,
63
- )
64
- assert input_tokens is None
65
- assert input_positions is None
66
- assert encoder_input_tokens is None
67
- assert encoder_input_positions is None
68
- assert attn_metadata is None
69
- assert return_seq_lens is None
70
-
71
-
72
-@pytest.mark.skipif(condition=current_platform.is_cpu(),
73
- reason="CPU backend is currently "
74
- "unsupported for encoder/ "
75
- "decoder models")
76
-@pytest.mark.parametrize("batch_size", BATCH_SIZES)
77
-def test_prepare_prompt(batch_size):
78
- '''
79
- Test the ability of the encoder/decoder model runner subclass to
80
- produce prefill-phase model inputs & attention metadata.
81
-
82
- Test behavior:
83
-
84
- * Instantiate BART base model & enc/dec model runner
85
- * Construct sequence-group metadata for dummy prompts
86
- * Test that encoder attention, decoder self-attention,
87
- and encoder/decoder cross-attention inputs are correct
88
-
89
- Arguments:
90
-
91
- * batch_size
92
- * backend_name: The attention backend under test
93
- * enforce_eager: Enforce eager mode if True (i.e. no CUDAGraph)
94
- '''
95
-
96
- model_runner = _create_model_runner(
97
- "facebook/bart-base",
98
- seed=0,
99
- dtype="float16",
100
- max_num_batched_tokens=100000,
101
- max_num_seqs=100000,
102
- enable_chunked_prefill=False,
103
- enforce_eager=True,
104
- )
105
-
106
- seq_lens: list[int] = []
107
- encoder_seq_lens: list[int] = []
108
- seq_group_metadata_list: list[SequenceGroupMetadata] = []
109
- block_tables = {0: [1]}
110
- cross_block_table = [2]
111
- for i in range(batch_size):
112
- # make sure all tokens fit into one block
113
- seq_len = i % (model_runner.block_size - 1) + 1
114
- seq_lens.append(seq_len)
115
- seq_data = SequenceData.from_seqs(range(seq_len))
116
- encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
117
- encoder_seq_lens.append(encoder_seq_len)
118
- encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
119
- seq_group_metadata = SequenceGroupMetadata(
120
- request_id=f"test_{i}",
121
- is_prompt=True,
122
- seq_data={0: seq_data},
123
- sampling_params=SamplingParams(temperature=0),
124
- block_tables=block_tables,
125
- encoder_seq_data=encoder_seq_data,
126
- cross_block_table=cross_block_table,
127
- )
128
- assert seq_group_metadata.token_chunk_size == seq_data.get_len()
129
- seq_group_metadata_list.append(seq_group_metadata)
130
-
131
- # Build
132
- # * Decoder model inputs
133
- # * Decoder self-attention KV caching data structures
134
- # * Encoder model inputs
135
- # * Encoder/decoder cross-attention KV caching data structures
136
- model_input = model_runner.prepare_model_input(seq_group_metadata_list)
137
-
138
- input_tokens = model_input.input_tokens
139
- input_positions = model_input.input_positions
140
- attn_metadata = model_input.attn_metadata
141
- return_seq_lens = model_input.seq_lens
142
- slot_mapping = attn_metadata.slot_mapping
143
- encoder_input_tokens = model_input.encoder_input_tokens
144
- encoder_input_positions = model_input.encoder_input_positions
145
- cross_slot_mapping = attn_metadata.cross_slot_mapping
146
- assert return_seq_lens == seq_lens
147
- assert len(slot_mapping) == len(input_tokens)
148
- assert len(cross_slot_mapping) == len(encoder_input_tokens)
149
-
150
- # Verify input metadata is correct for prompts.
151
- # - Decoder attention metadata
152
- device = model_runner.device
153
- assert attn_metadata.num_prefills > 0
154
- assert attn_metadata.num_decode_tokens == 0
155
- assert torch.equal(attn_metadata.seq_lens_tensor,
156
- torch.tensor(seq_lens, device=device, dtype=torch.int))
157
- assert attn_metadata.seq_lens == seq_lens
158
- assert attn_metadata.max_prefill_seq_len == max(seq_lens)
159
- assert attn_metadata.max_decode_seq_len == 0
160
- # - Encoder attention metadata
161
- assert attn_metadata.encoder_seq_lens == encoder_seq_lens
162
- assert torch.equal(
163
- attn_metadata.encoder_seq_lens_tensor,
164
- torch.tensor(encoder_seq_lens, device=device, dtype=torch.int))
165
- assert attn_metadata.max_encoder_seq_len == max(encoder_seq_lens)
166
- assert attn_metadata.num_encoder_tokens == sum(encoder_seq_lens)
167
-
168
- # Test decoder subquery start locs.
169
- start_idx = 0
170
- start_loc = [start_idx]
171
- for seq_len in seq_lens:
172
- start_idx += seq_len
173
- start_loc.append(start_idx)
174
- assert torch.equal(
175
- attn_metadata.query_start_loc,
176
- torch.tensor(start_loc, dtype=torch.int32, device=device),
177
- )
178
-
179
- # Test decoder seq start locs & context lengths
180
-
181
- assert torch.equal(
182
- attn_metadata.seq_start_loc,
183
- torch.tensor(start_loc, dtype=torch.int32, device=device),
184
- )
185
- assert torch.equal(
186
- attn_metadata.context_lens_tensor,
187
- torch.zeros(attn_metadata.context_lens_tensor.shape[0],
188
- dtype=torch.int,
189
- device=device),
190
- )
191
-
192
- # Verify block tables are correct for prompts
193
- # - Decoder self-attention
194
- expected = torch.tensor(
195
- [[] for _ in range(len(seq_group_metadata_list))],
196
- dtype=torch.int32,
197
- device=model_runner.device,
198
- )
199
- assert torch.equal(
200
- attn_metadata.block_tables,
201
- expected,
202
- )
203
- # - Encoder/decoder cross-attention
204
- assert torch.equal(
205
- attn_metadata.cross_block_tables,
206
- expected,
207
- )
208
-
209
- # Cuda graph should not be used for prefill.
210
- assert attn_metadata.use_cuda_graph is False
211
-
212
- # Verify the lengths of input tokens & positions
213
- # - Decoder
214
- assert len(input_tokens) == sum(seq_lens)
215
- assert len(input_positions) == sum(seq_lens)
216
- # -- An indirect check that model_input.input_tokens
217
- # and model_input.input_positions are correct -
218
- # by design of the test, the input tokens are
219
- # equal to the input position values, so if
220
- # the model_input data structure has the correct
221
- # values then these two should be equal
222
- assert torch.equal(
223
- input_tokens,
224
- input_positions,
225
- )
226
- # - Encoder
227
- assert len(encoder_input_tokens) == sum(encoder_seq_lens)
228
- # -- An indirect check that model_input.encoder_input_tokens
229
- # and model_input.encoder_input_positions are correct -
230
- # by design of the test, the input tokens are
231
- # equal to the input position values, so if
232
- # the model_input data structure has the correct
233
- # values then these two should be equal
234
- assert torch.equal(
235
- encoder_input_tokens,
236
- encoder_input_positions,
237
- )
238
-
239
- # Test that vLLM sampling infrastructure chooses the correct
240
- # sequence positions at which to sample (i.e. the end of
241
- # each sequence) in the prefill phase
242
-
243
- expected_selected_token_indices = []
244
- selected_token_start_idx = 0
245
- for seq_len in seq_lens:
246
- # Compute the index offset of the final token in each
247
- # prompt (recall that the prompts are concatenated)
248
- expected_selected_token_indices.append(selected_token_start_idx +
249
- seq_len - 1)
250
- selected_token_start_idx += seq_len
251
-
252
- sampling_metadata = model_input.sampling_metadata
253
- actual = sampling_metadata.selected_token_indices
254
- expected = torch.tensor(
255
- expected_selected_token_indices,
256
- device=actual.device,
257
- dtype=actual.dtype,
258
- )
259
- assert torch.equal(actual, expected)
260
-
261
-
262
-@pytest.mark.skipif(condition=current_platform.is_cpu(),
263
- reason="CPU backend is currently "
264
- "unsupported for encoder/ "
265
- "decoder models")
266
-@pytest.mark.parametrize("batch_size", BATCH_SIZES)
267
-@pytest.mark.parametrize("multiple_seqs_per_seq_group", [True, False])
268
-def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
269
- '''
270
- Test the ability of the encoder/decoder model runner subclass to
271
- produce decode-phase model inputs & attention metadata.
272
-
273
- Test behavior:
274
-
275
- * Instantiate BART base model & enc/dec model runner
276
- * Construct sequence-group metadata for dummy prompts
277
- * Test that encoder attention, decoder self-attention,
278
- and encoder/decoder cross-attention inputs are correct
279
-
280
- Arguments:
281
-
282
- * batch_size
283
- * multiple_seqs_per_seq_group
284
- * backend_name: The attention backend under test
285
- * enforce_eager: Enforce eager mode if True (i.e. no CUDAGraph)
286
- '''
287
-
288
- model_runner = _create_model_runner(
289
- "facebook/bart-base",
290
- seed=0,
291
- dtype="float16",
292
- max_num_batched_tokens=100000,
293
- max_num_seqs=100000,
294
- enable_chunked_prefill=False,
295
- enforce_eager=True,
296
- )
297
-
298
- seq_lens: list[int] = []
299
- encoder_seq_lens: list[int] = []
300
- seq_group_metadata_list: list[SequenceGroupMetadata] = []
301
- block_tables = {
302
- 0: [1],
303
- 1: [3]
304
- } if multiple_seqs_per_seq_group else {
305
- 0: [1]
306
- }
307
- cross_block_table = [2]
308
- for i in range(batch_size):
309
- # make sure all tokens fit into one block
310
- seq_len = i % (model_runner.block_size - 1) + 1
311
- seq_data = SequenceData.from_seqs(range(seq_len))
312
- encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
313
- encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
314
-
315
- seq_group_metadata = SequenceGroupMetadata(
316
- request_id=f"test_{i}",
317
- is_prompt=False,
318
- seq_data={
319
- 0: seq_data,
320
- 1: seq_data
321
- } if multiple_seqs_per_seq_group else {0: seq_data},
322
- sampling_params=SamplingParams(temperature=0),
323
- block_tables=block_tables,
324
- encoder_seq_data=encoder_seq_data,
325
- cross_block_table=cross_block_table,
326
- )
327
- assert seq_group_metadata.token_chunk_size == 1
328
- seq_group_metadata_list.append(seq_group_metadata)
329
- seq_lens.extend(
330
- [seq_len for _ in range(len(seq_group_metadata.seq_data))])
331
- encoder_seq_lens.extend(
332
- [encoder_seq_len for _ in range(len(seq_group_metadata.seq_data))])
333
-
334
- # Build
335
- # * Decoder model inputs
336
- # * Decoder self-attention KV caching data structures
337
- # * Encoder model inputs
338
- # * Encoder/decoder cross-attention KV caching data structures
339
- model_input = model_runner.prepare_model_input(seq_group_metadata_list)
340
- input_tokens = model_input.input_tokens
341
- input_positions = model_input.input_positions
342
- attn_metadata = model_input.attn_metadata
343
- return_seq_lens = model_input.seq_lens
344
- slot_mapping = attn_metadata.slot_mapping
345
- encoder_input_tokens = model_input.encoder_input_tokens
346
- encoder_input_positions = model_input.encoder_input_positions
347
- cross_slot_mapping = attn_metadata.cross_slot_mapping
348
- assert return_seq_lens == seq_lens
349
- assert len(slot_mapping) == len(input_tokens)
350
- assert len(cross_slot_mapping) == len(encoder_input_tokens)
351
-
352
- # Verify input metadata is correct for decode phase.
353
- # - Decoder attention metadata
354
- device = model_runner.device
355
- assert attn_metadata.num_prefills == 0
356
- assert attn_metadata.num_decode_tokens > 0
357
- assert torch.equal(attn_metadata.seq_lens_tensor,
358
- torch.tensor(seq_lens, device=device, dtype=torch.int))
359
- assert attn_metadata.seq_lens == seq_lens
360
- assert attn_metadata.max_prefill_seq_len == 0
361
- assert attn_metadata.max_decode_seq_len == max(seq_lens)
362
- # - Encoder attention metadata
363
- assert attn_metadata.encoder_seq_lens == encoder_seq_lens
364
- assert torch.equal(
365
- attn_metadata.encoder_seq_lens_tensor,
366
- torch.tensor(encoder_seq_lens, device=device, dtype=torch.int))
367
- assert attn_metadata.max_encoder_seq_len == max(encoder_seq_lens)
368
- assert attn_metadata.num_encoder_tokens == sum(encoder_seq_lens)
369
-
370
- # Test decoder subquery start locs.
371
- start_idx = 0
372
- start_loc = [start_idx]
373
- for seq_len in seq_lens:
374
- start_idx += 1
375
- start_loc.append(start_idx)
376
- assert torch.equal(
377
- attn_metadata.query_start_loc,
378
- torch.tensor(start_loc, dtype=torch.int32, device=device),
379
- )
380
-
381
- # Test decoder seq start locs. Note that for normal prefill it is
382
- # equivalent to query_start_loc.
383
- start_idx = 0
384
- seq_start_loc = [start_idx]
385
- for seq_len in seq_lens:
386
- start_idx += seq_len
387
- seq_start_loc.append(start_idx)
388
-
389
- # Test seq_start_loc and context lengths
390
-
391
- assert torch.equal(
392
- attn_metadata.seq_start_loc,
393
- torch.tensor(seq_start_loc, dtype=torch.int32, device=device),
394
- )
395
- assert torch.equal(
396
- attn_metadata.context_lens_tensor,
397
- torch.tensor([seq_len - 1 for seq_len in seq_lens],
398
- dtype=torch.int,
399
- device=device))
400
-
401
- # Verify block tables are correct for prompts
402
- # - Decoder self-attention
403
- flattened_block_tables = [
404
- block_table for block_table in block_tables.values()
405
- ]
406
- expected = torch.tensor(flattened_block_tables *
407
- len(seq_group_metadata_list),
408
- dtype=torch.int32,
409
- device=model_runner.device)
410
- assert torch.equal(
411
- attn_metadata.block_tables,
412
- expected,
413
- )
414
- # - Encoder/decoder cross-attention
415
- expected = torch.tensor([
416
- cross_block_table for seq_group_metadata in seq_group_metadata_list
417
- for _ in range(len(seq_group_metadata.seq_data))
418
- ],
419
- dtype=torch.int32,
420
- device=model_runner.device)
421
- assert torch.equal(
422
- attn_metadata.cross_block_tables,
423
- expected,
424
- )
425
-
426
- # Model runner's CUDAGraph setting should be propagated to attention
427
- # metadata.
428
- assert attn_metadata.use_cuda_graph is False
429
-
430
- # Verify the lengths of input tokens & positions
431
- # - Decoder
432
- assert len(input_tokens) == len(seq_lens)
433
- assert len(input_positions) == len(seq_lens)
434
- # -- An indirect check that model_input.input_tokens
435
- # and model_input.input_positions are correct -
436
- # by design of the test, the input tokens are
437
- # equal to the input position values, so if
438
- # the model_input data structure has the correct
439
- # values then these two should be equal
440
- assert torch.equal(
441
- input_tokens,
442
- input_positions,
443
- )
444
- # - Encoder
445
- assert len(encoder_input_tokens) == 0
446
- assert len(encoder_input_tokens) == 0
447
- # -- An indirect check that model_input.encoder_input_tokens
448
- # and model_input.encoder_input_positions are correct -
449
- # by design of the test, the input tokens are
450
- # equal to the input position values, so if
451
- # the model_input data structure has the correct
452
- # values then these two should be equal
453
- assert torch.equal(
454
- encoder_input_tokens,
455
- encoder_input_positions,
456
- )
457
-
458
- # Test that vLLM sampling infrastructure chooses the correct
459
- # sequence positions at which to sample (i.e. the end of
460
- # each sequence) in the decode phase
461
-
462
- expected_selected_token_indices = []
463
- for selected_token_start_idx, seq_len in enumerate(seq_lens):
464
- # Compute the index offset of the final token in each
465
- # sequence's decoded outputs; since a single token is
466
- # decoded per iteration per sequence, then the length
467
- # of the decoded tokens for a given sequence is 1 and
468
- # the final index offset into a given sequence's
469
- # generated tokens is 0 (i.e. the expected sampling index
470
- # for a given sequence is just `selected_token_start_idx`)
471
- expected_selected_token_indices.append(selected_token_start_idx)
472
-
473
- sampling_metadata = model_input.sampling_metadata
474
- actual = sampling_metadata.selected_token_indices
475
- expected = torch.tensor(
476
- expected_selected_token_indices,
477
- device=actual.device,
478
- dtype=actual.dtype,
479
- )
480
- assert torch.equal(actual, expected)
481
-
482
-
483
-@pytest.mark.parametrize("batch_size", list(range(1, 257)))
484
-@pytest.mark.parametrize("multiple_seqs_per_seq_group", [True, False])
485
-def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
486
- """
487
- Tests that for encoder-decoder models with CUDA Graph capture and replay
488
- enabled, the tensors used during the decode phase are correctly padded
489
- for varying input batch sizes.
490
- """
491
- model_runner = _create_model_runner(
492
- "facebook/bart-base",
493
- seed=0,
494
- dtype="float16",
495
- max_num_batched_tokens=100000,
496
- max_num_seqs=100000,
497
- enable_chunked_prefill=False,
498
- enforce_eager=False,
499
- )
500
- block_tables = {
501
- 0: [1],
502
- 1: [3]
503
- } if multiple_seqs_per_seq_group else {
504
- 0: [1]
505
- }
506
- seq_lens: list[int] = []
507
- encoder_seq_lens: list[int] = []
508
- seq_group_metadata_list: list[SequenceGroupMetadata] = []
509
-
510
- cross_block_table = [2]
511
- expanded_batch_size = 0
512
- for i in range(batch_size):
513
- # make sure all tokens fit into one block
514
- seq_len = i % (model_runner.block_size - 1) + 1
515
- seq_data = SequenceData.from_seqs(range(seq_len))
516
- encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
517
- encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
518
- seq_group_metadata = SequenceGroupMetadata(
519
- request_id=f"test_{i}",
520
- is_prompt=False,
521
- seq_data={
522
- 0: seq_data,
523
- 1: seq_data
524
- } if multiple_seqs_per_seq_group else {0: seq_data},
525
- sampling_params=SamplingParams(temperature=0),
526
- block_tables=block_tables,
527
- encoder_seq_data=encoder_seq_data,
528
- cross_block_table=cross_block_table,
529
- )
530
- assert seq_group_metadata.token_chunk_size == 1
531
- seq_lens.extend(
532
- [seq_len for _ in range(len(seq_group_metadata.seq_data))])
533
- encoder_seq_lens.extend(
534
- [encoder_seq_len for _ in range(len(seq_group_metadata.seq_data))])
535
- expanded_batch_size = expanded_batch_size + len(
536
- seq_group_metadata.seq_data)
537
- seq_group_metadata_list.append(seq_group_metadata)
538
-
539
- model_input = model_runner.prepare_model_input(seq_group_metadata_list)
540
- input_tokens = model_input.input_tokens
541
- input_positions = model_input.input_positions
542
- attn_metadata = model_input.attn_metadata
543
- return_seq_lens = model_input.seq_lens
544
- slot_mapping = attn_metadata.slot_mapping
545
- encoder_input_tokens = model_input.encoder_input_tokens
546
- encoder_input_positions = model_input.encoder_input_positions
547
- cross_slot_mapping = attn_metadata.cross_slot_mapping
548
-
549
- # With CUDA Graph capture and replay enabled, the decoder and encoder
550
- # input sequences will be padded. Create the expected padded tensors
551
- # accordingly.
552
- graph_batch_size = model_runner.vllm_config.pad_for_cudagraph(
553
- expanded_batch_size)
554
- cuda_graph_pad_size = graph_batch_size - expanded_batch_size
555
- padded_seq_lens = seq_lens + list(itertools.repeat(1, cuda_graph_pad_size))
556
- padded_encoder_seq_lens = encoder_seq_lens + list(
557
- itertools.repeat(1, cuda_graph_pad_size))
558
-
559
- assert return_seq_lens == padded_seq_lens
560
- assert len(slot_mapping) == len(input_tokens)
561
- assert len(cross_slot_mapping) == len(encoder_input_tokens)
562
-
563
- # Verify attention metadata
564
- device = model_runner.device
565
- assert attn_metadata.num_prefills == 0
566
- assert attn_metadata.num_decode_tokens > 0
567
- assert torch.equal(
568
- attn_metadata.seq_lens_tensor,
569
- torch.tensor(padded_seq_lens, device=device, dtype=torch.int))
570
- assert attn_metadata.seq_lens == padded_seq_lens
571
- assert attn_metadata.max_prefill_seq_len == 0
572
- assert attn_metadata.max_decode_seq_len == max(seq_lens)
573
- # - Encoder attention metadata
574
- assert attn_metadata.encoder_seq_lens == padded_encoder_seq_lens
575
- assert torch.equal(
576
- attn_metadata.encoder_seq_lens_tensor,
577
- torch.tensor(padded_encoder_seq_lens, device=device, dtype=torch.int))
578
- assert attn_metadata.max_encoder_seq_len == max(padded_encoder_seq_lens)
579
- assert attn_metadata.num_encoder_tokens == sum(padded_encoder_seq_lens)
580
-
581
- # Verify block tables are correct for prompts
582
- # - Decoder self-attention. Pad the block tables as expected.
583
- flattened_block_tables = [
584
- block_table for _ in range(len(seq_group_metadata_list))
585
- for block_table in block_tables.values()
586
- ]
587
- flattened_block_tables.extend([[] for _ in range(cuda_graph_pad_size)])
588
- expected = make_tensor_with_pad(
589
- flattened_block_tables,
590
- max_len=64,
591
- pad=0,
592
- dtype=torch.int32,
593
- device=model_runner.device,
594
- )
595
- assert torch.equal(
596
- attn_metadata.block_tables,
597
- expected,
598
- )
599
- # - Encoder/decoder cross-attention. Pad the cross-attention block tables
600
- # as expected.
601
- expected = [
602
- cross_block_table for seq_group_metadata in seq_group_metadata_list
603
- for _ in range(len(seq_group_metadata.seq_data))
604
- ]
605
- expected.extend([[] for _ in range(cuda_graph_pad_size)])
606
- expected = make_tensor_with_pad(
607
- expected,
608
- max_len=64,
609
- pad=0,
610
- dtype=torch.int32,
611
- device=model_runner.device,
612
- )
613
- assert torch.equal(
614
- attn_metadata.cross_block_tables,
615
- expected,
616
- )
617
-
618
- # Model runner's CUDAGraph setting should be propagated to attention
619
- # metadata.
620
- assert attn_metadata.use_cuda_graph is True
621
-
622
- # Verify the lengths of input tokens & positions
623
- # - Decoder
624
- assert len(input_tokens) == len(padded_seq_lens)
625
- assert len(input_positions) == len(padded_seq_lens)
626
- # -- An indirect check that model_input.input_tokens
627
- # and model_input.input_positions are correct -
628
- # by design of the test, the input tokens are
629
- # equal to the input position values, so if
630
- # the model_input data structure has the correct
631
- # values then these two should be equal
632
- assert torch.equal(
633
- input_tokens,
634
- input_positions,
635
- )
636
- # - Encoder
637
- assert len(encoder_input_tokens) == 0
638
- assert len(encoder_input_tokens) == 0
639
- # -- An indirect check that model_input.encoder_input_tokens
640
- # and model_input.encoder_input_positions are correct -
641
- # by design of the test, the input tokens are
642
- # equal to the input position values, so if
643
- # the model_input data structure has the correct
644
- # values then these two should be equal
645
- assert torch.equal(
646
- encoder_input_tokens,
647
- encoder_input_positions,
648
- )
vllm/config/__init__.py
+2
-9
@@ -1201,11 +1201,8 @@ class ModelConfig:
1201
getattr(self.hf_config, "max_source_positions", 0))
1202
self.max_seq_len_to_capture = min(self.max_seq_len_to_capture,
1203
effective_max_seq_len)
1204
- # CUDAGraph capture not supported for enc-dec models and mllama on ROCm
1205
- ROCM_UNSUPPORTED_MODELS = ['mllama']
1206
- unsupported_rocm = (self.hf_config.model_type
1207
- in ROCM_UNSUPPORTED_MODELS
1208
- or self.is_encoder_decoder)
1204
+ # CUDAGraph capture not supported for encoder-decoder models on ROCm
1205
+ unsupported_rocm = self.is_encoder_decoder
1206
1207
if (unsupported_rocm and not self.enforce_eager
1208
and current_platform.is_rocm()):
@@ -1671,10 +1668,6 @@ class ModelConfig:
1668
@property
1669
def is_encoder_decoder(self) -> bool:
1670
"""Extract the HF encoder/decoder model flag."""
1674
- """
1675
- For Mllama, VLLM overrides HF's is_encoder_decoder flag and sets it to
1676
- True to enable cross-attention
1677
- """
1671
return is_encoder_decoder(self.hf_config)
1672
1673
@property
vllm/engine/llm_engine.py
+1
-1
@@ -1789,7 +1789,7 @@ class LLMEngine:
1789
assert isinstance(mm_processor, EncDecMultiModalProcessor)
1790
1791
if mm_processor.pad_dummy_encoder_prompt:
1792
- return # Skip encoder length check for Whisper and Donut
1792
+ return # Skip encoder length check for Whisper
1793
1794
if model_config.is_multimodal_model:
1795
suggestion = (
vllm/model_executor/models/bart.py
deleted
-1319
@@ -1,1319 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Derived from BART implementation posted on HuggingFace; license below:
5
-#
6
-# coding=utf-8
7
-# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team.
8
-# All rights reserved.
9
-#
10
-# Licensed under the Apache License, Version 2.0 (the "License");
11
-# you may not use this file except in compliance with the License.
12
-# You may obtain a copy of the License at
13
-#
14
-# http://www.apache.org/licenses/LICENSE-2.0
15
-#
16
-# Unless required by applicable law or agreed to in writing, software
17
-# distributed under the License is distributed on an "AS IS" BASIS,
18
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19
-# See the License for the specific language governing permissions and
20
-# limitations under the License.
21
-"""PyTorch BART model."""
22
-import math
23
-from collections.abc import Iterable
24
-from typing import Optional
25
-
26
-import torch
27
-from torch import nn
28
-from transformers import BartConfig
29
-from transformers.utils import logging
30
-
31
-from vllm.attention import Attention, AttentionType
32
-from vllm.config import CacheConfig, VllmConfig
33
-from vllm.config.lora import LoRAConfig
34
-from vllm.distributed import get_tensor_model_parallel_world_size
35
-from vllm.model_executor.layers.activation import get_act_fn
36
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
37
- QKVCrossParallelLinear,
38
- QKVParallelLinear,
39
- RowParallelLinear)
40
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
41
-from vllm.model_executor.layers.quantization.base_config import (
42
- QuantizationConfig)
43
-from vllm.model_executor.layers.vocab_parallel_embedding import (
44
- ParallelLMHead, VocabParallelEmbedding)
45
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
-from vllm.model_executor.sampling_metadata import SamplingMetadata
47
-from vllm.sequence import IntermediateTensors
48
-
49
-from .interfaces import SupportsQuant, SupportsV0Only
50
-from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,
51
- maybe_prefix)
52
-
53
-logger = logging.get_logger(__name__)
54
-
55
-
56
-def get_bsz_seq_len(input_ids):
57
- shp = input_ids.shape
58
- ndim = len(shp)
59
- if ndim == 1:
60
- return 1, input_ids.numel()
61
- else:
62
- return shp[:2]
63
-
64
-
65
-class BartLearnedPositionalEmbedding(VocabParallelEmbedding):
66
- """
67
- This module learns positional embeddings up to a fixed maximum size.
68
- """
69
-
70
- def __init__(self, num_embeddings: int, embedding_dim: int):
71
- # Bart is set up so that if padding_idx is
72
- # specified then offset the embedding ids by 2
73
- # and adjust num_embeddings appropriately.
74
- # Other models don't have this hack
75
- self.offset = 2
76
- super().__init__(num_embeddings + self.offset, embedding_dim)
77
-
78
- def forward(
79
- self,
80
- positions: torch.Tensor,
81
- ) -> torch.Tensor:
82
- """`input_ids' shape is expected to be [bsz x seqlen]."""
83
- return super().forward(positions + self.offset)
84
-
85
-
86
-class BartScaledWordEmbedding(VocabParallelEmbedding):
87
- """
88
- This module overrides VocabParallelEmbedding's
89
- forward by multiplying with embeddings scale.
90
- """
91
-
92
- def __init__(self,
93
- num_embeddings: int,
94
- embedding_dim: int,
95
- embed_scale: float = 1.0):
96
- super().__init__(num_embeddings, embedding_dim)
97
- self.embed_scale = embed_scale
98
-
99
- def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
100
- return super().forward(input_ids) * self.embed_scale
101
-
102
-
103
-class BartParallelLMHead(ParallelLMHead):
104
- """
105
- This module overrides ParallelLMHead's
106
- forward by dividing by embeddings scale,
107
- yielding effectively the inverse of
108
- BartScaledWordEmbedding
109
- """
110
-
111
- def __init__(self,
112
- num_embeddings: int,
113
- embedding_dim: int,
114
- embed_scale: float = 1.0):
115
- super().__init__(num_embeddings, embedding_dim)
116
- self.embed_scale = embed_scale
117
-
118
- def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
119
- return super().forward(input_ids) / self.embed_scale
120
-
121
-
122
-class BartEncoderAttention(nn.Module):
123
-
124
- def __init__(
125
- self,
126
- embed_dim: int,
127
- num_heads: int,
128
- bias: bool = True,
129
- config: Optional[BartConfig] = None,
130
- cache_config: Optional[CacheConfig] = None,
131
- quant_config: Optional[QuantizationConfig] = None,
132
- prefix: str = "",
133
- ):
134
- super().__init__()
135
- self.d_model = config.d_model
136
- self.embed_dim = embed_dim
137
- self.total_num_heads = num_heads
138
- self.total_num_kv_heads = self.total_num_heads
139
- self.head_dim = embed_dim // num_heads
140
- self.config = config
141
-
142
- if (self.head_dim * num_heads) != self.embed_dim:
143
- raise ValueError(f"embed_dim must be divisible by num_heads "
144
- f"(got `embed_dim`: {self.embed_dim}"
145
- f" and `num_heads`: {num_heads}).")
146
- self.scaling = self.head_dim**-0.5
147
-
148
- self.qkv_proj = QKVParallelLinear(
149
- self.d_model,
150
- self.d_model // self.total_num_heads,
151
- self.total_num_heads,
152
- self.total_num_kv_heads,
153
- bias=bias,
154
- quant_config=quant_config,
155
- )
156
-
157
- self.out_proj = RowParallelLinear(
158
- embed_dim,
159
- embed_dim,
160
- bias=bias,
161
- quant_config=quant_config,
162
- )
163
-
164
- tp_world_size = get_tensor_model_parallel_world_size()
165
- assert self.total_num_heads % tp_world_size == 0
166
- self.num_heads = self.total_num_heads // tp_world_size
167
-
168
- if self.total_num_kv_heads >= tp_world_size:
169
- # Number of KV heads is greater than TP size, so we partition
170
- # the KV heads across multiple tensor parallel GPUs.
171
- assert self.total_num_kv_heads % tp_world_size == 0
172
- else:
173
- # Number of KV heads is less than TP size, so we replicate
174
- # the KV heads across multiple tensor parallel GPUs.
175
- assert tp_world_size % self.total_num_kv_heads == 0
176
- self.num_kv_heads = self.num_heads
177
- self.q_size = self.num_heads * self.head_dim
178
- self.kv_size = self.num_kv_heads * self.head_dim
179
-
180
- self.attn = Attention(self.num_heads,
181
- self.head_dim,
182
- self.scaling,
183
- num_kv_heads=self.num_kv_heads,
184
- cache_config=cache_config,
185
- quant_config=quant_config,
186
- prefix=f"{prefix}.attn",
187
- attn_type=AttentionType.ENCODER)
188
-
189
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
190
- """Input shape: Batch x Time x Channel"""
191
-
192
- qkv, _ = self.qkv_proj(hidden_states)
193
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
194
-
195
- attn_output = self.attn(q, k, v)
196
-
197
- output, _ = self.out_proj(attn_output)
198
- return output
199
-
200
-
201
-class BartDecoderSelfAttention(nn.Module):
202
-
203
- def __init__(
204
- self,
205
- embed_dim: int,
206
- num_heads: int,
207
- bias: bool = True,
208
- config: Optional[BartConfig] = None,
209
- cache_config: Optional[CacheConfig] = None,
210
- quant_config: Optional[QuantizationConfig] = None,
211
- prefix: str = "",
212
- ):
213
- super().__init__()
214
- self.d_model = config.d_model
215
- self.embed_dim = embed_dim
216
- self.total_num_heads = num_heads
217
- self.total_num_kv_heads = self.total_num_heads
218
- self.head_dim = embed_dim // num_heads
219
- self.config = config
220
-
221
- if (self.head_dim * num_heads) != self.embed_dim:
222
- raise ValueError(f"embed_dim must be divisible by num_heads "
223
- f"(got `embed_dim`: {self.embed_dim}"
224
- f" and `num_heads`: {num_heads}).")
225
- self.scaling = self.head_dim**-0.5
226
-
227
- self.qkv_proj = QKVParallelLinear(
228
- self.d_model,
229
- self.d_model // self.total_num_heads,
230
- self.total_num_heads,
231
- self.total_num_kv_heads,
232
- bias=bias,
233
- quant_config=quant_config,
234
- )
235
-
236
- self.out_proj = RowParallelLinear(
237
- embed_dim,
238
- embed_dim,
239
- bias=bias,
240
- quant_config=quant_config,
241
- )
242
-
243
- tp_world_size = get_tensor_model_parallel_world_size()
244
- assert self.total_num_heads % tp_world_size == 0
245
- self.num_heads = self.total_num_heads // tp_world_size
246
-
247
- if self.total_num_kv_heads >= tp_world_size:
248
- # Number of KV heads is greater than TP size, so we partition
249
- # the KV heads across multiple tensor parallel GPUs.
250
- assert self.total_num_kv_heads % tp_world_size == 0
251
- else:
252
- # Number of KV heads is less than TP size, so we replicate
253
- # the KV heads across multiple tensor parallel GPUs.
254
- assert tp_world_size % self.total_num_kv_heads == 0
255
- self.num_kv_heads = self.num_heads
256
- self.q_size = self.num_heads * self.head_dim
257
- self.kv_size = self.num_kv_heads * self.head_dim
258
-
259
- self.attn = Attention(self.num_heads,
260
- self.head_dim,
261
- self.scaling,
262
- num_kv_heads=self.num_kv_heads,
263
- cache_config=cache_config,
264
- quant_config=quant_config,
265
- prefix=f"{prefix}.attn",
266
- attn_type=AttentionType.DECODER)
267
-
268
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
269
- """Input shape: Batch x Time x Channel"""
270
-
271
- qkv, _ = self.qkv_proj(hidden_states)
272
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
273
-
274
- attn_output = self.attn(q, k, v)
275
-
276
- output, _ = self.out_proj(attn_output)
277
- return output
278
-
279
-
280
-class BartCrossAttention(nn.Module):
281
-
282
- def __init__(
283
- self,
284
- embed_dim: int,
285
- num_heads: int,
286
- bias: bool = True,
287
- config: Optional[BartConfig] = None,
288
- cache_config: Optional[CacheConfig] = None,
289
- quant_config: Optional[QuantizationConfig] = None,
290
- prefix: str = "",
291
- ):
292
- super().__init__()
293
- self.d_model = config.d_model
294
- self.embed_dim = embed_dim
295
- self.total_num_heads = num_heads
296
- self.total_num_kv_heads = self.total_num_heads
297
- self.head_dim = embed_dim // num_heads
298
- self.config = config
299
-
300
- if (self.head_dim * num_heads) != self.embed_dim:
301
- raise ValueError(f"embed_dim must be divisible by num_heads "
302
- f"(got `embed_dim`: {self.embed_dim}"
303
- f" and `num_heads`: {num_heads}).")
304
- self.scaling = self.head_dim**-0.5
305
-
306
- # TP sharding sizes is accounted for within "*Parallel" layers.
307
- self.qkv_proj = QKVCrossParallelLinear(self.d_model,
308
- self.d_model //
309
- self.total_num_heads,
310
- self.total_num_heads,
311
- self.total_num_kv_heads,
312
- bias,
313
- quant_config=quant_config)
314
-
315
- self.out_proj = RowParallelLinear(
316
- embed_dim,
317
- embed_dim,
318
- bias=bias,
319
- quant_config=quant_config,
320
- )
321
-
322
- tp_world_size = get_tensor_model_parallel_world_size()
323
- assert self.total_num_heads % tp_world_size == 0
324
- self.num_heads = self.total_num_heads // tp_world_size
325
-
326
- if self.total_num_kv_heads >= tp_world_size:
327
- # Number of KV heads is greater than TP size, so we partition
328
- # the KV heads across multiple tensor parallel GPUs.
329
- assert self.total_num_kv_heads % tp_world_size == 0
330
- else:
331
- # Number of KV heads is less than TP size, so we replicate
332
- # the KV heads across multiple tensor parallel GPUs.
333
- assert tp_world_size % self.total_num_kv_heads == 0
334
- self.num_kv_heads = self.num_heads # No GQA in bart
335
- self.attn = Attention(self.num_heads,
336
- self.head_dim,
337
- self.scaling,
338
- num_kv_heads=self.num_kv_heads,
339
- cache_config=cache_config,
340
- quant_config=quant_config,
341
- prefix=f"{prefix}.attn",
342
- attn_type=AttentionType.ENCODER_DECODER)
343
-
344
- def forward(
345
- self,
346
- decoder_hidden_states: torch.Tensor,
347
- encoder_hidden_states: Optional[torch.Tensor] = None,
348
- ) -> torch.Tensor:
349
- """Input shape: Batch x Time x Channel"""
350
-
351
- q, k, v = self.qkv_proj(decoder_hidden_states, encoder_hidden_states)
352
-
353
- attn_output = self.attn(q, k, v)
354
-
355
- output, _ = self.out_proj(attn_output)
356
- return output
357
-
358
-
359
-class BartEncoderLayer(nn.Module):
360
-
361
- def __init__(
362
- self,
363
- config: BartConfig,
364
- cache_config: Optional[CacheConfig] = None,
365
- quant_config: Optional[QuantizationConfig] = None,
366
- prefix: str = "",
367
- ):
368
- super().__init__()
369
- self.embed_dim = config.d_model
370
-
371
- self.self_attn = BartEncoderAttention(
372
- embed_dim=self.embed_dim,
373
- num_heads=config.encoder_attention_heads,
374
- config=config,
375
- cache_config=cache_config,
376
- quant_config=quant_config,
377
- prefix=f"{prefix}.self_attn",
378
- )
379
- self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
380
- self.activation_fn = get_act_fn(config.activation_function)
381
-
382
- ffn_hidden_size = self.embed_dim
383
- ffn_intermediate_size = config.encoder_ffn_dim
384
- ffn_has_bias = True
385
- self.fc1 = ColumnParallelLinear(
386
- ffn_hidden_size,
387
- ffn_intermediate_size,
388
- bias=ffn_has_bias,
389
- quant_config=quant_config,
390
- )
391
- self.act = get_act_fn("gelu")
392
- self.fc2 = RowParallelLinear(
393
- ffn_intermediate_size,
394
- ffn_hidden_size,
395
- bias=ffn_has_bias,
396
- quant_config=quant_config,
397
- )
398
-
399
- self.final_layer_norm = nn.LayerNorm(self.embed_dim)
400
-
401
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
402
- r"""
403
- Args:
404
- hidden_states: torch.Tensor of *encoder* input embeddings.
405
- Returns:
406
- Encoder layer output torch.Tensor
407
- """
408
- residual = hidden_states
409
- hidden_states = self.self_attn(hidden_states=hidden_states)
410
-
411
- hidden_states = residual + hidden_states
412
- hidden_states = self.self_attn_layer_norm(hidden_states)
413
-
414
- residual = hidden_states
415
- fc1_out, _ = self.fc1(hidden_states)
416
- hidden_states = self.activation_fn(fc1_out)
417
-
418
- hidden_states, _ = self.fc2(hidden_states)
419
-
420
- hidden_states = residual + hidden_states
421
- hidden_states = self.final_layer_norm(hidden_states)
422
-
423
- if hidden_states.dtype == torch.float16 and (
424
- torch.isinf(hidden_states).any()
425
- or torch.isnan(hidden_states).any()):
426
- hidden_states = cast_overflow_tensors(hidden_states)
427
-
428
- return hidden_states
429
-
430
-
431
-class BartDecoderLayer(nn.Module):
432
-
433
- def __init__(
434
- self,
435
- config: BartConfig,
436
- cache_config: Optional[CacheConfig] = None,
437
- quant_config: Optional[QuantizationConfig] = None,
438
- prefix: str = "",
439
- ):
440
- super().__init__()
441
- self.embed_dim = config.d_model
442
-
443
- self.self_attn = BartDecoderSelfAttention(
444
- embed_dim=self.embed_dim,
445
- num_heads=config.decoder_attention_heads,
446
- config=config,
447
- cache_config=cache_config,
448
- quant_config=quant_config,
449
- prefix=f"{prefix}.self_attn",
450
- )
451
- self.activation_fn = get_act_fn(config.activation_function)
452
-
453
- self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
454
- '''
455
- afeldman-nm: personally I would call this "cross-attention",
456
- however I left the name as "encoder_attn" to maintain consistency
457
- with the name of the pretrained weights.
458
- '''
459
- self.encoder_attn = BartCrossAttention(
460
- self.embed_dim,
461
- config.decoder_attention_heads,
462
- config=config,
463
- prefix=f"{prefix}.encoder_attn",
464
- )
465
- self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
466
-
467
- ffn_hidden_size = self.embed_dim
468
- ffn_intermediate_size = config.encoder_ffn_dim
469
- ffn_has_bias = True
470
- self.fc1 = ColumnParallelLinear(
471
- ffn_hidden_size,
472
- ffn_intermediate_size,
473
- bias=ffn_has_bias,
474
- quant_config=quant_config,
475
- )
476
- self.fc2 = RowParallelLinear(
477
- ffn_intermediate_size,
478
- ffn_hidden_size,
479
- bias=ffn_has_bias,
480
- quant_config=quant_config,
481
- )
482
-
483
- self.final_layer_norm = nn.LayerNorm(self.embed_dim)
484
-
485
- def forward(
486
- self,
487
- decoder_hidden_states: torch.Tensor,
488
- encoder_hidden_states: Optional[torch.Tensor] = None,
489
- ) -> torch.Tensor:
490
- r"""
491
- Args:
492
- decoder_hidden_states: torch.Tensor of *decoder* input embeddings.
493
- encoder_hidden_states: torch.Tensor of *encoder* input embeddings.
494
- Returns:
495
- Decoder layer output torch.Tensor
496
- """
497
- residual = decoder_hidden_states
498
-
499
- # Self Attention
500
- hidden_states = self.self_attn(hidden_states=decoder_hidden_states)
501
-
502
- hidden_states = residual + hidden_states
503
- hidden_states = self.self_attn_layer_norm(hidden_states)
504
-
505
- # Cross-Attention Block
506
-
507
- residual = hidden_states
508
-
509
- hidden_states = self.encoder_attn(
510
- decoder_hidden_states=hidden_states,
511
- encoder_hidden_states=encoder_hidden_states,
512
- )
513
-
514
- hidden_states = residual + hidden_states
515
- hidden_states = self.encoder_attn_layer_norm(hidden_states)
516
-
517
- # Fully Connected
518
- residual = hidden_states
519
- fc1_out, _ = self.fc1(hidden_states)
520
- hidden_states = self.activation_fn(fc1_out)
521
-
522
- hidden_states, _ = self.fc2(hidden_states)
523
-
524
- hidden_states = residual + hidden_states
525
- hidden_states = self.final_layer_norm(hidden_states)
526
-
527
- return hidden_states
528
-
529
-
530
-class BartEncoder(nn.Module):
531
- """
532
- Transformer encoder consisting of *config.encoder_layers*
533
- self attention layers. Each layer is a [`BartEncoderLayer`].
534
- Args:
535
- config: BartConfig
536
- embed_tokens (nn.Embedding): output embedding
537
- """
538
-
539
- def __init__(self,
540
- config: BartConfig,
541
- cache_config: Optional[CacheConfig] = None,
542
- quant_config: Optional[QuantizationConfig] = None,
543
- lora_config: Optional[LoRAConfig] = None,
544
- embed_tokens: Optional[nn.Embedding] = None,
545
- prefix: str = ""):
546
- super().__init__()
547
-
548
- self.cache_config = cache_config
549
- self.quant_config = quant_config
550
- self.lora_config = lora_config
551
- embed_dim = config.d_model
552
- self.max_source_positions = config.max_position_embeddings
553
- embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
554
-
555
- self.embed_tokens = BartScaledWordEmbedding(config.vocab_size,
556
- embed_dim,
557
- embed_scale=embed_scale)
558
-
559
- if embed_tokens is not None:
560
- self.embed_tokens.weight = embed_tokens.weight
561
-
562
- self.embed_positions = BartLearnedPositionalEmbedding(
563
- config.max_position_embeddings,
564
- embed_dim,
565
- )
566
- self.layers = nn.ModuleList([
567
- BartEncoderLayer(config,
568
- cache_config,
569
- quant_config,
570
- prefix=f"{prefix}.layers.{layer_idx}")
571
- for layer_idx in range(config.encoder_layers)
572
- ])
573
-
574
- self.layernorm_embedding = nn.LayerNorm(embed_dim)
575
-
576
- def forward(
577
- self,
578
- input_ids: torch.Tensor,
579
- positions: torch.Tensor,
580
- inputs_embeds: Optional[torch.Tensor] = None,
581
- ) -> torch.Tensor:
582
- r"""
583
- Args:
584
- input_ids: Indices of *encoder* input sequence tokens in the
585
- vocabulary.
586
- Padding will be ignored by default should you provide it.
587
- positions: Positions of *encoder* input sequence tokens.
588
- Returns:
589
- Decoder output torch.Tensor
590
- """
591
- # retrieve input_ids and inputs_embeds
592
- if inputs_embeds is None:
593
- inputs_embeds = self.embed_tokens(input_ids)
594
-
595
- embed_pos = self.embed_positions(positions)
596
- embed_pos = embed_pos.to(inputs_embeds.device)
597
-
598
- hidden_states = inputs_embeds + embed_pos
599
- hidden_states = self.layernorm_embedding(hidden_states)
600
-
601
- for encoder_layer in self.layers:
602
- hidden_states = encoder_layer(hidden_states=hidden_states)
603
-
604
- return hidden_states
605
-
606
-
607
-class BartDecoder(nn.Module):
608
- """
609
- Transformer decoder consisting of *config.decoder_layers* layers.
610
- Each layer is a [`BartDecoderLayer`]
611
- Args:
612
- config: BartConfig
613
- embed_tokens (nn.Embedding): output embedding
614
- """
615
-
616
- def __init__(
617
- self,
618
- config: BartConfig,
619
- cache_config: Optional[CacheConfig] = None,
620
- quant_config: Optional[QuantizationConfig] = None,
621
- lora_config: Optional[LoRAConfig] = None,
622
- embed_tokens: Optional[nn.Embedding] = None,
623
- prefix: str = "",
624
- ):
625
- super().__init__()
626
- self.cache_config = cache_config
627
- self.quant_config = quant_config
628
- self.lora_config = lora_config
629
- self.max_target_positions = config.max_position_embeddings
630
- embed_scale = math.sqrt(
631
- config.d_model) if config.scale_embedding else 1.0
632
-
633
- self.embed_tokens = BartScaledWordEmbedding(config.vocab_size,
634
- config.d_model,
635
- embed_scale=embed_scale)
636
-
637
- if embed_tokens is not None:
638
- self.embed_tokens.weight = embed_tokens.weight
639
-
640
- self.embed_positions = BartLearnedPositionalEmbedding(
641
- config.max_position_embeddings,
642
- config.d_model,
643
- )
644
-
645
- self.layers = nn.ModuleList(
646
- [BartDecoderLayer(config,cache_config,quant_config,
647
- prefix=f"{prefix}.layers.{layer_idx}") \
648
- for layer_idx in range(config.decoder_layers)])
649
-
650
- self.layernorm_embedding = nn.LayerNorm(config.d_model)
651
-
652
- def forward(
653
- self,
654
- decoder_input_ids: torch.Tensor,
655
- decoder_positions: torch.Tensor,
656
- encoder_hidden_states: Optional[torch.Tensor],
657
- inputs_embeds: Optional[torch.Tensor] = None,
658
- ) -> torch.Tensor:
659
- r"""
660
- Args:
661
- decoder_input_ids: Indices of *decoder* input sequence tokens
662
- in the vocabulary.
663
- Padding will be ignored by default should you provide it.
664
- decoder_positions: Positions of *decoder* input sequence tokens.
665
- encoder_hidden_states: Tensor of encoder output embeddings.
666
- Returns:
667
- Decoder output torch.Tensor
668
- """
669
- if inputs_embeds is None:
670
- inputs_embeds = self.embed_tokens(decoder_input_ids)
671
- else:
672
- decoder_positions = inputs_embeds[:, -1]
673
-
674
- # embed positions
675
- embed_pos = self.embed_positions(decoder_positions)
676
- embed_pos = embed_pos.to(inputs_embeds.device)
677
-
678
- hidden_states = inputs_embeds + embed_pos
679
- hidden_states = self.layernorm_embedding(hidden_states)
680
-
681
- # decoder layers
682
-
683
- for decoder_layer in self.layers:
684
- hidden_states = decoder_layer(
685
- decoder_hidden_states=hidden_states,
686
- encoder_hidden_states=encoder_hidden_states,
687
- )
688
-
689
- return hidden_states
690
-
691
-
692
-class BartModel(nn.Module, SupportsQuant):
693
- _tied_weights_keys = [
694
- "encoder.embed_tokens.weight",
695
- "decoder.embed_tokens.weight",
696
- ]
697
-
698
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
699
- super().__init__()
700
-
701
- config = vllm_config.model_config.hf_config
702
- cache_config = vllm_config.cache_config
703
- quant_config = vllm_config.quant_config
704
- lora_config = vllm_config.lora_config
705
-
706
- self.config = config
707
-
708
- lora_vocab = (lora_config.lora_extra_vocab_size *
709
- (lora_config.max_loras or 1)) if lora_config else 0
710
- self.vocab_size = config.vocab_size + lora_vocab
711
- self.org_vocab_size = config.vocab_size
712
-
713
- self.encoder = BartEncoder(config,
714
- cache_config,
715
- quant_config=quant_config,
716
- prefix=f"{prefix}.encoder")
717
- self.decoder = BartDecoder(config,
718
- cache_config,
719
- quant_config=quant_config,
720
- prefix=f"{prefix}.decoder")
721
-
722
- def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
723
- encoder_input_ids: torch.Tensor,
724
- encoder_positions: torch.Tensor) -> torch.Tensor:
725
- r"""
726
- Args:
727
- input_ids: Indices of *decoder* input sequence tokens
728
- in the vocabulary.
729
- Padding will be ignored by default should you provide it.
730
- positions: Positions of *decoder* input sequence tokens.
731
- encoder_input_ids: Indices of *encoder* input sequence tokens
732
- in the vocabulary.
733
- encoder_positions: Positions of *encoder* input sequence tokens.
734
- Returns:
735
- Model output torch.Tensor
736
- """
737
-
738
- encoder_hidden_states = None
739
-
740
- if encoder_input_ids.numel() > 0:
741
- # Run encoder attention if a non-zero number of encoder tokens
742
- # are provided as input
743
- encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
744
- positions=encoder_positions)
745
-
746
- # decoder outputs consists of
747
- # (dec_features, past_key_value, dec_hidden, dec_attn)
748
- decoder_outputs = self.decoder(
749
- decoder_input_ids=input_ids,
750
- decoder_positions=positions,
751
- encoder_hidden_states=encoder_hidden_states)
752
-
753
- return decoder_outputs
754
-
755
- def load_weights(self, weights: Iterable[tuple[str,
756
- torch.Tensor]]) -> set[str]:
757
- stacked_params_mapping = [
758
- # (param_name, shard_name, shard_id)
759
- ("qkv_proj", "q_proj", "q"),
760
- ("qkv_proj", "k_proj", "k"),
761
- ("qkv_proj", "v_proj", "v"),
762
- ]
763
-
764
- other_weights = []
765
- loaded_stacked_params = []
766
- model_params_dict = dict(self.named_parameters())
767
-
768
- for name, loaded_weight in weights:
769
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
770
- if weight_name not in name:
771
- continue
772
- name = name.replace(weight_name, param_name)
773
- if name not in model_params_dict:
774
- continue
775
- param = model_params_dict[name]
776
- weight_loader = param.weight_loader
777
- weight_loader(param, loaded_weight, shard_id)
778
- loaded_stacked_params.append(name)
779
- break
780
- else:
781
- if name in model_params_dict:
782
- other_weights.append((name, loaded_weight))
783
-
784
- loader = AutoWeightsLoader(self)
785
- loaded_params = loader.load_weights(other_weights)
786
- loaded_params.update(loaded_stacked_params)
787
- return loaded_params
788
-
789
-
790
-class BartForConditionalGeneration(nn.Module, SupportsV0Only, SupportsQuant):
791
- hf_to_vllm_mapper = WeightsMapper(
792
- orig_to_new_prefix={
793
- "decoder.": "model.decoder.",
794
- "encoder.": "model.encoder.",
795
- "shared.": "model.shared."
796
- },
797
- orig_to_new_substr={
798
- "beta": "bias",
799
- "gamma": "weight",
800
- "LayerNorm": "layernorm",
801
- },
802
- )
803
-
804
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
805
-
806
- super().__init__()
807
- config = vllm_config.model_config.hf_config
808
- lora_config = vllm_config.lora_config
809
- # currently all existing BART models have `tie_word_embeddings` enabled
810
- assert config.tie_word_embeddings
811
- self.config = config
812
- self.model = BartModel(vllm_config=vllm_config,
813
- prefix=maybe_prefix(prefix, "model"))
814
-
815
- self.unpadded_vocab_size = config.vocab_size
816
- if lora_config:
817
- self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
818
-
819
- embed_scale = math.sqrt(
820
- config.d_model) if config.scale_embedding else 1.0
821
-
822
- self.lm_head = BartParallelLMHead(config.vocab_size,
823
- config.d_model,
824
- embed_scale=embed_scale)
825
- self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
826
- config.vocab_size)
827
-
828
- def forward(
829
- self,
830
- input_ids: torch.Tensor,
831
- positions: torch.Tensor,
832
- intermediate_tensors: Optional[IntermediateTensors] = None,
833
- *,
834
- encoder_input_ids: torch.Tensor,
835
- encoder_positions: torch.Tensor,
836
- **kwargs,
837
- ) -> torch.Tensor:
838
- r"""
839
- Args:
840
- input_ids: torch.Tensor of *decoder* input token ids.
841
- positions: torch.Tensor of *decoder* position indices.
842
- encoder_input_ids: torch.Tensor of *encoder* input token ids.
843
- encoder_positions: torch.Tensor of *encoder* position indices.
844
- Returns:
845
- Output torch.Tensor
846
- """
847
- return self.model(input_ids, positions, encoder_input_ids,
848
- encoder_positions)
849
-
850
- def compute_logits(
851
- self,
852
- hidden_states: torch.Tensor,
853
- sampling_metadata: SamplingMetadata,
854
- ) -> Optional[torch.Tensor]:
855
- logits = self.logits_processor(self.lm_head, hidden_states,
856
- sampling_metadata)
857
- return logits
858
-
859
- def load_weights(self, weights: Iterable[tuple[str,
860
- torch.Tensor]]) -> set[str]:
861
- weights_tuple_list = list(weights)
862
-
863
- shared_embedding_weight = None
864
- for name, loaded_weight in weights_tuple_list:
865
- if ('shared.weight' in name
866
- or 'encoder.embed_tokens.weight' in name
867
- or 'decoder.embed_tokens.weight' in name
868
- or 'lm_head.weight' in name):
869
- assert shared_embedding_weight is None, (
870
- "Conflicting embedding weights.")
871
- shared_embedding_weight = loaded_weight
872
-
873
- loader = AutoWeightsLoader(
874
- self,
875
- skip_prefixes=(["cls.", "pooler."]),
876
- )
877
- loaded_params = loader.load_weights(weights_tuple_list,
878
- mapper=self.hf_to_vllm_mapper)
879
-
880
- if shared_embedding_weight is not None:
881
- weight_loader = getattr(self.lm_head.weight, "weight_loader",
882
- default_weight_loader)
883
- weight_loader(self.lm_head.weight, shared_embedding_weight)
884
-
885
- self.model.encoder.embed_tokens.weight = self.lm_head.weight
886
- self.model.decoder.embed_tokens.weight = self.lm_head.weight
887
- loaded_params.update({
888
- 'model.encoder.embed_tokens.weight', 'lm_head.weight',
889
- 'model.decoder.embed_tokens.weight'
890
- })
891
-
892
- return loaded_params
893
-
894
-
895
-class MBartEncoderLayer(BartEncoderLayer):
896
-
897
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
898
- r"""
899
- Args:
900
- hidden_states: torch.Tensor of *encoder* input embeddings.
901
- Returns:
902
- Encoder layer output torch.Tensor
903
- """
904
- residual = hidden_states
905
- hidden_states = self.self_attn_layer_norm(hidden_states)
906
- hidden_states = self.self_attn(hidden_states=hidden_states)
907
-
908
- hidden_states = residual + hidden_states
909
-
910
- residual = hidden_states
911
- hidden_states = self.final_layer_norm(hidden_states)
912
- fc1_out, _ = self.fc1(hidden_states)
913
- hidden_states = self.activation_fn(fc1_out)
914
-
915
- hidden_states, _ = self.fc2(hidden_states)
916
-
917
- hidden_states = residual + hidden_states
918
-
919
- if hidden_states.dtype == torch.float16 and (
920
- torch.isinf(hidden_states).any()
921
- or torch.isnan(hidden_states).any()):
922
- hidden_states = cast_overflow_tensors(hidden_states)
923
-
924
- return hidden_states
925
-
926
-
927
-class MBartDecoderLayer(BartDecoderLayer):
928
-
929
- def forward(
930
- self,
931
- decoder_hidden_states: torch.Tensor,
932
- encoder_hidden_states: Optional[torch.Tensor] = None,
933
- ) -> torch.Tensor:
934
- residual = decoder_hidden_states
935
- hidden_states = self.self_attn_layer_norm(decoder_hidden_states)
936
-
937
- # Self Attention
938
- hidden_states = self.self_attn(hidden_states=hidden_states)
939
-
940
- hidden_states = residual + hidden_states
941
-
942
- # Cross-Attention Block
943
-
944
- residual = hidden_states
945
- hidden_states = self.encoder_attn_layer_norm(hidden_states)
946
-
947
- hidden_states = self.encoder_attn(
948
- decoder_hidden_states=hidden_states,
949
- encoder_hidden_states=encoder_hidden_states,
950
- )
951
-
952
- hidden_states = residual + hidden_states
953
-
954
- # Fully Connected
955
- residual = hidden_states
956
- hidden_states = self.final_layer_norm(hidden_states)
957
- fc1_out, _ = self.fc1(hidden_states)
958
- hidden_states = self.activation_fn(fc1_out)
959
-
960
- hidden_states, _ = self.fc2(hidden_states)
961
-
962
- hidden_states = residual + hidden_states
963
-
964
- return hidden_states
965
-
966
-
967
-class MBartEncoder(nn.Module):
968
- """
969
- Transformer encoder consisting of *config.encoder_layers*
970
- self attention layers. Each layer is a [`BartEncoderLayer`].
971
- Args:
972
- config: BartConfig
973
- embed_tokens (nn.Embedding): output embedding
974
- """
975
-
976
- def __init__(self,
977
- config: BartConfig,
978
- cache_config: Optional[CacheConfig] = None,
979
- quant_config: Optional[QuantizationConfig] = None,
980
- lora_config: Optional[LoRAConfig] = None,
981
- embed_tokens: Optional[nn.Embedding] = None,
982
- prefix: str = ""):
983
- super().__init__()
984
-
985
- self.cache_config = cache_config
986
- self.quant_config = quant_config
987
- self.lora_config = lora_config
988
- embed_dim = config.d_model
989
- self.max_source_positions = config.max_position_embeddings
990
- embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
991
-
992
- self.embed_tokens = BartScaledWordEmbedding(config.vocab_size,
993
- embed_dim,
994
- embed_scale=embed_scale)
995
-
996
- if embed_tokens is not None:
997
- self.embed_tokens.weight = embed_tokens.weight
998
-
999
- self.embed_positions = BartLearnedPositionalEmbedding(
1000
- config.max_position_embeddings,
1001
- embed_dim,
1002
- )
1003
- self.layers = nn.ModuleList([
1004
- MBartEncoderLayer(config,
1005
- cache_config,
1006
- quant_config,
1007
- prefix=f"{prefix}.layers.{layer_idx}")
1008
- for layer_idx in range(config.encoder_layers)
1009
- ])
1010
-
1011
- self.layernorm_embedding = nn.LayerNorm(embed_dim)
1012
- self.layer_norm = nn.LayerNorm(config.d_model) # 改动
1013
-
1014
- def forward(
1015
- self,
1016
- input_ids: torch.Tensor,
1017
- positions: torch.Tensor,
1018
- inputs_embeds: Optional[torch.Tensor] = None,
1019
- ) -> torch.Tensor:
1020
- r"""
1021
- Args:
1022
- input_ids: Indices of *encoder* input sequence tokens in the
1023
- vocabulary.
1024
- Padding will be ignored by default should you provide it.
1025
- positions: Positions of *encoder* input sequence tokens.
1026
- Returns:
1027
- Decoder output torch.Tensor
1028
- """
1029
- # retrieve input_ids and inputs_embeds
1030
- if inputs_embeds is None:
1031
- inputs_embeds = self.embed_tokens(input_ids)
1032
-
1033
- embed_pos = self.embed_positions(positions)
1034
- embed_pos = embed_pos.to(inputs_embeds.device)
1035
-
1036
- hidden_states = inputs_embeds + embed_pos
1037
- hidden_states = self.layernorm_embedding(hidden_states)
1038
-
1039
- for encoder_layer in self.layers:
1040
- hidden_states = encoder_layer(hidden_states=hidden_states)
1041
-
1042
- hidden_states = self.layer_norm(hidden_states)
1043
- return hidden_states
1044
-
1045
-
1046
-class MBartDecoder(nn.Module):
1047
- """
1048
- Transformer decoder consisting of *config.decoder_layers* layers.
1049
- Each layer is a [`BartDecoderLayer`]
1050
- Args:
1051
- config: BartConfig
1052
- embed_tokens (nn.Embedding): output embedding
1053
- """
1054
-
1055
- def __init__(
1056
- self,
1057
- config: BartConfig,
1058
- cache_config: Optional[CacheConfig] = None,
1059
- quant_config: Optional[QuantizationConfig] = None,
1060
- lora_config: Optional[LoRAConfig] = None,
1061
- embed_tokens: Optional[nn.Embedding] = None,
1062
- prefix: str = "",
1063
- ):
1064
- super().__init__()
1065
- self.cache_config = cache_config
1066
- self.quant_config = quant_config
1067
- self.lora_config = lora_config
1068
- self.max_target_positions = config.max_position_embeddings
1069
- embed_scale = math.sqrt(
1070
- config.d_model) if config.scale_embedding else 1.0
1071
-
1072
- self.embed_tokens = BartScaledWordEmbedding(config.vocab_size,
1073
- config.d_model,
1074
- embed_scale=embed_scale)
1075
-
1076
- if embed_tokens is not None:
1077
- self.embed_tokens.weight = embed_tokens.weight
1078
-
1079
- self.embed_positions = BartLearnedPositionalEmbedding(
1080
- config.max_position_embeddings,
1081
- config.d_model,
1082
- )
1083
-
1084
- self.layers = nn.ModuleList(
1085
- [MBartDecoderLayer(config, cache_config, quant_config,
1086
- prefix=f"{prefix}.layers.{layer_idx}") \
1087
- for layer_idx in range(config.decoder_layers)])
1088
-
1089
- self.layernorm_embedding = nn.LayerNorm(config.d_model)
1090
- self.layer_norm = nn.LayerNorm(config.d_model)
1091
-
1092
- def forward(
1093
- self,
1094
- decoder_input_ids: torch.Tensor,
1095
- decoder_positions: torch.Tensor,
1096
- encoder_hidden_states: Optional[torch.Tensor],
1097
- inputs_embeds: Optional[torch.Tensor] = None,
1098
- ) -> torch.Tensor:
1099
- r"""
1100
- Args:
1101
- decoder_input_ids: Indices of *decoder* input sequence tokens
1102
- in the vocabulary.
1103
- Padding will be ignored by default should you provide it.
1104
- decoder_positions: Positions of *decoder* input sequence tokens.
1105
- encoder_hidden_states: Tensor of encoder output embeddings.
1106
- Returns:
1107
- Decoder output torch.Tensor
1108
- """
1109
- if inputs_embeds is None:
1110
- inputs_embeds = self.embed_tokens(decoder_input_ids)
1111
- else:
1112
- decoder_positions = inputs_embeds[:, -1]
1113
-
1114
- # embed positions
1115
- embed_pos = self.embed_positions(decoder_positions)
1116
- embed_pos = embed_pos.to(inputs_embeds.device)
1117
-
1118
- hidden_states = inputs_embeds + embed_pos
1119
- hidden_states = self.layernorm_embedding(hidden_states)
1120
-
1121
- # decoder layers
1122
-
1123
- for decoder_layer in self.layers:
1124
- hidden_states = decoder_layer(
1125
- decoder_hidden_states=hidden_states,
1126
- encoder_hidden_states=encoder_hidden_states,
1127
- )
1128
-
1129
- hidden_states = self.layer_norm(hidden_states)
1130
- return hidden_states
1131
-
1132
-
1133
-class MBartModel(nn.Module, SupportsQuant):
1134
- _tied_weights_keys = [
1135
- "encoder.embed_tokens.weight", "decoder.embed_tokens.weight"
1136
- ]
1137
-
1138
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1139
- super().__init__()
1140
-
1141
- config = vllm_config.model_config.hf_config
1142
- cache_config = vllm_config.cache_config
1143
- quant_config = vllm_config.quant_config
1144
- lora_config = vllm_config.lora_config
1145
-
1146
- self.config = config
1147
-
1148
- lora_vocab = (lora_config.lora_extra_vocab_size *
1149
- (lora_config.max_loras or 1)) if lora_config else 0
1150
- self.vocab_size = config.vocab_size + lora_vocab
1151
- self.org_vocab_size = config.vocab_size
1152
-
1153
- self.encoder = MBartEncoder(config,
1154
- cache_config,
1155
- quant_config=quant_config,
1156
- prefix=f"{prefix}.encoder")
1157
- self.decoder = MBartDecoder(config,
1158
- cache_config,
1159
- quant_config=quant_config,
1160
- prefix=f"{prefix}.decoder")
1161
-
1162
- def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
1163
- encoder_input_ids: torch.Tensor,
1164
- encoder_positions: torch.Tensor) -> torch.Tensor:
1165
- r"""
1166
- Args:
1167
- input_ids: Indices of *decoder* input sequence tokens
1168
- in the vocabulary.
1169
- Padding will be ignored by default should you provide it.
1170
- positions: Positions of *decoder* input sequence tokens.
1171
- encoder_input_ids: Indices of *encoder* input sequence tokens
1172
- in the vocabulary.
1173
- encoder_positions: Positions of *encoder* input sequence tokens.
1174
- Returns:
1175
- Model output torch.Tensor
1176
- """
1177
-
1178
- encoder_hidden_states = None
1179
-
1180
- if encoder_input_ids.numel() > 0:
1181
- # Run encoder attention if a non-zero number of encoder tokens
1182
- # are provided as input
1183
- encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
1184
- positions=encoder_positions)
1185
-
1186
- # decoder outputs consists of
1187
- # (dec_features, past_key_value, dec_hidden, dec_attn)
1188
- decoder_outputs = self.decoder(
1189
- decoder_input_ids=input_ids,
1190
- decoder_positions=positions,
1191
- encoder_hidden_states=encoder_hidden_states)
1192
-
1193
- return decoder_outputs
1194
-
1195
-
1196
-class MBartForConditionalGeneration(nn.Module, SupportsV0Only, SupportsQuant):
1197
- base_model_prefix = "model"
1198
-
1199
- hf_to_vllm_mapper = WeightsMapper(
1200
- orig_to_new_prefix={
1201
- "decoder.": "model.decoder.",
1202
- "encoder.": "model.encoder.",
1203
- "shared.": "model.shared."
1204
- },
1205
- orig_to_new_substr={
1206
- "beta": "bias",
1207
- "gamma": "weight",
1208
- "LayerNorm": "layernorm",
1209
- },
1210
- )
1211
-
1212
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1213
- super().__init__()
1214
- config = vllm_config.model_config.hf_config
1215
- lora_config = vllm_config.lora_config
1216
- assert config.tie_word_embeddings
1217
- self.config = config
1218
- self.model = MBartModel(vllm_config=vllm_config,
1219
- prefix=maybe_prefix(prefix, "model"))
1220
-
1221
- self.unpadded_vocab_size = config.vocab_size
1222
- if lora_config:
1223
- self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
1224
-
1225
- embed_scale = math.sqrt(
1226
- config.d_model) if config.scale_embedding else 1.0
1227
-
1228
- self.lm_head = BartParallelLMHead(config.vocab_size,
1229
- config.d_model,
1230
- embed_scale=embed_scale)
1231
-
1232
- self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
1233
- config.vocab_size)
1234
-
1235
- def forward(
1236
- self,
1237
- input_ids: torch.Tensor,
1238
- positions: torch.Tensor,
1239
- intermediate_tensors: Optional[IntermediateTensors] = None,
1240
- *,
1241
- encoder_input_ids: torch.Tensor,
1242
- encoder_positions: torch.Tensor,
1243
- **kwargs,
1244
- ) -> torch.Tensor:
1245
- return self.model(input_ids, positions, encoder_input_ids,
1246
- encoder_positions)
1247
-
1248
- def compute_logits(
1249
- self,
1250
- hidden_states: torch.Tensor,
1251
- sampling_metadata: SamplingMetadata,
1252
- ) -> Optional[torch.Tensor]:
1253
- logits = self.logits_processor(self.lm_head, hidden_states,
1254
- sampling_metadata)
1255
- return logits
1256
-
1257
- def load_weights(self, weights: Iterable[tuple[str,
1258
- torch.Tensor]]) -> set[str]:
1259
- stacked_params_mapping = [
1260
- ("qkv_proj", "q_proj", "q"),
1261
- ("qkv_proj", "k_proj", "k"),
1262
- ("qkv_proj", "v_proj", "v"),
1263
- ]
1264
- model_params_dict = dict(self.named_parameters())
1265
- loaded_params = set()
1266
- remaining_weights = []
1267
- shared_embedding_weight = None
1268
-
1269
- for name, loaded_weight in weights:
1270
- if any(skip in name
1271
- for skip in ["cls.", "pooler.", "final_logits_bias"]):
1272
- continue
1273
- if any(embed_name in name for embed_name in [
1274
- 'shared.weight', 'encoder.embed_tokens.weight',
1275
- 'decoder.embed_tokens.weight'
1276
- ]):
1277
- if shared_embedding_weight is None:
1278
- shared_embedding_weight = loaded_weight
1279
- continue
1280
- is_stacked = False
1281
- for param_name, weight_name, shard_id in stacked_params_mapping:
1282
- if weight_name not in name:
1283
- continue
1284
- vllm_name = name
1285
- for src, dst in self.hf_to_vllm_mapper.orig_to_new_substr.items(
1286
- ):
1287
- vllm_name = vllm_name.replace(src, dst)
1288
- for src, dst in self.hf_to_vllm_mapper.orig_to_new_prefix.items(
1289
- ):
1290
- if vllm_name.startswith(src):
1291
- vllm_name = dst + vllm_name[len(src):]
1292
- break
1293
- vllm_name = vllm_name.replace(weight_name, param_name)
1294
- if vllm_name in model_params_dict:
1295
- param = model_params_dict[vllm_name]
1296
- weight_loader = getattr(param, "weight_loader",
1297
- default_weight_loader)
1298
- weight_loader(param, loaded_weight, shard_id)
1299
- loaded_params.add(vllm_name)
1300
- is_stacked = True
1301
- break
1302
- if not is_stacked:
1303
- remaining_weights.append((name, loaded_weight))
1304
- loader = AutoWeightsLoader(self, skip_prefixes=["cls.", "pooler."])
1305
- auto_loaded_params = loader.load_weights(remaining_weights,
1306
- mapper=self.hf_to_vllm_mapper)
1307
- loaded_params.update(auto_loaded_params)
1308
- if shared_embedding_weight is not None:
1309
- lm_head_param = self.lm_head.weight
1310
- weight_loader = getattr(lm_head_param, "weight_loader",
1311
- default_weight_loader)
1312
- weight_loader(lm_head_param, shared_embedding_weight)
1313
- self.model.encoder.embed_tokens.weight = self.lm_head.weight
1314
- self.model.decoder.embed_tokens.weight = self.lm_head.weight
1315
- loaded_params.update({
1316
- 'model.encoder.embed_tokens.weight', 'lm_head.weight',
1317
- 'model.decoder.embed_tokens.weight'
1318
- })
1319
- return loaded_params
vllm/model_executor/models/donut.py
deleted
-381
@@ -1,381 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import math
5
-from collections.abc import Iterable, Mapping, Sequence
6
-from typing import Annotated, Literal, Optional, Union
7
-
8
-import torch
9
-import torch.nn as nn
10
-from transformers import BatchFeature, NougatProcessor
11
-
12
-from vllm.config import VllmConfig
13
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15
-from vllm.model_executor.models.bart import BartParallelLMHead, MBartDecoder
16
-from vllm.model_executor.models.interfaces import (MultiModalEmbeddings,
17
- SupportsMultiModal,
18
- SupportsV0Only)
19
-from vllm.model_executor.models.swin import SwinModel
20
-from vllm.model_executor.models.utils import (AutoWeightsLoader,
21
- _flatten_embeddings, flatten_bn)
22
-from vllm.model_executor.sampling_metadata import SamplingMetadata
23
-from vllm.multimodal import MULTIMODAL_REGISTRY
24
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25
- MultiModalKwargsItems)
26
-from vllm.multimodal.parse import MultiModalDataItems
27
-from vllm.multimodal.processing import (BaseProcessingInfo,
28
- EncDecMultiModalProcessor,
29
- PromptIndexTargets, PromptInsertion,
30
- PromptUpdate)
31
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
32
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
33
-
34
-
35
-class MBartDecoderWrapper(nn.Module):
36
-
37
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
38
- super().__init__()
39
- config = vllm_config.model_config.hf_config
40
- cache_config = vllm_config.cache_config
41
- quant_config = vllm_config.quant_config
42
-
43
- self.decoder = MBartDecoder(config,
44
- cache_config,
45
- quant_config=quant_config,
46
- prefix=f"{prefix}.decoder")
47
-
48
- def forward(self, *args, **kwargs):
49
- return self.decoder(*args, **kwargs)
50
-
51
-
52
-class DonutLanguageForConditionalGeneration(nn.Module, SupportsV0Only):
53
-
54
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
55
- super().__init__()
56
-
57
- config = vllm_config.model_config.hf_config
58
-
59
- self.config = config
60
- self.model = MBartDecoderWrapper(vllm_config=vllm_config,
61
- prefix=f"{prefix}.model")
62
- embed_scale = math.sqrt(
63
- config.d_model) if config.scale_embedding else 1.0
64
-
65
- self.vocab_size = config.vocab_size
66
- self.lm_head = BartParallelLMHead(self.vocab_size,
67
- config.d_model,
68
- embed_scale=embed_scale)
69
-
70
- self.logits_processor = LogitsProcessor(self.vocab_size,
71
- config.vocab_size)
72
-
73
- def forward(
74
- self,
75
- input_ids: torch.Tensor,
76
- positions: torch.Tensor,
77
- inputs_embeds: torch.Tensor,
78
- **kwargs,
79
- ) -> torch.Tensor:
80
- r"""
81
- Args:
82
- input_ids: torch.Tensor of *decoder* input token ids.
83
- positions: torch.Tensor of *decoder* position indices.
84
- Returns:
85
- Output torch.Tensor
86
- """
87
-
88
- return self.model(decoder_input_ids=input_ids,
89
- decoder_positions=positions,
90
- encoder_hidden_states=inputs_embeds)
91
-
92
- def compute_logits(
93
- self,
94
- hidden_states: torch.Tensor,
95
- sampling_metadata: SamplingMetadata,
96
- ) -> Optional[torch.Tensor]:
97
- logits = self.logits_processor(self.lm_head, hidden_states,
98
- sampling_metadata)
99
- return logits
100
-
101
- def load_weights(self, weights: Iterable[tuple[str,
102
- torch.Tensor]]) -> set[str]:
103
- stacked_params_mapping = [
104
- # (param_name, shard_name, shard_id)
105
- ("qkv_proj", "q_proj", "q"),
106
- ("qkv_proj", "k_proj", "k"),
107
- ("qkv_proj", "v_proj", "v"),
108
- ]
109
-
110
- params_dict = dict(self.named_parameters())
111
- loaded_params: set[str] = set()
112
- for name, loaded_weight in weights:
113
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
114
- if weight_name not in name:
115
- continue
116
- name = name.replace(weight_name, param_name)
117
- param = params_dict[name]
118
- weight_loader = param.weight_loader
119
- weight_loader(param, loaded_weight, shard_id)
120
- break
121
- else:
122
- if "final_logits_bias" in name:
123
- continue
124
- # if self.config.tie_word_embeddings and "embed_tokens" in name:
125
- # continue
126
- param = params_dict[name]
127
- weight_loader = getattr(param, "weight_loader",
128
- default_weight_loader)
129
- weight_loader(param, loaded_weight)
130
- loaded_params.add(name)
131
- return loaded_params
132
-
133
-
134
-class DonutImagePixelInputs(TensorSchema):
135
- """
136
- Dimensions:
137
- - b: Batch size
138
- - c: Number of channels (3)
139
- - h: Height
140
- - w: Width
141
- """
142
- type: Literal["pixel_values"]
143
- data: Annotated[torch.Tensor, TensorShape("b", 3, "h", "w")]
144
-
145
-
146
-class DonutProcessingInfo(BaseProcessingInfo):
147
-
148
- def get_hf_config(self):
149
- return self.ctx.get_hf_config()
150
-
151
- def get_hf_processor(self):
152
- return self.ctx.get_hf_processor()
153
-
154
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
155
- return {"image": 1}
156
-
157
- def get_num_image_tokens(self) -> int:
158
- return 1
159
-
160
-
161
-class DonutDummyInputsBuilder(BaseDummyInputsBuilder[DonutProcessingInfo]):
162
-
163
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
164
- return ""
165
-
166
- def get_dummy_mm_data(
167
- self,
168
- seq_len: int,
169
- mm_counts: Mapping[str, int],
170
- ) -> MultiModalDataDict:
171
- num_images = mm_counts.get("image", 0)
172
-
173
- target_width, target_height = self.info.get_hf_config(
174
- ).encoder.image_size
175
-
176
- return {
177
- "image":
178
- self._get_dummy_images(width=target_width,
179
- height=target_height,
180
- num_images=num_images)
181
- }
182
-
183
-
184
-class DonutMultiModalProcessor(EncDecMultiModalProcessor[DonutProcessingInfo]):
185
-
186
- def _hf_processor_applies_updates(
187
- self,
188
- prompt_text: str,
189
- mm_items: MultiModalDataItems,
190
- hf_processor_mm_kwargs: Mapping[str, object],
191
- tokenization_kwargs: Mapping[str, object],
192
- ) -> bool:
193
- return False
194
-
195
- def create_encoder_prompt(
196
- self,
197
- prompt: Union[str, list[int]],
198
- mm_data: MultiModalDataDict,
199
- ) -> Union[str, list[int]]:
200
- return prompt
201
-
202
- def create_decoder_prompt(
203
- self,
204
- prompt: Union[str, list[int]],
205
- mm_data: MultiModalDataDict,
206
- ) -> Union[str, list[int]]:
207
- return prompt
208
-
209
- @property
210
- def pad_dummy_encoder_prompt(self) -> bool:
211
- return True
212
-
213
- def _call_hf_processor(
214
- self,
215
- prompt: str,
216
- mm_data: Mapping[str, object],
217
- mm_kwargs: Mapping[str, object],
218
- tok_kwargs: Mapping[str, object],
219
- ) -> BatchFeature:
220
- hf_processor = self.info.get_hf_processor()
221
- if mm_data:
222
- processed_outputs = super()._call_hf_processor(
223
- prompt, mm_data, mm_kwargs, tok_kwargs)
224
- if isinstance(hf_processor, NougatProcessor):
225
- processed_outputs["input_ids"] = processed_outputs["labels"]
226
- else:
227
- tokenizer = hf_processor.tokenizer
228
- processed_outputs = tokenizer(prompt,
229
- add_special_tokens=False,
230
- return_tensors="pt")
231
- return processed_outputs
232
-
233
- def _get_mm_fields_config(
234
- self,
235
- hf_inputs: BatchFeature,
236
- hf_processor_mm_kwargs: Mapping[str, object],
237
- ) -> Mapping[str, MultiModalFieldConfig]:
238
- return dict(pixel_values=MultiModalFieldConfig.batched("image"))
239
-
240
- def _get_prompt_updates(
241
- self,
242
- mm_items: MultiModalDataItems,
243
- hf_processor_mm_kwargs: Mapping[str, object],
244
- out_mm_kwargs: MultiModalKwargsItems,
245
- ) -> Sequence[PromptUpdate]:
246
- hf_processor = self.info.get_hf_processor()
247
- tokenizer = hf_processor.tokenizer
248
- pad_token_id = tokenizer.pad_token_id
249
- num_image_tokens = self.info.get_num_image_tokens()
250
- image_tokens = [pad_token_id] * num_image_tokens
251
-
252
- return [
253
- PromptInsertion(
254
- modality="image",
255
- target=PromptIndexTargets.start(),
256
- insertion=image_tokens,
257
- )
258
- ]
259
-
260
-
261
-@MULTIMODAL_REGISTRY.register_processor(DonutMultiModalProcessor,
262
- info=DonutProcessingInfo,
263
- dummy_inputs=DonutDummyInputsBuilder)
264
-class DonutForConditionalGeneration(nn.Module, SupportsMultiModal,
265
- SupportsV0Only):
266
-
267
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
268
- super().__init__()
269
- config = vllm_config.model_config.hf_config
270
- processor_config = vllm_config.model_config.hf_image_processor_config
271
-
272
- self.config = config
273
- self.vision_config = config.encoder
274
- self.processor_config = processor_config
275
- self.encoder = SwinModel(config=config.encoder)
276
-
277
- self.decoder = DonutLanguageForConditionalGeneration(
278
- vllm_config=vllm_config.with_hf_config(config.decoder),
279
- prefix=f"{prefix}.decoder",
280
- )
281
- self.pad_token_id = config.pad_token_id
282
-
283
- def _parse_and_validate_image_input(self, **kwargs: object):
284
- pixel_values: Optional[Union[list[list[torch.Tensor]],
285
- list[torch.Tensor],
286
- torch.Tensor]] = kwargs.pop(
287
- "pixel_values", None)
288
- image_embeds: Optional[Union[list[list[torch.Tensor]],
289
- list[torch.Tensor],
290
- torch.Tensor]] = kwargs.pop(
291
- "image_embeds", None)
292
-
293
- if pixel_values is None and image_embeds is None:
294
- return None
295
-
296
- if pixel_values is not None and image_embeds is not None:
297
- raise ValueError(
298
- "Both pixel values and image embeds are provided.")
299
-
300
- if pixel_values is not None:
301
- h, w = self.config.encoder.image_size
302
- return DonutImagePixelInputs(type="pixel_values",
303
- data=flatten_bn(pixel_values,
304
- concat=True),
305
- resolve_bindings={
306
- "h": h,
307
- "w": w,
308
- })
309
-
310
- if image_embeds is not None:
311
- raise NotImplementedError
312
-
313
- raise AssertionError("This line should be unreachable.")
314
-
315
- def _process_image_input(
316
- self, image_input: DonutImagePixelInputs) -> torch.Tensor:
317
- assert image_input["type"] == "pixel_values"
318
- pixel_values = image_input["data"]
319
- dtype = next(self.encoder.parameters()).dtype
320
- pixel_values = pixel_values.to(dtype)
321
- return self.encoder(pixel_values)
322
-
323
- def get_language_model(self) -> torch.nn.Module:
324
- return self.decoder
325
-
326
- def get_multimodal_embeddings(
327
- self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
328
- image_input = self._parse_and_validate_image_input(**kwargs)
329
- if image_input is None:
330
- return None
331
- vision_embeddings = self._process_image_input(image_input)
332
- return vision_embeddings
333
-
334
- def get_input_embeddings(
335
- self,
336
- input_ids: torch.Tensor,
337
- multimodal_embeddings: MultiModalEmbeddings,
338
- ) -> torch.Tensor:
339
- return _flatten_embeddings(multimodal_embeddings)
340
-
341
- def forward(
342
- self,
343
- input_ids: torch.Tensor,
344
- positions: torch.Tensor,
345
- *,
346
- encoder_input_ids: torch.Tensor,
347
- encoder_positions: torch.Tensor,
348
- **kwargs,
349
- ) -> torch.Tensor:
350
- r"""
351
- Args:
352
- input_ids: torch.Tensor of *decoder* input token ids.
353
- positions: torch.Tensor of *decoder* position indices.
354
- encoder_input_ids: torch.Tensor of *encoder* input token ids.
355
- encoder_positions: torch.Tensor of *encoder* position indices
356
- Returns:
357
- Output torch.Tensor
358
- """
359
-
360
- inputs_embeds = None
361
- if encoder_input_ids.numel() > 0:
362
- vision_embeddings = self.get_multimodal_embeddings(**kwargs)
363
- inputs_embeds = self.get_input_embeddings(encoder_input_ids,
364
- vision_embeddings)
365
-
366
- hidden_states = self.decoder(input_ids,
367
- positions,
368
- inputs_embeds=inputs_embeds)
369
- return hidden_states
370
-
371
- def compute_logits(
372
- self,
373
- hidden_states: torch.Tensor,
374
- sampling_metadata: SamplingMetadata,
375
- ) -> Optional[torch.Tensor]:
376
- return self.decoder.compute_logits(hidden_states, sampling_metadata)
377
-
378
- def load_weights(self, weights: Iterable[tuple[str,
379
- torch.Tensor]]) -> set[str]:
380
- loader = AutoWeightsLoader(self)
381
- return loader.load_weights(weights)
vllm/model_executor/models/florence2.py
deleted
-1097
@@ -1,1097 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import math
5
-from collections import OrderedDict
6
-from collections.abc import Iterable, Mapping, Sequence
7
-from typing import Annotated, Literal, Optional, Union
8
-
9
-import torch
10
-import torch.nn as nn
11
-import torch.nn.functional as F
12
-from einops import rearrange
13
-from transformers import BartTokenizer, BatchFeature, PretrainedConfig
14
-
15
-from vllm.config import VllmConfig
16
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
-from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
19
- BartParallelLMHead,
20
- BartScaledWordEmbedding)
21
-from vllm.model_executor.sampling_metadata import SamplingMetadata
22
-from vllm.multimodal import MULTIMODAL_REGISTRY
23
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
- MultiModalKwargsItems)
25
-from vllm.multimodal.parse import MultiModalDataItems
26
-from vllm.multimodal.processing import (BaseProcessingInfo,
27
- EncDecMultiModalProcessor,
28
- PromptIndexTargets, PromptInsertion,
29
- PromptUpdate)
30
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
31
-from vllm.sequence import IntermediateTensors
32
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
33
-
34
-from .interfaces import (MultiModalEmbeddings, SupportsMultiModal,
35
- SupportsV0Only)
36
-from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings
37
-
38
-
39
-class Florence2ImagePixelInputs(TensorSchema):
40
- """
41
- Dimensions:
42
- - b: Batch size
43
- - c: Number of channels (3)
44
- - h: Height of the image
45
- - w: Width of the image
46
- """
47
-
48
- type: Literal["pixel_values"]
49
-
50
- data: Annotated[
51
- torch.Tensor,
52
- TensorShape("b", 3, "h", "w"),
53
- ]
54
-
55
-
56
-# ViT implementation are all copied from
57
-# https://huggingface.co/microsoft/Florence-2-base/blob/main/modeling_florence2.py
58
-class LearnedAbsolutePositionEmbedding2D(nn.Module):
59
- """
60
- This module learns positional embeddings up to a fixed maximum size.
61
- """
62
-
63
- def __init__(self, embedding_dim=256, num_pos=50):
64
- super().__init__()
65
- self.row_embeddings = nn.Embedding(num_pos, embedding_dim // 2)
66
- self.column_embeddings = nn.Embedding(
67
- num_pos, embedding_dim - (embedding_dim // 2))
68
-
69
- def forward(self, pixel_values):
70
- """
71
- pixel_values: (batch_size, height, width, num_channels)
72
- returns: (batch_size, height, width, embedding_dim * 2)
73
- """
74
- if len(pixel_values.shape) != 4:
75
- raise ValueError('pixel_values must be a 4D tensor')
76
- height, width = pixel_values.shape[1:3]
77
- width_values = torch.arange(width, device=pixel_values.device)
78
- height_values = torch.arange(height, device=pixel_values.device)
79
- x_emb = self.column_embeddings(width_values)
80
- y_emb = self.row_embeddings(height_values)
81
- # (height, width, embedding_dim * 2)
82
- pos = torch.cat([
83
- x_emb.unsqueeze(0).repeat(height, 1, 1),
84
- y_emb.unsqueeze(1).repeat(1, width, 1)
85
- ],
86
- dim=-1)
87
- # (embedding_dim * 2, height, width)
88
- pos = pos.permute(2, 0, 1)
89
- pos = pos.unsqueeze(0)
90
- # (batch_size, embedding_dim * 2, height, width)
91
- pos = pos.repeat(pixel_values.shape[0], 1, 1, 1)
92
- # (batch_size, height, width, embedding_dim * 2)
93
- pos = pos.permute(0, 2, 3, 1)
94
- return pos
95
-
96
-
97
-class PositionalEmbeddingCosine1D(nn.Module):
98
- """
99
- This class implements a very simple positional encoding. It follows closely
100
- the encoder from the link below:
101
- https://pytorch.org/tutorials/beginner/translation_transformer.html
102
- Args:
103
- embed_dim: The dimension of the embeddings.
104
- dropout_prob: The dropout probability.
105
- max_seq_len: The maximum length to precompute the positional encodings.
106
- """
107
-
108
- def __init__(self, embed_dim: int = 512, max_seq_len: int = 1024) -> None:
109
- super().__init__()
110
- self.embed_dim = embed_dim
111
- self.max_seq_len = max_seq_len
112
- # Generate the sinusoidal arrays.
113
- factor = math.log(10000)
114
- denominator = torch.exp(-factor * torch.arange(0, self.embed_dim, 2) /
115
- self.embed_dim)
116
- # Matrix where rows correspond to a positional embedding as a function
117
- # of the position index (i.e., the row index).
118
- frequencies = \
119
- torch.arange(0, self.max_seq_len) \
120
- .reshape(self.max_seq_len, 1) * denominator
121
- pos_idx_to_embed = torch.zeros((self.max_seq_len, self.embed_dim))
122
- # Populate uneven entries.
123
- pos_idx_to_embed[:, 0::2] = torch.sin(frequencies)
124
- pos_idx_to_embed[:, 1::2] = torch.cos(frequencies)
125
- # Save the positional embeddings in a constant buffer.
126
- # self.register_buffer("pos_idx_to_embed", pos_idx_to_embed)
127
- self.pos_idx_to_embed = nn.Parameter(pos_idx_to_embed,
128
- requires_grad=False)
129
-
130
- def forward(self, seq_embeds: torch.Tensor) -> torch.Tensor:
131
- """
132
- Args:
133
- seq_embeds: The sequence embeddings in order. Allowed size:
134
- 1. [T, D], where T is the length of the sequence, and D is the
135
- frame embedding dimension.
136
- 2. [B, T, D], where B is the batch size and T and D are the
137
- same as above.
138
- Returns a tensor of with the same dimensions as the input: i.e.,
139
- [1, T, D] or [T, D].
140
- """
141
- shape_len = len(seq_embeds.shape)
142
- assert 2 <= shape_len <= 3
143
- len_seq = seq_embeds.size(-2)
144
- assert len_seq <= self.max_seq_len
145
- pos_embeds = self.pos_idx_to_embed[0:seq_embeds.size(-2), :]
146
- # Adapt pre-computed positional embeddings to the input.
147
- if shape_len == 3:
148
- pos_embeds = pos_embeds.view(
149
- (1, pos_embeds.size(0), pos_embeds.size(1)))
150
- return pos_embeds
151
-
152
-
153
-class MySequential(nn.Sequential):
154
-
155
- def forward(self, *inputs):
156
- for module in self._modules.values():
157
- if isinstance(inputs, tuple):
158
- inputs = module(*inputs)
159
- else:
160
- inputs = module(inputs)
161
- return inputs
162
-
163
-
164
-class PreNorm(nn.Module):
165
-
166
- def __init__(self, norm, fn):
167
- super().__init__()
168
- self.norm = norm
169
- self.fn = fn
170
-
171
- def forward(self, x, *args, **kwargs):
172
- shortcut = x
173
- if self.norm is not None:
174
- x, size = self.fn(self.norm(x), *args, **kwargs)
175
- else:
176
- x, size = self.fn(x, *args, **kwargs)
177
-
178
- x = shortcut + x
179
-
180
- return x, size
181
-
182
-
183
-class Mlp(nn.Module):
184
-
185
- def __init__(
186
- self,
187
- in_features,
188
- hidden_features=None,
189
- out_features=None,
190
- act_layer=nn.GELU,
191
- ):
192
- super().__init__()
193
- out_features = out_features or in_features
194
- hidden_features = hidden_features or in_features
195
- self.net = nn.Sequential(
196
- OrderedDict([("fc1", nn.Linear(in_features, hidden_features)),
197
- ("act", act_layer()),
198
- ("fc2", nn.Linear(hidden_features, out_features))]))
199
-
200
- def forward(self, x, size):
201
- return self.net(x), size
202
-
203
-
204
-class DepthWiseConv2d(nn.Module):
205
-
206
- def __init__(
207
- self,
208
- dim_in,
209
- kernel_size,
210
- padding,
211
- stride,
212
- bias=True,
213
- ):
214
- super().__init__()
215
- self.dw = nn.Conv2d(dim_in,
216
- dim_in,
217
- kernel_size=kernel_size,
218
- padding=padding,
219
- groups=dim_in,
220
- stride=stride,
221
- bias=bias)
222
-
223
- def forward(self, x, size):
224
- B, N, C = x.shape
225
- H, W = size
226
- assert N == H * W
227
-
228
- x = self.dw(x.transpose(1, 2).view(B, C, H, W))
229
- size = (x.size(-2), x.size(-1))
230
- x = x.flatten(2).transpose(1, 2)
231
- return x, size
232
-
233
-
234
-class ConvEmbed(nn.Module):
235
- """ Image to Patch Embedding
236
- """
237
-
238
- def __init__(self,
239
- patch_size=7,
240
- in_chans=3,
241
- embed_dim=64,
242
- stride=4,
243
- padding=2,
244
- norm_layer=None,
245
- pre_norm=True):
246
- super().__init__()
247
- self.patch_size = patch_size
248
-
249
- self.proj = nn.Conv2d(in_chans,
250
- embed_dim,
251
- kernel_size=patch_size,
252
- stride=stride,
253
- padding=padding)
254
-
255
- dim_norm = in_chans if pre_norm else embed_dim
256
- self.norm = norm_layer(dim_norm) if norm_layer else None
257
-
258
- self.pre_norm = pre_norm
259
-
260
- def forward(self, x, size):
261
- H, W = size
262
- if len(x.size()) == 3:
263
- if self.norm and self.pre_norm:
264
- x = self.norm(x)
265
- x = rearrange(x, 'b (h w) c -> b c h w', h=H, w=W)
266
-
267
- x = self.proj(x)
268
-
269
- _, _, H, W = x.shape
270
- x = rearrange(x, 'b c h w -> b (h w) c')
271
- if self.norm and not self.pre_norm:
272
- x = self.norm(x)
273
-
274
- return x, (H, W)
275
-
276
-
277
-class ChannelAttention(nn.Module):
278
-
279
- def __init__(self, dim, groups=8, qkv_bias=True):
280
- super().__init__()
281
-
282
- self.groups = groups
283
- self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
284
- self.proj = nn.Linear(dim, dim)
285
-
286
- def forward(self, x, size):
287
- B, N, C = x.shape
288
-
289
- qkv = self.qkv(x).reshape(B, N, 3, self.groups,
290
- C // self.groups).permute(2, 0, 3, 1, 4)
291
- q, k, v = qkv[0], qkv[1], qkv[2]
292
-
293
- q = q * (float(N)**-0.5)
294
- attention = q.transpose(-1, -2) @ k
295
- attention = attention.softmax(dim=-1)
296
- x = (attention @ v.transpose(-1, -2)).transpose(-1, -2)
297
- x = x.transpose(1, 2).reshape(B, N, C)
298
- x = self.proj(x)
299
- return x, size
300
-
301
-
302
-class ChannelBlock(nn.Module):
303
-
304
- def __init__(self,
305
- dim,
306
- groups,
307
- mlp_ratio=4.,
308
- qkv_bias=True,
309
- drop_path_rate=0.,
310
- act_layer=nn.GELU,
311
- norm_layer=nn.LayerNorm,
312
- conv_at_attn=True,
313
- conv_at_ffn=True):
314
- super().__init__()
315
-
316
- self.conv1 = PreNorm(None, DepthWiseConv2d(
317
- dim, 3, 1, 1)) if conv_at_attn else None
318
- self.channel_attn = PreNorm(
319
- norm_layer(dim),
320
- ChannelAttention(dim, groups=groups, qkv_bias=qkv_bias),
321
- )
322
- self.conv2 = PreNorm(None, DepthWiseConv2d(dim, 3, 1,
323
- 1)) if conv_at_ffn else None
324
- self.ffn = PreNorm(
325
- norm_layer(dim),
326
- Mlp(in_features=dim,
327
- hidden_features=int(dim * mlp_ratio),
328
- act_layer=act_layer),
329
- )
330
-
331
- def forward(self, x, size):
332
- if self.conv1:
333
- x, size = self.conv1(x, size)
334
- x, size = self.channel_attn(x, size)
335
-
336
- if self.conv2:
337
- x, size = self.conv2(x, size)
338
- x, size = self.ffn(x, size)
339
-
340
- return x, size
341
-
342
-
343
-def window_partition(x, window_size: int):
344
- B, H, W, C = x.shape
345
- x = x.view(B, H // window_size, window_size, W // window_size, window_size,
346
- C)
347
- windows = x.permute(0, 1, 3, 2, 4,
348
- 5).contiguous().view(-1, window_size, window_size, C)
349
- return windows
350
-
351
-
352
-def window_reverse(windows, batch_size: int, window_size: int, H: int, W: int):
353
- B = batch_size
354
-
355
- x = windows.view(B, H // window_size, W // window_size, window_size,
356
- window_size, -1)
357
- x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1)
358
- return x
359
-
360
-
361
-class WindowAttention(nn.Module):
362
-
363
- def __init__(self, dim, num_heads, window_size, qkv_bias=True):
364
-
365
- super().__init__()
366
- self.dim = dim
367
- self.window_size = window_size
368
- self.num_heads = num_heads
369
- head_dim = dim // num_heads
370
- self.scale = float(head_dim)**-0.5
371
-
372
- self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
373
- self.proj = nn.Linear(dim, dim)
374
-
375
- self.softmax = nn.Softmax(dim=-1)
376
-
377
- def forward(self, x, size):
378
-
379
- H, W = size
380
- B, L, C = x.shape
381
- assert L == H * W, "input feature has wrong size"
382
-
383
- x = x.view(B, H, W, C)
384
-
385
- pad_l = pad_t = 0
386
- pad_r = (self.window_size - W % self.window_size) % self.window_size
387
- pad_b = (self.window_size - H % self.window_size) % self.window_size
388
- x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b))
389
- _, Hp, Wp, _ = x.shape
390
-
391
- x = window_partition(x, self.window_size)
392
- x = x.view(-1, self.window_size * self.window_size, C)
393
-
394
- # W-MSA/SW-MSA
395
- # attn_windows = self.attn(x_windows)
396
-
397
- B_, N, C = x.shape
398
- qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads,
399
- C // self.num_heads).permute(2, 0, 3, 1, 4)
400
- q, k, v = qkv[0], qkv[1], qkv[2]
401
-
402
- q = q * self.scale
403
- attn = (q @ k.transpose(-2, -1))
404
- attn = self.softmax(attn)
405
-
406
- x = (attn @ v).transpose(1, 2).reshape(B_, N, C)
407
- x = self.proj(x)
408
-
409
- # merge windows
410
- x = x.view(-1, self.window_size, self.window_size, C)
411
- x = window_reverse(x, B, self.window_size, Hp, Wp)
412
-
413
- if pad_r > 0 or pad_b > 0:
414
- x = x[:, :H, :W, :].contiguous()
415
-
416
- x = x.view(B, H * W, C)
417
-
418
- return x, size
419
-
420
-
421
-class SpatialBlock(nn.Module):
422
-
423
- def __init__(self,
424
- dim,
425
- num_heads,
426
- window_size,
427
- mlp_ratio=4.,
428
- qkv_bias=True,
429
- drop_path_rate=0.,
430
- act_layer=nn.GELU,
431
- norm_layer=nn.LayerNorm,
432
- conv_at_attn=True,
433
- conv_at_ffn=True):
434
- super().__init__()
435
-
436
- self.conv1 = PreNorm(None, DepthWiseConv2d(
437
- dim, 3, 1, 1)) if conv_at_attn else None
438
- self.window_attn = PreNorm(
439
- norm_layer(dim),
440
- WindowAttention(dim, num_heads, window_size, qkv_bias=qkv_bias),
441
- )
442
- self.conv2 = PreNorm(None, DepthWiseConv2d(dim, 3, 1,
443
- 1)) if conv_at_ffn else None
444
- self.ffn = PreNorm(
445
- norm_layer(dim),
446
- Mlp(in_features=dim,
447
- hidden_features=int(dim * mlp_ratio),
448
- act_layer=act_layer),
449
- )
450
-
451
- def forward(self, x, size):
452
- if self.conv1:
453
- x, size = self.conv1(x, size)
454
- x, size = self.window_attn(x, size)
455
-
456
- if self.conv2:
457
- x, size = self.conv2(x, size)
458
- x, size = self.ffn(x, size)
459
- return x, size
460
-
461
-
462
-class DaViT(nn.Module):
463
-
464
- def __init__(
465
- self,
466
- in_chans=3,
467
- num_classes=1000,
468
- depths=(1, 1, 3, 1),
469
- patch_size=(7, 2, 2, 2),
470
- patch_stride=(4, 2, 2, 2),
471
- patch_padding=(3, 0, 0, 0),
472
- patch_prenorm=(False, False, False, False),
473
- embed_dims=(64, 128, 192, 256),
474
- num_heads=(3, 6, 12, 24),
475
- num_groups=(3, 6, 12, 24),
476
- window_size=7,
477
- mlp_ratio=4.,
478
- qkv_bias=True,
479
- drop_path_rate=0.1,
480
- norm_layer=nn.LayerNorm,
481
- enable_checkpoint=False,
482
- conv_at_attn=True,
483
- conv_at_ffn=True,
484
- ):
485
- super().__init__()
486
-
487
- self.num_classes = num_classes
488
- self.embed_dims = embed_dims
489
- self.num_heads = num_heads
490
- self.num_groups = num_groups
491
- self.num_stages = len(self.embed_dims)
492
- self.enable_checkpoint = enable_checkpoint
493
- assert self.num_stages == len(self.num_heads) == len(self.num_groups)
494
-
495
- num_stages = len(embed_dims)
496
- dpr = [
497
- x.item() for x in torch.linspace(0, drop_path_rate,
498
- sum(depths) * 2)
499
- ]
500
-
501
- depth_offset = 0
502
- convs = []
503
- blocks = []
504
- for i in range(num_stages):
505
- conv_embed = ConvEmbed(
506
- patch_size=patch_size[i],
507
- stride=patch_stride[i],
508
- padding=patch_padding[i],
509
- in_chans=in_chans if i == 0 else self.embed_dims[i - 1],
510
- embed_dim=self.embed_dims[i],
511
- norm_layer=norm_layer,
512
- pre_norm=patch_prenorm[i])
513
- convs.append(conv_embed)
514
-
515
- block = MySequential(*[
516
- MySequential(
517
- OrderedDict([('spatial_block',
518
- SpatialBlock(
519
- embed_dims[i],
520
- num_heads[i],
521
- window_size,
522
- drop_path_rate=dpr[depth_offset + j * 2],
523
- qkv_bias=qkv_bias,
524
- mlp_ratio=mlp_ratio,
525
- conv_at_attn=conv_at_attn,
526
- conv_at_ffn=conv_at_ffn,
527
- )),
528
- ('channel_block',
529
- ChannelBlock(
530
- embed_dims[i],
531
- num_groups[i],
532
- drop_path_rate=dpr[depth_offset + j * 2 +
533
- 1],
534
- qkv_bias=qkv_bias,
535
- mlp_ratio=mlp_ratio,
536
- conv_at_attn=conv_at_attn,
537
- conv_at_ffn=conv_at_ffn,
538
- ))])) for j in range(depths[i])
539
- ])
540
- blocks.append(block)
541
- depth_offset += depths[i] * 2
542
-
543
- self.convs = nn.ModuleList(convs)
544
- self.blocks = nn.ModuleList(blocks)
545
-
546
- self.avgpool = nn.AdaptiveAvgPool1d(1)
547
-
548
- @property
549
- def dim_out(self):
550
- return self.embed_dims[-1]
551
-
552
- def forward_features_unpool(self, x):
553
- """
554
- forward until avg pooling
555
- Args:
556
- x (_type_): input image tensor
557
- """
558
- input_size = (x.size(2), x.size(3))
559
- for conv, block in zip(self.convs, self.blocks):
560
- x, input_size = conv(x, input_size)
561
- x, input_size = block(x, input_size)
562
- return x
563
-
564
- def forward_features(self, x):
565
- x = self.forward_features_unpool(x)
566
-
567
- # (batch_size, num_tokens, token_dim)
568
- x = self.avgpool(x.transpose(1, 2))
569
- # (batch_size, 1, num_tokens)
570
- x = torch.flatten(x, 1)
571
- x = self.norms(x)
572
-
573
- return x
574
-
575
- def forward(self, x):
576
- x = self.forward_features(x)
577
- x = self.head(x)
578
- return x
579
-
580
- @classmethod
581
- def from_config(cls, config):
582
- return cls(
583
- depths=config.depths,
584
- embed_dims=config.dim_embed,
585
- num_heads=config.num_heads,
586
- num_groups=config.num_groups,
587
- patch_size=config.patch_size,
588
- patch_stride=config.patch_stride,
589
- patch_padding=config.patch_padding,
590
- patch_prenorm=config.patch_prenorm,
591
- drop_path_rate=config.drop_path_rate,
592
- window_size=config.window_size,
593
- )
594
-
595
-
596
-# Language backbone and processor implementation
597
-class Florence2LanguageModel(nn.Module):
598
-
599
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
600
- super().__init__()
601
-
602
- config = vllm_config.model_config.hf_config
603
- cache_config = vllm_config.cache_config
604
- quant_config = vllm_config.quant_config
605
-
606
- self.config = config
607
-
608
- self.vocab_size = config.vocab_size
609
-
610
- self.shared = BartScaledWordEmbedding(self.vocab_size, config.d_model)
611
- self.encoder = BartEncoder(config,
612
- cache_config=cache_config,
613
- quant_config=quant_config,
614
- prefix=f"{prefix}.encoder")
615
- self.decoder = BartDecoder(config,
616
- cache_config=cache_config,
617
- quant_config=quant_config,
618
- prefix=f"{prefix}.decoder")
619
-
620
- if self.config.tie_word_embeddings:
621
- self.encoder.embed_tokens.weight = self.shared.weight
622
- self.decoder.embed_tokens.weight = self.shared.weight
623
-
624
- def forward(
625
- self,
626
- input_ids: torch.Tensor,
627
- positions: torch.Tensor,
628
- encoder_input_ids: torch.Tensor,
629
- encoder_positions: torch.Tensor,
630
- inputs_embeds: Optional[torch.Tensor] = None,
631
- ) -> torch.Tensor:
632
- r"""
633
- Args:
634
- input_ids: Indices of *decoder* input sequence tokens
635
- in the vocabulary.
636
- Padding will be ignored by default should you
637
- provide it.
638
- positions: Positions of *decoder* input sequence tokens.
639
- encoder_input_ids: Indices of *encoder* input sequence tokens
640
- in the vocabulary.
641
- encoder_positions: Positions of *encoder* input sequence tokens.
642
- Returns:
643
- Model output torch.Tensor
644
- """
645
-
646
- encoder_hidden_states = None
647
-
648
- if ((inputs_embeds is not None and inputs_embeds.numel() > 0)
649
- or encoder_input_ids.numel() > 0):
650
- # Run encoder attention if a non-zero number of encoder tokens
651
- # are provided as input
652
- encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
653
- positions=encoder_positions,
654
- inputs_embeds=inputs_embeds)
655
-
656
- # decoder outputs consists of
657
- # (dec_features, past_key_value, dec_hidden, dec_attn)
658
- decoder_outputs = self.decoder(
659
- decoder_input_ids=input_ids,
660
- decoder_positions=positions,
661
- encoder_hidden_states=encoder_hidden_states)
662
-
663
- return decoder_outputs
664
-
665
-
666
-class Florence2LanguageForConditionalGeneration(nn.Module, SupportsV0Only):
667
-
668
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
669
- super().__init__()
670
-
671
- config = vllm_config.model_config.hf_config
672
-
673
- self.config = config
674
- self.model = Florence2LanguageModel(vllm_config=vllm_config,
675
- prefix=f"{prefix}.model")
676
- embed_scale = math.sqrt(
677
- config.d_model) if config.scale_embedding else 1.0
678
-
679
- self.vocab_size = config.vocab_size
680
- self.lm_head = BartParallelLMHead(self.vocab_size,
681
- config.d_model,
682
- embed_scale=embed_scale)
683
- if self.config.tie_word_embeddings:
684
- self.lm_head.tie_weights(self.model.shared)
685
-
686
- self.logits_processor = LogitsProcessor(self.vocab_size,
687
- config.vocab_size)
688
-
689
- def forward(
690
- self,
691
- input_ids: torch.Tensor,
692
- positions: torch.Tensor,
693
- encoder_input_ids: torch.Tensor,
694
- encoder_positions: torch.Tensor,
695
- inputs_embeds: Optional[torch.Tensor] = None,
696
- **kwargs,
697
- ) -> torch.Tensor:
698
- r"""
699
- Args:
700
- input_ids: torch.Tensor of *decoder* input token ids.
701
- positions: torch.Tensor of *decoder* position indices.
702
- encoder_input_ids: torch.Tensor of *encoder* input token ids.
703
- encoder_positions: torch.Tensor of *encoder* position indices
704
- Returns:
705
- Output torch.Tensor
706
- """
707
-
708
- return self.model(input_ids,
709
- positions,
710
- encoder_input_ids,
711
- encoder_positions,
712
- inputs_embeds=inputs_embeds)
713
-
714
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
715
- return self.model.encoder.embed_tokens(input_ids)
716
-
717
- def compute_logits(
718
- self,
719
- hidden_states: torch.Tensor,
720
- sampling_metadata: SamplingMetadata,
721
- ) -> Optional[torch.Tensor]:
722
- logits = self.logits_processor(self.lm_head, hidden_states,
723
- sampling_metadata)
724
- return logits
725
-
726
- def load_weights(self, weights: Iterable[tuple[str,
727
- torch.Tensor]]) -> set[str]:
728
- stacked_params_mapping = [
729
- # (param_name, shard_name, shard_id)
730
- ("qkv_proj", "q_proj", "q"),
731
- ("qkv_proj", "k_proj", "k"),
732
- ("qkv_proj", "v_proj", "v"),
733
- ]
734
-
735
- params_dict = dict(self.named_parameters())
736
- loaded_params: set[str] = set()
737
- for name, loaded_weight in weights:
738
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
739
- if weight_name not in name:
740
- continue
741
- name = name.replace(weight_name, param_name)
742
- param = params_dict[name]
743
- weight_loader = param.weight_loader
744
- weight_loader(param, loaded_weight, shard_id)
745
- break
746
- else:
747
- if "final_logits_bias" in name:
748
- continue
749
- if self.config.tie_word_embeddings and ("embed_tokens" in name
750
- or "lm_head" in name):
751
- continue
752
- param = params_dict[name]
753
- weight_loader = getattr(param, "weight_loader",
754
- default_weight_loader)
755
- weight_loader(param, loaded_weight)
756
- loaded_params.add(name)
757
- return loaded_params
758
-
759
-
760
-class Florence2ProcessingInfo(BaseProcessingInfo):
761
-
762
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
763
- return {"image": 1}
764
-
765
- def get_num_image_tokens(self) -> int:
766
- processor_config = self.ctx.get_hf_image_processor_config()
767
- return processor_config["image_seq_length"]
768
-
769
-
770
-class Florence2DummyInputsBuilder(
771
- BaseDummyInputsBuilder[Florence2ProcessingInfo]):
772
-
773
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
774
- return ""
775
-
776
- def get_dummy_mm_data(
777
- self,
778
- seq_len: int,
779
- mm_counts: Mapping[str, int],
780
- ) -> MultiModalDataDict:
781
- num_images = mm_counts.get("image", 0)
782
-
783
- target_width = target_height = self.info.get_hf_config().projection_dim
784
-
785
- return {
786
- "image":
787
- self._get_dummy_images(width=target_width,
788
- height=target_height,
789
- num_images=num_images)
790
- }
791
-
792
-
793
-class Florence2MultiModalProcessor(
794
- EncDecMultiModalProcessor[Florence2ProcessingInfo]):
795
-
796
- def _hf_processor_applies_updates(
797
- self,
798
- prompt_text: str,
799
- mm_items: MultiModalDataItems,
800
- hf_processor_mm_kwargs: Mapping[str, object],
801
- tokenization_kwargs: Mapping[str, object],
802
- ) -> bool:
803
- return False
804
-
805
- def create_encoder_prompt(
806
- self,
807
- prompt: Union[str, list[int]],
808
- mm_data: MultiModalDataDict,
809
- ) -> Union[str, list[int]]:
810
- return prompt
811
-
812
- def create_decoder_prompt(
813
- self,
814
- prompt: Union[str, list[int]],
815
- mm_data: MultiModalDataDict,
816
- ) -> Union[str, list[int]]:
817
- return [self.info.get_hf_config().eos_token_id]
818
-
819
- def _apply_hf_processor_tokens_only(
820
- self,
821
- prompt_tokens: list[int],
822
- ) -> list[int]:
823
- hf_processor = self.info.get_hf_processor()
824
- tokenizer: BartTokenizer = hf_processor.tokenizer
825
- prompt_text = tokenizer.decode(prompt_tokens)
826
- # convert task tokens to prompt
827
- prompt_text = hf_processor._construct_prompts([prompt_text])[0]
828
- prompt_tokens = tokenizer.encode(prompt_text, add_special_tokens=False)
829
- return prompt_tokens
830
-
831
- def _call_hf_processor(
832
- self,
833
- prompt: str,
834
- mm_data: Mapping[str, object],
835
- mm_kwargs: Mapping[str, object],
836
- tok_kwargs: Mapping[str, object],
837
- ) -> BatchFeature:
838
- if mm_data:
839
- processed_outputs = super()._call_hf_processor(
840
- prompt, mm_data, mm_kwargs, tok_kwargs)
841
- else:
842
- hf_processor = self.info.get_hf_processor()
843
- tokenizer = hf_processor.tokenizer
844
- prompt = hf_processor._construct_prompts([prompt])[0]
845
- processed_outputs = tokenizer(prompt,
846
- add_special_tokens=True,
847
- return_tensors="pt")
848
- return processed_outputs
849
-
850
- def _get_mm_fields_config(
851
- self,
852
- hf_inputs: BatchFeature,
853
- hf_processor_mm_kwargs: Mapping[str, object],
854
- ) -> Mapping[str, MultiModalFieldConfig]:
855
- return dict(pixel_values=MultiModalFieldConfig.batched("image"))
856
-
857
- def _get_prompt_updates(
858
- self,
859
- mm_items: MultiModalDataItems,
860
- hf_processor_mm_kwargs: Mapping[str, object],
861
- out_mm_kwargs: MultiModalKwargsItems,
862
- ) -> Sequence[PromptUpdate]:
863
- hf_config = self.info.get_hf_config()
864
- pad_token_id = hf_config.pad_token_id
865
- num_image_tokens = self.info.get_num_image_tokens()
866
- image_tokens = [pad_token_id] * num_image_tokens
867
-
868
- return [
869
- PromptInsertion(
870
- modality="image",
871
- target=PromptIndexTargets.start(),
872
- insertion=image_tokens,
873
- )
874
- ]
875
-
876
-
877
-@MULTIMODAL_REGISTRY.register_processor(
878
- Florence2MultiModalProcessor,
879
- info=Florence2ProcessingInfo,
880
- dummy_inputs=Florence2DummyInputsBuilder)
881
-class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,
882
- SupportsV0Only):
883
-
884
- @classmethod
885
- def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
886
- if modality.startswith("image"):
887
- return None
888
-
889
- raise ValueError("Only image modality is supported")
890
-
891
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
892
- super().__init__()
893
- config = vllm_config.model_config.hf_config
894
- processor_config = vllm_config.model_config.hf_image_processor_config
895
-
896
- self.config = config
897
- self.vision_config = config.vision_config
898
- self.processor_config = processor_config
899
- assert config.vision_config.model_type == 'davit', (
900
- 'only DaViT is supported for now')
901
- self.vision_tower = DaViT.from_config(config=config.vision_config)
902
- self._build_image_projection_layers(config)
903
- self.language_model = Florence2LanguageForConditionalGeneration(
904
- vllm_config=vllm_config.with_hf_config(config.text_config),
905
- prefix=f"{prefix}.language_model",
906
- )
907
- self.pad_token_id = config.pad_token_id
908
-
909
- def _build_image_projection_layers(self, config: PretrainedConfig):
910
- image_dim_out = config.vision_config.dim_embed[-1]
911
- dim_projection = config.vision_config.projection_dim
912
- self.image_projection = nn.Parameter(
913
- torch.empty(image_dim_out, dim_projection))
914
- self.image_proj_norm = nn.LayerNorm(dim_projection)
915
- image_pos_embed_config = config.vision_config.image_pos_embed
916
- if image_pos_embed_config['type'] == 'learned_abs_2d':
917
- self.image_pos_embed = LearnedAbsolutePositionEmbedding2D(
918
- embedding_dim=image_dim_out,
919
- num_pos=image_pos_embed_config['max_pos_embeddings'])
920
- else:
921
- raise NotImplementedError("Florence2 only supports learned_abs_2d "
922
- "as image position embedding.")
923
-
924
- self.image_feature_source = config.vision_config.image_feature_source
925
-
926
- # temporal embedding
927
- visual_temporal_embedding_config = (
928
- self.vision_config.visual_temporal_embedding)
929
- if visual_temporal_embedding_config['type'] == 'COSINE':
930
- self.visual_temporal_embed = PositionalEmbeddingCosine1D(
931
- embed_dim=image_dim_out,
932
- max_seq_len=visual_temporal_embedding_config[
933
- 'max_temporal_embeddings'])
934
- else:
935
- raise NotImplementedError(
936
- 'Florence2 only supports COSINE as temporal embedding.')
937
-
938
- def _parse_and_validate_image_input(self, **kwargs: object):
939
- pixel_values: Optional[Union[list[list[torch.Tensor]],
940
- list[torch.Tensor],
941
- torch.Tensor]] = kwargs.pop(
942
- "pixel_values", None)
943
- image_embeds: Optional[Union[list[list[torch.Tensor]],
944
- list[torch.Tensor],
945
- torch.Tensor]] = kwargs.pop(
946
- "image_embeds", None)
947
-
948
- if pixel_values is None and image_embeds is None:
949
- return None
950
-
951
- if pixel_values is not None and image_embeds is not None:
952
- raise ValueError(
953
- "Both pixel values and image embeds are provided.")
954
-
955
- if pixel_values is not None:
956
- size = self.processor_config["size"]
957
- expected_h, expected_w = size["height"], size["width"]
958
-
959
- return Florence2ImagePixelInputs(
960
- type="pixel_values",
961
- data=flatten_bn(pixel_values, concat=True),
962
- resolve_bindings={
963
- "h": expected_h,
964
- "w": expected_w
965
- },
966
- )
967
-
968
- if image_embeds is not None:
969
- raise NotImplementedError
970
-
971
- raise AssertionError("This line should be unreachable.")
972
-
973
- def _encode_image(self, pixel_values: torch.Tensor) -> torch.Tensor:
974
- dtype = next(self.vision_tower.parameters()).dtype
975
- pixel_values = pixel_values.to(dtype)
976
-
977
- batch_size, T = pixel_values.size(0), 1
978
- x = self.vision_tower.forward_features_unpool(pixel_values)
979
- if self.image_pos_embed is not None:
980
- x = x.view(batch_size * T, -1, x.shape[-1])
981
- num_tokens = x.shape[-2]
982
- h, w = int(num_tokens**0.5), int(num_tokens**0.5)
983
- assert h * w == num_tokens, (
984
- 'only support square feature maps for now')
985
- x = x.view(batch_size * T, h, w, x.shape[-1])
986
- pos_embed = self.image_pos_embed(x)
987
- x = x + pos_embed
988
- x = x.view(batch_size, T * h * w, x.shape[-1])
989
-
990
- if self.visual_temporal_embed is not None:
991
- visual_temporal_embed = self.visual_temporal_embed(
992
- x.view(batch_size, T, -1, x.shape[-1])[:, :, 0])
993
- x = x.view(batch_size, T, -1,
994
- x.shape[-1]) + visual_temporal_embed.view(
995
- 1, T, 1, x.shape[-1])
996
-
997
- x_feat_dict = {}
998
-
999
- spatial_avg_pool_x = x.view(batch_size, T, -1, x.shape[-1]).mean(dim=2)
1000
- x_feat_dict['spatial_avg_pool'] = spatial_avg_pool_x
1001
-
1002
- temporal_avg_pool_x = x.view(batch_size, T, -1,
1003
- x.shape[-1]).mean(dim=1)
1004
- x_feat_dict['temporal_avg_pool'] = temporal_avg_pool_x
1005
-
1006
- x = x.view(batch_size, T, -1, x.shape[-1])[:, -1]
1007
- x_feat_dict['last_frame'] = x
1008
-
1009
- new_x = []
1010
- for _image_feature_source in self.image_feature_source:
1011
- if _image_feature_source not in x_feat_dict:
1012
- raise ValueError('invalid image feature source: {}'.format(
1013
- _image_feature_source))
1014
- new_x.append(x_feat_dict[_image_feature_source])
1015
-
1016
- x = torch.cat(new_x, dim=1)
1017
-
1018
- x = x @ self.image_projection
1019
- x = self.image_proj_norm(x)
1020
-
1021
- return x
1022
-
1023
- def _process_image_input(
1024
- self, image_input: Florence2ImagePixelInputs) -> torch.Tensor:
1025
- assert image_input["type"] == "pixel_values"
1026
- pixel_values = image_input["data"]
1027
- return self._encode_image(pixel_values)
1028
-
1029
- def get_language_model(self) -> torch.nn.Module:
1030
- return self.language_model
1031
-
1032
- def get_multimodal_embeddings(self,
1033
- **kwargs: object) -> MultiModalEmbeddings:
1034
- image_input = self._parse_and_validate_image_input(**kwargs)
1035
- if image_input is None:
1036
- return []
1037
- vision_embeddings = self._process_image_input(image_input)
1038
- return vision_embeddings
1039
-
1040
- def get_input_embeddings(
1041
- self,
1042
- input_ids: torch.Tensor,
1043
- multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
1044
- ) -> torch.Tensor:
1045
- inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1046
- if multimodal_embeddings is not None \
1047
- and len(multimodal_embeddings) != 0:
1048
- inputs_embeds = merge_multimodal_embeddings(
1049
- input_ids, inputs_embeds, multimodal_embeddings,
1050
- self.pad_token_id)
1051
- return inputs_embeds
1052
-
1053
- def forward(
1054
- self,
1055
- input_ids: torch.Tensor,
1056
- positions: torch.Tensor,
1057
- intermediate_tensors: Optional[IntermediateTensors] = None,
1058
- *,
1059
- encoder_input_ids: torch.Tensor,
1060
- encoder_positions: torch.Tensor,
1061
- **kwargs,
1062
- ) -> torch.Tensor:
1063
- r"""
1064
- Args:
1065
- input_ids: torch.Tensor of *decoder* input token ids.
1066
- positions: torch.Tensor of *decoder* position indices.
1067
- encoder_input_ids: torch.Tensor of *encoder* input token ids.
1068
- encoder_positions: torch.Tensor of *encoder* position indices
1069
- Returns:
1070
- Output torch.Tensor
1071
- """
1072
- vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1073
- if encoder_input_ids.numel() > 0 or vision_embeddings is not None:
1074
- inputs_embeds = self.get_input_embeddings(encoder_input_ids,
1075
- vision_embeddings)
1076
- else:
1077
- inputs_embeds = None
1078
-
1079
- hidden_states = self.language_model(input_ids,
1080
- positions,
1081
- encoder_input_ids,
1082
- encoder_positions,
1083
- inputs_embeds=inputs_embeds)
1084
- return hidden_states
1085
-
1086
- def compute_logits(
1087
- self,
1088
- hidden_states: torch.Tensor,
1089
- sampling_metadata: SamplingMetadata,
1090
- ) -> Optional[torch.Tensor]:
1091
- return self.language_model.compute_logits(hidden_states,
1092
- sampling_metadata)
1093
-
1094
- def load_weights(self, weights: Iterable[tuple[str,
1095
- torch.Tensor]]) -> set[str]:
1096
- loader = AutoWeightsLoader(self)
1097
- return loader.load_weights(weights)
vllm/model_executor/models/mllama.py
deleted
-1697
@@ -1,1697 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
5
-#
6
-# Licensed under the Apache License, Version 2.0 (the "License");
7
-# you may not use this file except in compliance with the License.
8
-# You may obtain a copy of the License at
9
-#
10
-# http://www.apache.org/licenses/LICENSE-2.0
11
-#
12
-# Unless required by applicable law or agreed to in writing, software
13
-# distributed under the License is distributed on an "AS IS" BASIS,
14
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
-# See the License for the specific language governing permissions and
16
-# limitations under the License.
17
-"""PyTorch Mllama model."""
18
-import math
19
-from collections.abc import Iterable, Mapping, Sequence
20
-from typing import Annotated, Literal, Optional, Union
21
-
22
-import numpy as np
23
-import torch
24
-import torch.nn.functional as F
25
-import transformers.models.mllama.configuration_mllama as config_mllama
26
-from PIL.Image import Image
27
-from torch import nn
28
-from transformers import BatchFeature, MllamaConfig
29
-from transformers.modeling_outputs import (BaseModelOutput,
30
- CausalLMOutputWithPast)
31
-from transformers.models.mllama.image_processing_mllama import (
32
- get_optimal_tiled_canvas)
33
-from transformers.models.mllama.processing_mllama import (
34
- MllamaProcessor, get_cross_attention_token_mask)
35
-
36
-import vllm.distributed.parallel_state as ps
37
-from vllm.attention import Attention, AttentionMetadata, AttentionType
38
-from vllm.attention.layer import MultiHeadAttention
39
-from vllm.attention.ops.paged_attn import PagedAttention
40
-from vllm.attention.selector import _Backend
41
-from vllm.config import VllmConfig
42
-from vllm.distributed import get_pp_group, get_tp_group
43
-from vllm.forward_context import get_forward_context
44
-from vllm.logger import init_logger
45
-from vllm.model_executor.layers.layernorm import RMSNorm
46
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
47
- QKVCrossParallelLinear,
48
- QKVParallelLinear,
49
- RowParallelLinear)
50
-from vllm.model_executor.layers.logits_processor import LogitsProcessor
51
-from vllm.model_executor.layers.quantization import QuantizationConfig
52
-from vllm.model_executor.layers.vocab_parallel_embedding import (
53
- DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
54
-from vllm.model_executor.model_loader.weight_utils import (
55
- default_weight_loader, maybe_remap_kv_scale_name)
56
-from vllm.model_executor.models.module_mapping import MultiModelKeys
57
-from vllm.model_executor.sampling_metadata import SamplingMetadata
58
-from vllm.multimodal import MULTIMODAL_REGISTRY
59
-from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
60
- MultiModalFieldConfig,
61
- MultiModalKwargsItems, MultiModalUUIDDict)
62
-from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
63
- MultiModalDataItems)
64
-from vllm.multimodal.processing import (BaseProcessingInfo,
65
- EncDecMultiModalProcessor,
66
- PromptReplacement, PromptUpdate)
67
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
68
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
69
-
70
-from .clip import CLIPMLP
71
-from .interfaces import SupportsMultiModal, SupportsV0Only
72
-from .llama import LlamaDecoderLayer, LlamaMLP
73
-from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
74
-
75
-logger = init_logger(__name__)
76
-
77
-
78
-class MllamaImagePixelInputs(TensorSchema):
79
- """
80
- Dimensions:
81
- - batch_size: Batch size
82
- - max_num_image: Max number of images
83
- - max_num_chunk: Max number of chunks
84
- - max_num_tiles: Max number of tiles per image
85
- - num_channel: Number of channels
86
- - height: Height
87
- - width: Width
88
- """
89
-
90
- type: Literal["pixel_values"] = "pixel_values"
91
-
92
- data: Annotated[torch.Tensor,
93
- TensorShape("batch_size", "max_num_image", "max_num_chunk",
94
- "num_channel", "height", "width")]
95
-
96
- aspect_ratio_ids: Annotated[torch.Tensor,
97
- TensorShape("batch_size", "max_num_image")]
98
-
99
- aspect_ratio_mask: Annotated[
100
- torch.Tensor,
101
- TensorShape("batch_size", "max_num_image", "max_num_tiles")]
102
-
103
-
104
-# TODO: support LlamaImageEmbeddingInputs
105
-
106
-
107
-def calc_token_per_chunk(image_size: int) -> int:
108
- assert image_size % 14 == 0, "chunk size should be multiple of 14"
109
- token_per_chunk = (image_size // 14)**2 + 1
110
- return token_per_chunk
111
-
112
-
113
-class MllamaProcessingInfo(BaseProcessingInfo):
114
-
115
- def get_hf_config(self) -> MllamaConfig:
116
- return self.ctx.get_hf_config(MllamaConfig)
117
-
118
- def get_hf_processor(self, **kwargs: object) -> MllamaProcessor:
119
- return self.ctx.get_hf_processor(MllamaProcessor, **kwargs)
120
-
121
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
122
- return {"image": None}
123
-
124
- def get_token_per_chunk_from_config(self) -> int:
125
- image_size = self.get_hf_config().vision_config.image_size
126
- return calc_token_per_chunk(image_size)
127
-
128
- def get_num_tiles_per_image(self, image_height: int,
129
- image_width: int) -> int:
130
- vision_config = self.get_hf_config().vision_config
131
- max_num_tiles = vision_config.max_num_tiles
132
- image_size = vision_config.image_size
133
- tiled_height, tiled_width = get_optimal_tiled_canvas(
134
- image_height,
135
- image_width,
136
- max_num_tiles,
137
- tile_size=image_size,
138
- )
139
- num_tiles_height = tiled_height // image_size
140
- num_tiles_width = tiled_width // image_size
141
- return num_tiles_height * num_tiles_width
142
-
143
- def get_image_size_with_most_features(self) -> ImageSize:
144
- vision_config = self.get_hf_config().vision_config
145
- image_size = vision_config.image_size
146
- max_num_tiles = vision_config.max_num_tiles
147
- # Result in the max possible feature size (h:w = 16:1)
148
- return ImageSize(height=max_num_tiles * image_size, width=image_size)
149
-
150
-
151
-class MllamaDummyInputsBuilder(BaseDummyInputsBuilder[MllamaProcessingInfo]):
152
-
153
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
154
- num_images = mm_counts.get("image", 0)
155
-
156
- processor = self.info.get_hf_processor()
157
- image_token = processor.image_token
158
-
159
- return image_token * num_images
160
-
161
- def get_dummy_mm_data(
162
- self,
163
- seq_len: int,
164
- mm_counts: Mapping[str, int],
165
- ) -> MultiModalDataDict:
166
- num_images = mm_counts.get("image", 0)
167
-
168
- target_width, target_height = \
169
- self.info.get_image_size_with_most_features()
170
-
171
- return {
172
- "image":
173
- self._get_dummy_images(width=target_width,
174
- height=target_height,
175
- num_images=num_images)
176
- }
177
-
178
-
179
-class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
180
- ):
181
-
182
- def apply(
183
- self,
184
- prompt: Union[str, list[int]],
185
- mm_data: MultiModalDataDict,
186
- hf_processor_mm_kwargs: Mapping[str, object],
187
- tokenization_kwargs: Optional[Mapping[str, object]] = None,
188
- mm_uuids: Optional[MultiModalUUIDDict] = None,
189
- ) -> MultiModalEncDecInputs:
190
- mm_inputs = super().apply(prompt,
191
- mm_data,
192
- hf_processor_mm_kwargs,
193
- tokenization_kwargs,
194
- mm_uuids=mm_uuids)
195
-
196
- image_token_id = self.info.get_hf_config().image_token_index
197
- # Check that the number of image tokens in the decoder prompt matches
198
- # the number of images provided in mm_data
199
- num_image_tokens = mm_inputs['prompt_token_ids'].count(image_token_id)
200
- image_data = mm_data.get("image", [])
201
- num_images = 1 if isinstance(image_data, Image) else len(image_data)
202
- if num_image_tokens != num_images:
203
- raise ValueError(
204
- f"The number of image tokens ({num_image_tokens}) must be"
205
- f" the same as the number of images ({num_images})")
206
-
207
- # Given prompt: <IMG0> P0 P1 <IMG1> <IMG2> P3 P4 D5 D6...., (P-prefill, D-decode) # noqa: E501
208
- # P0 & P1 do cross attention with placeholder of <IMG0>
209
- # P3 P4 D5 D6 do cross attention with placeholder of <IMG1> and <IMG2>
210
- # Example input to encoder and decoder:
211
- # {
212
- # 'encoder': {
213
- # 'type': 'token',
214
- # 'prompt_token_ids': [128256, 128256, ..., 128256],
215
- # 'prompt': '<|image|><|image|>...<|image|>',
216
- # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
217
- # },
218
- # 'decoder': {
219
- # 'type': 'token',
220
- # 'prompt_token_ids': [128000, 128256, 128000, 3923, 374, 279, 2262, 315, 420, 2217, 30], # noqa: E501
221
- # 'prompt': '<|image|><|begin_of_text|>What is the content of this image?', # noqa: E501
222
- # 'multi_modal_data': {'image': <PIL.Image.Image image mode=RGB size=1770x1180 at 0x7FDE2C624880>}, # noqa: E501
223
- # },
224
- # }
225
-
226
- if mm_data:
227
- hf_processor = self.info.get_hf_processor()
228
- image_token: str = hf_processor.image_token
229
-
230
- # Since only the last group of consecutive images
231
- # are attended by the decoded tokens, we only need to
232
- # get the number of tokens for those images.
233
- token_per_chunk = self.info.get_token_per_chunk_from_config()
234
- num_decode_images = self._get_num_image_in_last_group(
235
- mm_inputs["prompt_token_ids"])
236
- num_encode_images = num_images - num_decode_images
237
-
238
- # Set encoder prompt length based on the number of tiles.
239
- # This tells the block manager to allocate correct number
240
- # of slots for encoder tokens.
241
- num_tiles = mm_inputs["mm_kwargs"].get_data()["num_tiles"]
242
- decode_tiles = num_tiles[num_encode_images:num_images].sum().item()
243
- num_tokens = decode_tiles * token_per_chunk
244
- mm_inputs["encoder_prompt_token_ids"] = [image_token_id
245
- ] * num_tokens
246
- mm_inputs["encoder_prompt"] = image_token * num_tokens
247
-
248
- return mm_inputs
249
-
250
- def _get_num_image_in_last_group(self, prompt_token_ids: list[int]) -> int:
251
- num_images = 0
252
- for token_id in prompt_token_ids[::-1]:
253
- if token_id == self.info.get_hf_config().image_token_index:
254
- num_images += 1
255
- elif num_images > 0:
256
- break
257
- return num_images
258
-
259
- def _call_hf_processor(
260
- self,
261
- prompt: str,
262
- mm_data: Mapping[str, object],
263
- mm_kwargs: Mapping[str, object],
264
- tok_kwargs: Mapping[str, object],
265
- ) -> BatchFeature:
266
- tokenizer = self.info.get_tokenizer()
267
- if mm_data:
268
- num_tiles = [
269
- self.info.get_num_tiles_per_image(img.height, img.width)
270
- for img in mm_data["images"]
271
- ]
272
- processed_outputs = super()._call_hf_processor(
273
- prompt, mm_data, mm_kwargs, tok_kwargs)
274
- processed_outputs["num_tiles"] = torch.tensor(num_tiles)
275
- for k in ('pixel_values', 'aspect_ratio_ids', "aspect_ratio_mask"):
276
- processed_outputs[k] = processed_outputs[k].squeeze(0)
277
-
278
- processed_token_ids = processed_outputs.pop("input_ids")
279
- start_idx, end_idx = 0, processed_token_ids.size(1)
280
- processed_prompt_text = tokenizer.decode(processed_token_ids[0])
281
-
282
- hf_processor = self.info.get_hf_processor()
283
- bos_token = hf_processor.bos_token
284
- # Remove the bos_token from the start of prompt,
285
- # because we all know there would be image_token.
286
- if processed_prompt_text.startswith(bos_token):
287
- start_idx += 1
288
- # Remove the bos_token from the end of prompt,
289
- # because text is empty in this case.
290
- if processed_prompt_text.endswith(bos_token):
291
- end_idx -= 1
292
- processed_outputs[
293
- "input_ids"] = processed_token_ids[:, start_idx:end_idx]
294
- else:
295
- processed_outputs = tokenizer(prompt,
296
- add_special_tokens=False,
297
- return_tensors="pt")
298
- return processed_outputs
299
-
300
- def _get_mm_fields_config(
301
- self,
302
- hf_inputs: BatchFeature,
303
- hf_processor_mm_kwargs: Mapping[str, object],
304
- ) -> Mapping[str, MultiModalFieldConfig]:
305
- return dict(
306
- pixel_values=MultiModalFieldConfig.batched("image"),
307
- aspect_ratio_ids=MultiModalFieldConfig.batched("image"),
308
- aspect_ratio_mask=MultiModalFieldConfig.batched("image"),
309
- num_tiles=MultiModalFieldConfig.batched("image"),
310
- )
311
-
312
- def create_encoder_prompt(
313
- self,
314
- prompt: Union[str, list[int]],
315
- mm_data: MultiModalDataDict,
316
- ) -> Union[str, list[int]]:
317
- data = mm_data.get("image", [])
318
- num_images = 1 if isinstance(data, Image) else len(data)
319
- image_token_id = self.info.get_hf_config().image_token_index
320
- return [image_token_id] * num_images
321
-
322
- def _get_prompt_updates(
323
- self,
324
- mm_items: MultiModalDataItems,
325
- hf_processor_mm_kwargs: Mapping[str, object],
326
- out_mm_kwargs: MultiModalKwargsItems,
327
- ) -> Sequence[PromptUpdate]:
328
- token_per_chunk = self.info.get_token_per_chunk_from_config()
329
- image_token_id = self.info.get_hf_config().image_token_index
330
-
331
- def get_replacement_mllama(item_idx):
332
- images = mm_items.get_items("image", ImageProcessorItems)
333
- image_size = images.get_image_size(item_idx)
334
- num_tile = self.info.get_num_tiles_per_image(
335
- image_height=image_size.height,
336
- image_width=image_size.width,
337
- )
338
- num_tokens = num_tile * token_per_chunk
339
- return [image_token_id] * num_tokens
340
-
341
- return [
342
- PromptReplacement(
343
- modality="image",
344
- target=[image_token_id],
345
- replacement=get_replacement_mllama,
346
- )
347
- ]
348
-
349
-
350
-def _prepare_aspect_ratio_attention_mask(
351
- aspect_ratio_mask: torch.Tensor,
352
- num_patches: int,
353
- target_length: int,
354
- dtype: torch.dtype,
355
-) -> torch.Tensor:
356
- # Expand aspect ratio mask to target_length
357
- batch_size, max_num_tiles = aspect_ratio_mask.shape
358
- attention_mask = aspect_ratio_mask.view(batch_size, max_num_tiles, 1,
359
- 1).to(dtype)
360
- attention_mask = attention_mask.repeat(1, 1, target_length, 1)
361
-
362
- # Mask padding patches
363
- pad_patches = target_length - num_patches
364
- attention_mask[:, :, -pad_patches:] = 0
365
-
366
- # Invert the mask (0 -> 1, 1 -> 0)
367
- attention_mask = 1 - attention_mask
368
-
369
- # Reshape to 2D and create 4D attention mask
370
- # (batch_size, 1, max_num_tiles*target_length, max_num_tiles*target_length)
371
- attention_mask = attention_mask.reshape(batch_size,
372
- max_num_tiles * target_length, 1)
373
- attention_mask = attention_mask @ attention_mask.transpose(
374
- -1, -2) * torch.finfo(dtype).min
375
- attention_mask = attention_mask.unsqueeze(1)
376
-
377
- return attention_mask
378
-
379
-
380
-class ColumnParallelConv2dPatch(torch.nn.Module):
381
- """Conv2D Patching layer with model parallelism.
382
- Column parallel over unfolded input.
383
- Arguments:
384
- in_channels: Input channels.
385
- out_channels: Output channels.
386
- kernel_size: Size of convolution kernel.
387
- stride (default 1): Stride for convolution.
388
- bias (default False): Use bias in Conv2d.
389
- Input: (bsz, in_channels, width, height)
390
- Output: (bsz, num_tokens, out_channels)
391
- """
392
-
393
- def __init__(
394
- self,
395
- in_channels: int,
396
- out_channels: int,
397
- kernel_size: Union[int, tuple[int, int]],
398
- stride: Union[int, tuple[int, int]],
399
- bias: bool = False,
400
- ) -> None:
401
- super().__init__()
402
- if isinstance(kernel_size, int):
403
- kernel_size = (kernel_size, kernel_size)
404
- self._unfold = torch.nn.Unfold(kernel_size=kernel_size, stride=stride)
405
- self._linear = ColumnParallelLinear(
406
- in_channels * kernel_size[0] * kernel_size[1],
407
- out_channels,
408
- bias=bias,
409
- )
410
-
411
- def forward(self, x: torch.Tensor) -> torch.Tensor:
412
- x = self._unfold(x)
413
- x = x.permute(0, 2, 1)
414
- x, _ = self._linear(x)
415
- return x
416
-
417
-
418
-class MllamaPrecomputedAspectRatioEmbedding(nn.Module):
419
-
420
- def __init__(self,
421
- config: config_mllama.MllamaVisionConfig,
422
- is_gated: bool = True):
423
- super().__init__()
424
- self.max_num_tiles = config.max_num_tiles
425
- self.hidden_size = config.hidden_size
426
- self.max_aspect_ratio_id = config.max_aspect_ratio_id
427
- self.is_gated = is_gated
428
-
429
- self.embedding = nn.Embedding(self.max_aspect_ratio_id + 1,
430
- self.max_num_tiles * self.hidden_size)
431
- if is_gated:
432
- self.gate = nn.Parameter(torch.zeros(1))
433
-
434
- def forward(self, hidden_state: torch.Tensor,
435
- aspect_ratio_ids: torch.Tensor) -> torch.Tensor:
436
- embeddings = self.embedding(aspect_ratio_ids)
437
- embeddings = embeddings.reshape(-1, self.max_num_tiles, 1,
438
- self.hidden_size)
439
-
440
- if self.is_gated:
441
- embeddings = embeddings * self.gate.tanh()
442
-
443
- hidden_state = hidden_state + embeddings
444
- return hidden_state
445
-
446
-
447
-class MllamaPrecomputedPositionEmbedding(nn.Module):
448
-
449
- def __init__(self, config: config_mllama.MllamaVisionConfig):
450
- super().__init__()
451
- self.max_num_tiles = config.max_num_tiles
452
- self.max_aspect_ratio_id = config.max_aspect_ratio_id
453
- self.num_patches = (config.image_size // config.patch_size)**2 + 1
454
- self.hidden_size = config.hidden_size
455
- self.scale = config.hidden_size**-0.5
456
-
457
- self.gate = nn.Parameter(torch.zeros(1))
458
-
459
- # position embedding
460
- position_embedding = torch.randn(self.num_patches, self.hidden_size)
461
- self.embedding = nn.Parameter(self.scale * position_embedding)
462
-
463
- # tile position embedding
464
- self.tile_embedding = nn.Embedding(
465
- self.max_aspect_ratio_id + 1,
466
- self.max_num_tiles * self.num_patches * self.hidden_size)
467
-
468
- def forward(self, hidden_state: torch.Tensor,
469
- aspect_ratio_ids: torch.Tensor) -> torch.Tensor:
470
- # position embeddings
471
- gated_position_embedding = (1 - self.gate.tanh()) * self.embedding
472
- hidden_state = hidden_state + gated_position_embedding.view(
473
- 1, 1, self.num_patches, self.hidden_size)
474
-
475
- # precomputed tile position embeddings
476
- tile_position_embedding = self.tile_embedding(aspect_ratio_ids)
477
- batch_size = hidden_state.shape[0]
478
- tile_position_embedding = tile_position_embedding.reshape(
479
- batch_size, self.max_num_tiles, self.num_patches, self.hidden_size)
480
- gated_tile_position_embedding = self.gate.tanh(
481
- ) * tile_position_embedding
482
- hidden_state = hidden_state + gated_tile_position_embedding
483
-
484
- return hidden_state
485
-
486
-
487
-# TODO: support other attention backends for attention in vision model
488
-class MllamaVisionSdpaAttention(nn.Module):
489
-
490
- def __init__(self,
491
- config: config_mllama.MllamaVisionConfig,
492
- quant_config: Optional[QuantizationConfig] = None,
493
- prefix: str = ""):
494
- super().__init__()
495
-
496
- tensor_parallel_size = get_tp_group().world_size
497
- self.embed_dim = config.hidden_size
498
- self.num_heads = config.attention_heads
499
- self.head_dim = config.hidden_size // config.attention_heads
500
- self.num_local_heads = self.num_heads // tensor_parallel_size
501
- self.q_size = self.num_local_heads * self.head_dim
502
- self.kv_size = self.num_local_heads * self.head_dim
503
-
504
- self.qkv_proj = QKVParallelLinear(
505
- self.embed_dim,
506
- self.head_dim,
507
- self.num_heads,
508
- bias=False,
509
- quant_config=quant_config,
510
- prefix=f"{prefix}.qkv_proj",
511
- )
512
- self.o_proj = RowParallelLinear(
513
- self.num_heads * self.head_dim,
514
- self.embed_dim,
515
- bias=False,
516
- input_is_parallel=True,
517
- quant_config=quant_config,
518
- prefix=f"{prefix}.o_proj",
519
- )
520
-
521
- # Use unified MultiHeadAttention with automatic backend selection
522
- self.attn = MultiHeadAttention(self.num_local_heads, self.head_dim,
523
- 1.0 / math.sqrt(self.head_dim))
524
-
525
- def forward(
526
- self,
527
- hidden_state: torch.Tensor,
528
- attention_mask: Optional[torch.Tensor] = None,
529
- ) -> torch.Tensor:
530
- qkv, _ = self.qkv_proj(hidden_state)
531
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
532
-
533
- # Use unified MultiHeadAttention with automatic backend selection
534
- attn_output = self.attn(q, k, v)
535
-
536
- attn_output = attn_output.reshape(attn_output.shape[0],
537
- attn_output.shape[1], -1)
538
- output, _ = self.o_proj(attn_output)
539
- return output
540
-
541
-
542
-class MllamaVisionEncoderLayer(nn.Module):
543
-
544
- def __init__(
545
- self,
546
- config: config_mllama.MllamaVisionConfig,
547
- quant_config: Optional[QuantizationConfig],
548
- prefix: str = "",
549
- is_gated: bool = False,
550
- ) -> None:
551
- super().__init__()
552
-
553
- self.hidden_size = config.hidden_size
554
- self.num_attention_heads = config.attention_heads
555
- self.is_gated = is_gated
556
- self.intermediate_size = config.intermediate_size
557
-
558
- self.self_attn = MllamaVisionSdpaAttention(
559
- config, quant_config=quant_config, prefix=f"{prefix}.self_attn")
560
- self.mlp = CLIPMLP(config,
561
- quant_config=quant_config,
562
- prefix=f"{prefix}.mlp")
563
-
564
- self.input_layernorm = nn.LayerNorm(self.hidden_size,
565
- eps=config.norm_eps)
566
- self.post_attention_layernorm = nn.LayerNorm(self.hidden_size,
567
- eps=config.norm_eps)
568
-
569
- # there used to be an if else here, no code path
570
- if is_gated:
571
- self.gate_attn = nn.Parameter(torch.ones(1) * math.pi / 4)
572
- self.gate_ffn = nn.Parameter(torch.ones(1) * math.pi / 4)
573
-
574
- def forward(
575
- self,
576
- hidden_state: torch.Tensor,
577
- attention_mask: Optional[torch.Tensor] = None,
578
- ):
579
- # Self Attention
580
- residual = hidden_state
581
- hidden_state = self.input_layernorm(hidden_state)
582
- hidden_state = self.self_attn(hidden_state,
583
- attention_mask=attention_mask)
584
- gate_attn = 1 if not self.is_gated else self.gate_attn.tanh()
585
- hidden_state = residual + gate_attn * hidden_state
586
-
587
- # Feed forward
588
- residual = hidden_state
589
- hidden_state = self.post_attention_layernorm(hidden_state)
590
- hidden_state = self.mlp(hidden_state)
591
- gate_ffn = 1 if not self.is_gated else self.gate_ffn.tanh()
592
- hidden_state = residual + gate_ffn * hidden_state
593
-
594
- return hidden_state
595
-
596
-
597
-class MllamaVisionEncoder(nn.Module):
598
-
599
- def __init__(
600
- self,
601
- config: config_mllama.MllamaVisionConfig,
602
- quant_config: Optional[QuantizationConfig],
603
- num_layers: int = 32,
604
- is_gated: bool = False,
605
- output_hidden_states=None,
606
- prefix: str = "",
607
- ) -> None:
608
- super().__init__()
609
- self.config = config
610
- self.layers = nn.ModuleList([
611
- MllamaVisionEncoderLayer(config,
612
- quant_config=quant_config,
613
- is_gated=is_gated,
614
- prefix=f"{prefix}.layers.{layer_idx}")
615
- for layer_idx in range(num_layers)
616
- ])
617
- self.output_hidden_states = output_hidden_states or []
618
-
619
- def forward(
620
- self,
621
- hidden_states: torch.Tensor,
622
- attention_mask: Optional[torch.Tensor] = None,
623
- ) -> Union[BaseModelOutput]:
624
- encoder_states = ()
625
-
626
- for i, encoder_layer in enumerate(self.layers):
627
- if i in self.output_hidden_states:
628
- encoder_states = encoder_states + (hidden_states, )
629
- hidden_states = encoder_layer(
630
- hidden_states,
631
- attention_mask,
632
- )
633
-
634
- if len(self.layers) - 1 in self.output_hidden_states:
635
- encoder_states = encoder_states + (hidden_states, )
636
-
637
- return hidden_states, encoder_states
638
-
639
-
640
-class MllamaVisionModel(nn.Module):
641
-
642
- def __init__(
643
- self,
644
- config: config_mllama.MllamaVisionConfig,
645
- quant_config: Optional[QuantizationConfig],
646
- prefix: str = "",
647
- ) -> None:
648
- super().__init__()
649
-
650
- self.image_size = config.image_size
651
- self.patch_size = config.patch_size
652
- self.max_num_tiles = config.max_num_tiles
653
- self.hidden_size = config.hidden_size
654
- self.in_channels = config.num_channels
655
- self.intermediate_layers_indices = config.intermediate_layers_indices
656
-
657
- self.num_patches = (self.image_size // self.patch_size)**2 + 1
658
- self.scale = config.hidden_size**-0.5
659
-
660
- self.patch_embedding = ColumnParallelConv2dPatch(
661
- in_channels=config.num_channels,
662
- out_channels=self.hidden_size,
663
- kernel_size=self.patch_size,
664
- stride=self.patch_size,
665
- bias=False,
666
- )
667
-
668
- self.class_embedding = nn.Parameter(self.scale *
669
- torch.randn(self.hidden_size))
670
- self.gated_positional_embedding = MllamaPrecomputedPositionEmbedding(
671
- config)
672
-
673
- self.pre_tile_positional_embedding = \
674
- MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True)
675
- self.post_tile_positional_embedding = \
676
- MllamaPrecomputedAspectRatioEmbedding(config, is_gated=True)
677
-
678
- # layer norms
679
- self.layernorm_pre = nn.LayerNorm(self.hidden_size)
680
- self.layernorm_post = nn.LayerNorm(self.hidden_size)
681
-
682
- # encoders
683
- self.transformer = MllamaVisionEncoder(
684
- config,
685
- quant_config,
686
- config.num_hidden_layers,
687
- is_gated=False,
688
- output_hidden_states=config.intermediate_layers_indices,
689
- prefix=f"{prefix}.transformer",
690
- )
691
- self.global_transformer = MllamaVisionEncoder(
692
- config,
693
- quant_config,
694
- config.num_global_layers,
695
- is_gated=True,
696
- prefix=f"{prefix}.global_transformer",
697
- )
698
-
699
- def apply_class_embedding(self,
700
- hidden_state: torch.Tensor) -> torch.Tensor:
701
- batch_size, _, hidden_size = hidden_state.shape
702
- class_embedding = self.class_embedding.expand(batch_size, 1,
703
- hidden_size)
704
- hidden_state = torch.cat([class_embedding, hidden_state], dim=1)
705
- return hidden_state
706
-
707
- def forward(self, pixel_values: torch.Tensor,
708
- aspect_ratio_ids: torch.Tensor,
709
- aspect_ratio_mask: torch.Tensor) -> torch.Tensor:
710
- batch_size, num_concurrent_media, num_tiles, num_channels, \
711
- height, width = pixel_values.shape
712
-
713
- pixel_values = pixel_values.reshape(
714
- batch_size * num_concurrent_media * num_tiles, num_channels,
715
- height, width)
716
- aspect_ratio_ids = aspect_ratio_ids.reshape(
717
- batch_size * num_concurrent_media, -1)
718
-
719
- # patch embedding
720
- patch_embeds = self.patch_embedding(
721
- pixel_values.to(self.layernorm_pre.weight.dtype))
722
- hidden_state = patch_embeds
723
- hidden_state = ps.get_tp_group().all_gather(hidden_state)
724
-
725
- # tile embeddings
726
- _, num_patches, dim = hidden_state.shape
727
- hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
728
- num_tiles, -1, dim)
729
- hidden_state = self.pre_tile_positional_embedding(
730
- hidden_state, aspect_ratio_ids)
731
-
732
- # apply cls token
733
- hidden_state = hidden_state.reshape(
734
- batch_size * num_concurrent_media * num_tiles, num_patches, dim)
735
- hidden_state = self.apply_class_embedding(hidden_state)
736
- num_patches += 1
737
-
738
- # apply position embeddings
739
- hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
740
- num_tiles, num_patches, dim)
741
- hidden_state = self.gated_positional_embedding(hidden_state,
742
- aspect_ratio_ids)
743
-
744
- # apply encoder
745
- hidden_state = self.layernorm_pre(hidden_state)
746
-
747
- # Compute the number of tokens to pad
748
- num_padding_patches = (8 - (hidden_state.shape[-2] % 8)) % 8
749
- # Compute padding tuple for pad function
750
- padding = (
751
- 0, 0, 0, num_padding_patches
752
- ) # (pad_left, pad_right, pad_left for dim -2, pad_right for dim -2)
753
- # Pad the tensor
754
- hidden_state = F.pad(hidden_state, padding, mode="constant", value=0)
755
- slice_index = -num_padding_patches if num_padding_patches > 0 else None
756
-
757
- attention_mask = aspect_ratio_mask.reshape(
758
- batch_size * num_concurrent_media, -1)
759
- attention_mask = _prepare_aspect_ratio_attention_mask(
760
- aspect_ratio_mask=attention_mask,
761
- num_patches=self.num_patches,
762
- target_length=hidden_state.shape[2],
763
- dtype=self.layernorm_pre.weight.dtype,
764
- )
765
-
766
- hidden_state = hidden_state.view(batch_size * num_concurrent_media, -1,
767
- dim)
768
- output = self.transformer(
769
- hidden_state,
770
- attention_mask=attention_mask,
771
- )
772
- hidden_state, intermediate_hidden_states = output[0], output[1]
773
- intermediate_hidden_states = torch.stack(intermediate_hidden_states,
774
- dim=-1)
775
-
776
- # apply global encoder
777
- hidden_state = self.layernorm_post(hidden_state)
778
- hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
779
- num_tiles,
780
- num_patches + num_padding_patches,
781
- dim)
782
- hidden_state = self.post_tile_positional_embedding(
783
- hidden_state, aspect_ratio_ids)
784
- hidden_state = hidden_state.reshape(
785
- batch_size * num_concurrent_media,
786
- num_tiles * (num_patches + num_padding_patches), dim)
787
- hidden_state = self.global_transformer(
788
- hidden_state, attention_mask=attention_mask)[0]
789
- hidden_state = hidden_state.reshape(batch_size * num_concurrent_media,
790
- num_tiles,
791
- num_patches + num_padding_patches,
792
- dim)
793
- hidden_state = hidden_state[:, :, :slice_index]
794
-
795
- # adding intermediate layer outputs
796
- hidden_state = hidden_state.reshape(batch_size, num_concurrent_media,
797
- num_tiles, num_patches, dim)
798
- intermediate_hidden_states = intermediate_hidden_states.reshape(
799
- batch_size * num_concurrent_media, num_tiles,
800
- num_patches + num_padding_patches, -1)
801
- intermediate_hidden_states = intermediate_hidden_states[:, :, :
802
- slice_index]
803
- intermediate_hidden_states = intermediate_hidden_states.reshape(
804
- batch_size, num_concurrent_media, num_tiles, num_patches, -1)
805
- hidden_state = torch.cat([hidden_state, intermediate_hidden_states],
806
- dim=-1)
807
- return hidden_state
808
-
809
- def load_weights(self, weights: Iterable[tuple[str,
810
- torch.Tensor]]) -> set[str]:
811
- stacked_params_mapping = [
812
- # (param_name, shard_name, shard_id)
813
- (".qkv_proj", ".q_proj", "q"),
814
- (".qkv_proj", ".k_proj", "k"),
815
- (".qkv_proj", ".v_proj", "v"),
816
- ]
817
- params_dict = dict(self.named_parameters())
818
- updated_params: set[str] = set()
819
- for name, loaded_weight in weights:
820
- if 'patch_embedding._linear.weight' in name:
821
- loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
822
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
823
- if weight_name not in name:
824
- continue
825
- name = name.replace(weight_name, param_name)
826
- param = params_dict[name]
827
- updated_params.add(name)
828
- weight_loader = param.weight_loader
829
- weight_loader(param, loaded_weight, shard_id)
830
- break
831
- else:
832
- param = params_dict.pop(name)
833
- weight_loader = getattr(param, "weight_loader",
834
- default_weight_loader)
835
- weight_loader(param, loaded_weight)
836
- updated_params.add(name)
837
- return updated_params
838
-
839
-
840
-class MllamaTextRMSNorm(nn.Module):
841
-
842
- def __init__(self, hidden_size, eps=1e-6):
843
- """
844
- MllamaTextRMSNorm is equivalent to T5LayerNorm
845
- """
846
- super().__init__()
847
- self.weight = nn.Parameter(torch.ones(hidden_size))
848
- self.variance_epsilon = eps
849
-
850
- def forward(self, hidden_states):
851
- input_dtype = hidden_states.dtype
852
- hidden_states = hidden_states.to(torch.float32)
853
- variance = hidden_states.pow(2).mean(-1, keepdim=True)
854
- hidden_states = hidden_states * torch.rsqrt(variance +
855
- self.variance_epsilon)
856
- return self.weight * hidden_states.to(input_dtype)
857
-
858
- def extra_repr(self):
859
- return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
860
-
861
-
862
-class MllamaTextCrossAttention(nn.Module):
863
- """Multi-headed attention from 'Attention Is All You Need' paper"""
864
-
865
- def __init__(
866
- self,
867
- config: Optional[config_mllama.MllamaTextConfig] = None,
868
- layer_idx: Optional[int] = None,
869
- quant_config: Optional[QuantizationConfig] = None,
870
- prefix: str = "",
871
- ):
872
- super().__init__()
873
- self.config = config
874
- self.pipeline_parallel_rank = get_pp_group().rank_in_group
875
- self.tensor_parallel_size = get_tp_group().world_size
876
- self.num_heads = config.num_attention_heads
877
- self.num_key_value_heads = config.num_key_value_heads
878
-
879
- self.num_local_heads = self.num_heads // self.tensor_parallel_size
880
- self.num_local_key_value_heads = \
881
- self.num_key_value_heads // self.tensor_parallel_size
882
- self.hidden_size = config.hidden_size
883
- self.head_dim = config.hidden_size // self.num_heads
884
- self.num_key_value_heads = config.num_key_value_heads
885
-
886
- self.layer_idx = layer_idx
887
- self.num_key_value_groups = self.num_heads // self.num_key_value_heads
888
- self.q_local_size = self.num_local_heads * self.head_dim
889
- self.kv_local_size = self.num_local_key_value_heads * self.head_dim
890
-
891
- self.qkv_proj = QKVCrossParallelLinear(
892
- self.hidden_size,
893
- self.head_dim,
894
- self.num_heads,
895
- self.num_key_value_heads,
896
- bias=False,
897
- quant_config=quant_config,
898
- prefix=f"{prefix}.qkv_proj",
899
- )
900
-
901
- self.o_proj = RowParallelLinear(
902
- self.num_heads * self.head_dim,
903
- self.hidden_size,
904
- bias=False,
905
- input_is_parallel=True,
906
- quant_config=quant_config,
907
- prefix=f"{prefix}.o_proj",
908
- )
909
- # vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
910
- # use huggingface's instead
911
- self.q_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
912
- self.k_norm = MllamaTextRMSNorm(self.head_dim, eps=config.rms_norm_eps)
913
- self.scaling = self.head_dim**-0.5
914
-
915
- self.attn = Attention(
916
- self.num_local_heads,
917
- self.head_dim,
918
- self.scaling,
919
- self.num_local_key_value_heads,
920
- prefix=f"{prefix}.attn",
921
- attn_type=AttentionType.ENCODER_DECODER,
922
- )
923
-
924
- def forward(
925
- self,
926
- hidden_states: torch.Tensor,
927
- attention_mask: Optional[torch.Tensor],
928
- kv_range_for_decode: Optional[list[tuple[int, int]]],
929
- cross_attention_states: Optional[torch.Tensor],
930
- ) -> torch.Tensor:
931
- q, k, v = self.qkv_proj(hidden_states, cross_attention_states)
932
- if cross_attention_states is not None:
933
- k = k.view(-1, self.num_local_key_value_heads, self.head_dim)
934
- v = v.view(-1, self.num_local_key_value_heads, self.head_dim)
935
- k = self.k_norm(k)
936
-
937
- q = q.view(-1, self.num_local_heads, self.head_dim)
938
- q = self.q_norm(q)
939
-
940
- if attention_mask is not None:
941
- output = self._attention_with_mask(q, k, v, attention_mask,
942
- kv_range_for_decode)
943
- else:
944
- output = self.attn(
945
- q.view(-1, self.num_local_heads * self.head_dim), k, v)
946
- out, _ = self.o_proj(output)
947
- return out
948
-
949
- def _attention_with_mask(
950
- self,
951
- q: torch.Tensor,
952
- k: torch.Tensor,
953
- v: torch.Tensor,
954
- attention_mask: torch.Tensor,
955
- kv_range_for_decode: list[tuple[int, int]],
956
- ) -> torch.Tensor:
957
- kv_cache = self.attn.kv_cache[self.pipeline_parallel_rank]
958
- attn_metadata: AttentionMetadata = get_forward_context().attn_metadata
959
- # Skip writing kv-cache for the initial profiling run.
960
- # TODO (NickLucche) replace with custom attn bias and use standard attn
961
- if len(kv_cache.shape) > 1:
962
- i = torch.ones(1, dtype=torch.float32)
963
- if self.attn.backend in (_Backend.FLASH_ATTN,
964
- _Backend.FLASH_ATTN_VLLM_V1):
965
- cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
966
- cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
967
- torch.ops._C_cache_ops.reshape_and_cache_flash(
968
- cached_k,
969
- cached_v,
970
- kv_cache[0],
971
- kv_cache[1],
972
- attn_metadata.
973
- cross_slot_mapping, # type: ignore[union-attr]
974
- "auto",
975
- i,
976
- i,
977
- )
978
- elif self.attn.backend in (_Backend.XFORMERS, _Backend.ROCM_FLASH,
979
- _Backend.TORCH_SDPA):
980
- key_cache, value_cache = PagedAttention.split_kv_cache(
981
- kv_cache, self.num_local_key_value_heads, self.head_dim)
982
- cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
983
- cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
984
- PagedAttention.write_to_paged_cache(
985
- cached_k, cached_v, key_cache, value_cache,
986
- attn_metadata.cross_slot_mapping, "auto", i, i)
987
- else:
988
- raise ValueError(
989
- f"Unsupported Attention backend {self.attn.backend} "
990
- "enum found. Expected the Attention backend to be "
991
- "FLASH_ATTN, FLASH_ATTN_VLLM_V1, "
992
- "XFORMERS or TORCH_SDPA.")
993
-
994
- # We have to call torch.sdpa for prefill when using a
995
- # custom cross-attention mask. Because the mask is not a
996
- # standard causal mask, neither a block diagonal mask which
997
- # can be optimized by xformers.BlockDiagonalMask.
998
- # The mask is specially calculated for supporting multi
999
- # images and interleaved images.
1000
- q_len = q.shape[0]
1001
- kv_len = k.shape[0]
1002
- q = q.transpose(0, 1).view(self.num_local_key_value_heads,
1003
- self.num_key_value_groups, q_len,
1004
- self.head_dim).contiguous()
1005
- k = k.transpose(0,
1006
- 1)[:,
1007
- None, :, :].expand(self.num_local_key_value_heads,
1008
- self.num_key_value_groups,
1009
- kv_len,
1010
- self.head_dim).contiguous()
1011
- v = v.transpose(0,
1012
- 1)[:,
1013
- None, :, :].expand(self.num_local_key_value_heads,
1014
- self.num_key_value_groups,
1015
- kv_len,
1016
- self.head_dim).contiguous()
1017
- attention_mask = attention_mask.view(1, 1, q_len, kv_len)
1018
- output = F.scaled_dot_product_attention(q,
1019
- k,
1020
- v,
1021
- attn_mask=attention_mask,
1022
- is_causal=False)
1023
- output = output.permute(2, 0, 1, 3).reshape(
1024
- q_len, self.num_local_heads * self.head_dim)
1025
- return output
1026
-
1027
-
1028
-class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
1029
- """Cross-attention transformer block with tanh-gated attention
1030
- and feedforward."""
1031
-
1032
- def __init__(
1033
- self,
1034
- config: config_mllama.MllamaTextConfig,
1035
- layer_idx: int,
1036
- quant_config: Optional[QuantizationConfig],
1037
- prefix: str = "",
1038
- ) -> None:
1039
- super().__init__()
1040
-
1041
- self.layer_idx = layer_idx
1042
- self.cross_attn = MllamaTextCrossAttention(
1043
- config=config,
1044
- layer_idx=layer_idx,
1045
- quant_config=quant_config,
1046
- prefix=f"{prefix}.cross_attn",
1047
- )
1048
-
1049
- self.input_layernorm = RMSNorm(config.hidden_size,
1050
- eps=config.rms_norm_eps)
1051
- self.cross_attn_attn_gate = torch.nn.Parameter(torch.zeros(1))
1052
-
1053
- self.mlp = LlamaMLP(
1054
- hidden_size=config.hidden_size,
1055
- intermediate_size=config.intermediate_size,
1056
- hidden_act=config.hidden_act,
1057
- quant_config=quant_config,
1058
- prefix=f"{prefix}.mlp",
1059
- )
1060
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
1061
- eps=config.rms_norm_eps)
1062
- self.cross_attn_mlp_gate = torch.nn.Parameter(torch.zeros(1))
1063
-
1064
- def forward(
1065
- self,
1066
- hidden_states: torch.Tensor,
1067
- cross_attention_states: torch.Tensor,
1068
- cross_attention_mask: torch.Tensor,
1069
- kv_range_for_decode: Optional[list[tuple[int, int]]],
1070
- full_text_row_masked_out_mask: torch.Tensor,
1071
- ) -> torch.Tensor:
1072
- residual = hidden_states
1073
- hidden_states = self.input_layernorm(hidden_states)
1074
-
1075
- hidden_states = self.cross_attn(
1076
- hidden_states=hidden_states,
1077
- attention_mask=cross_attention_mask,
1078
- kv_range_for_decode=kv_range_for_decode,
1079
- cross_attention_states=cross_attention_states,
1080
- )
1081
- hidden_states = full_text_row_masked_out_mask * hidden_states
1082
- hidden_states = residual + self.cross_attn_attn_gate.tanh(
1083
- ) * hidden_states
1084
-
1085
- residual = hidden_states
1086
- hidden_states = self.post_attention_layernorm(hidden_states)
1087
- hidden_states = self.mlp(hidden_states)
1088
- hidden_states = full_text_row_masked_out_mask * hidden_states
1089
- hidden_states = residual + self.cross_attn_mlp_gate.tanh(
1090
- ) * hidden_states
1091
- return hidden_states
1092
-
1093
-
1094
-class MllamaTextModel(nn.Module):
1095
- config_class = config_mllama.MllamaTextConfig
1096
- base_model_prefix = "model"
1097
-
1098
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1099
- super().__init__()
1100
-
1101
- config = vllm_config.model_config.hf_config.text_config
1102
- cache_config = vllm_config.cache_config
1103
- quant_config = vllm_config.quant_config
1104
-
1105
- self.vocab_size = config.vocab_size
1106
- self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
1107
- config.hidden_size)
1108
- self.cross_attention_layers = config.cross_attention_layers
1109
-
1110
- layers = []
1111
- for layer_idx in range(config.num_hidden_layers):
1112
- if layer_idx in self.cross_attention_layers:
1113
- layers.append(
1114
- MllamaCrossAttentionDecoderLayer(
1115
- config,
1116
- layer_idx,
1117
- quant_config=quant_config,
1118
- prefix=f"{prefix}.layers.{layer_idx}",
1119
- ))
1120
- else:
1121
- # TODO: force LlamaDecoderLayer to config.attention_bias=False
1122
- layers.append(
1123
- LlamaDecoderLayer(
1124
- config,
1125
- cache_config=cache_config,
1126
- quant_config=quant_config,
1127
- prefix=f"{prefix}.layers.{layer_idx}",
1128
- ))
1129
-
1130
- self.layers = nn.ModuleList(layers)
1131
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
1132
-
1133
- def forward(
1134
- self,
1135
- input_ids: torch.LongTensor,
1136
- positions: Optional[torch.LongTensor],
1137
- cross_attention_states: Optional[torch.LongTensor],
1138
- cross_attention_mask: Optional[torch.LongTensor],
1139
- kv_range_for_decode: Optional[list[tuple[int, int]]],
1140
- full_text_row_masked_out_mask: Optional[tuple[torch.Tensor,
1141
- torch.Tensor]],
1142
- skip_cross_attention: bool,
1143
- ) -> torch.Tensor:
1144
- inputs_embeds = self.embed_tokens(input_ids)
1145
- hidden_states = inputs_embeds
1146
-
1147
- for idx, decoder_layer in enumerate(self.layers):
1148
- if idx in self.cross_attention_layers:
1149
- if not skip_cross_attention:
1150
- hidden_states = decoder_layer(
1151
- hidden_states=hidden_states,
1152
- cross_attention_states=cross_attention_states,
1153
- cross_attention_mask=cross_attention_mask,
1154
- kv_range_for_decode=kv_range_for_decode,
1155
- full_text_row_masked_out_mask=
1156
- full_text_row_masked_out_mask,
1157
- )
1158
- else:
1159
- hidden_states, residual = decoder_layer(
1160
- positions=positions,
1161
- hidden_states=hidden_states,
1162
- residual=None,
1163
- )
1164
- hidden_states = hidden_states + residual
1165
- hidden_states = self.norm(hidden_states)
1166
- return hidden_states
1167
-
1168
-
1169
-class MllamaForCausalLM(nn.Module):
1170
- config_class = config_mllama.MllamaTextConfig
1171
- base_model_prefix = "language_model"
1172
- _no_split_modules = [
1173
- "MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
1174
- ]
1175
-
1176
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1177
- super().__init__()
1178
-
1179
- config = vllm_config.model_config.hf_config.text_config
1180
- quant_config = vllm_config.quant_config
1181
- self.quant_config = quant_config
1182
-
1183
- self.vocab_size = config.vocab_size
1184
- self.model = MllamaTextModel(vllm_config=vllm_config,
1185
- prefix=f"{prefix}.model")
1186
- self.lm_head = ParallelLMHead(
1187
- config.vocab_size,
1188
- config.hidden_size,
1189
- org_num_embeddings=config.vocab_size,
1190
- padding_size=DEFAULT_VOCAB_PADDING_SIZE,
1191
- quant_config=quant_config,
1192
- prefix=f"{prefix}.lm_head",
1193
- )
1194
-
1195
- def forward(
1196
- self,
1197
- input_ids: torch.LongTensor,
1198
- positions: Optional[torch.LongTensor],
1199
- cross_attention_states: Optional[torch.LongTensor],
1200
- cross_attention_mask: Optional[torch.LongTensor],
1201
- kv_range_for_decode: Optional[list[tuple[int, int]]],
1202
- full_text_row_masked_out_mask: Optional[tuple[torch.Tensor,
1203
- torch.Tensor]],
1204
- skip_cross_attention: bool,
1205
- ) -> torch.Tensor:
1206
- hidden_states = self.model(
1207
- input_ids=input_ids,
1208
- positions=positions,
1209
- cross_attention_states=cross_attention_states,
1210
- cross_attention_mask=cross_attention_mask,
1211
- kv_range_for_decode=kv_range_for_decode,
1212
- full_text_row_masked_out_mask=full_text_row_masked_out_mask,
1213
- skip_cross_attention=skip_cross_attention,
1214
- )
1215
- return hidden_states
1216
-
1217
- def load_weights(self, weights: Iterable[tuple[str,
1218
- torch.Tensor]]) -> set[str]:
1219
- stacked_params_mapping = [
1220
- # (param_name, shard_name, shard_id)
1221
- (".qkv_proj", ".q_proj", "q"),
1222
- (".qkv_proj", ".k_proj", "k"),
1223
- (".qkv_proj", ".v_proj", "v"),
1224
- (".gate_up_proj", ".gate_proj", 0),
1225
- (".gate_up_proj", ".up_proj", 1),
1226
- ]
1227
- params_dict = dict(self.named_parameters())
1228
- updated_params: set[str] = set()
1229
- for name, loaded_weight in weights:
1230
- if 'patch_embedding.weight' in name:
1231
- name = name.replace('patch_embedding.weight',
1232
- 'patch_embedding._linear.weight')
1233
- loaded_weight = loaded_weight.view(loaded_weight.shape[0], -1)
1234
- if (self.quant_config is not None and
1235
- (scale_name := self.quant_config.get_cache_scale(name))):
1236
- # Loading kv cache quantization scales
1237
- param = params_dict[scale_name]
1238
- weight_loader = getattr(param, "weight_loader",
1239
- default_weight_loader)
1240
- loaded_weight = (loaded_weight if loaded_weight.dim() == 0 else
1241
- loaded_weight[0])
1242
- weight_loader(param, loaded_weight)
1243
- updated_params.add(scale_name)
1244
- continue
1245
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
1246
- if weight_name not in name:
1247
- continue
1248
- name = name.replace(weight_name, param_name)
1249
- param = params_dict[name]
1250
- updated_params.add(name)
1251
- weight_loader = param.weight_loader
1252
- weight_loader(param, loaded_weight, shard_id)
1253
- break
1254
- else:
1255
- orig_name = name
1256
- name = maybe_remap_kv_scale_name(name, params_dict)
1257
- if name is None:
1258
- logger.debug("Missing name %s, orig name %s", name,
1259
- orig_name)
1260
- continue
1261
-
1262
- param = params_dict.pop(name)
1263
- weight_loader = getattr(param, "weight_loader",
1264
- default_weight_loader)
1265
- weight_loader(param, loaded_weight)
1266
- updated_params.add(name)
1267
- return updated_params
1268
-
1269
-
1270
-@MULTIMODAL_REGISTRY.register_processor(MllamaMultiModalProcessor,
1271
- info=MllamaProcessingInfo,
1272
- dummy_inputs=MllamaDummyInputsBuilder)
1273
-class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,
1274
- SupportsV0Only):
1275
- packed_modules_mapping = {
1276
- "qkv_proj": ["q_proj", "k_proj", "v_proj"],
1277
- "gate_up_proj": ["gate_proj", "up_proj"]
1278
- }
1279
-
1280
- hf_to_vllm_mapper = WeightsMapper(
1281
- orig_to_new_prefix={
1282
- # mapping for new names in checkpoint saved after transformers v4.52
1283
- "model.vision_model.": "vision_model.",
1284
- "model.multi_modal_projector.": "multi_modal_projector.",
1285
- "model.language_model.": "language_model.model.",
1286
- "lm_head.": "language_model.lm_head.",
1287
- },
1288
- orig_to_new_suffix={
1289
- "patch_embedding.weight": "patch_embedding._linear.weight",
1290
- },
1291
- )
1292
-
1293
- @classmethod
1294
- def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
1295
- if modality.startswith("image"):
1296
- return "<|image|>"
1297
-
1298
- raise ValueError("Only image modality is supported")
1299
-
1300
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1301
- super().__init__()
1302
- config: MllamaConfig = vllm_config.model_config.hf_config
1303
- quant_config = vllm_config.quant_config
1304
- self.config = config
1305
- self.quant_config = quant_config
1306
- self.vocab_size = config.text_config.vocab_size
1307
- self.hidden_size = config.text_config.hidden_size
1308
- self.max_num_tiles = config.vision_config.max_num_tiles
1309
- self.vision_output_dim = config.vision_config.vision_output_dim
1310
- self.pad_token_id = \
1311
- config.pad_token_id if config.pad_token_id is not None else -1
1312
- self.image_size = config.vision_config.image_size
1313
- self.image_token_id = config.image_token_index
1314
-
1315
- self.vision_model = MllamaVisionModel(config.vision_config,
1316
- quant_config,
1317
- prefix=maybe_prefix(
1318
- prefix, "vision_model"))
1319
- self.language_model = MllamaForCausalLM(
1320
- vllm_config=vllm_config,
1321
- prefix=maybe_prefix(prefix, "language_model"),
1322
- )
1323
- self.multi_modal_projector = ColumnParallelLinear(
1324
- config.vision_config.vision_output_dim,
1325
- config.text_config.hidden_size,
1326
- bias=True,
1327
- quant_config=quant_config,
1328
- gather_output=True,
1329
- prefix=maybe_prefix(prefix, "multi_modal_projector"),
1330
- )
1331
- self.logits_processor = LogitsProcessor(config.output_hidden_states,
1332
- config.text_config.vocab_size)
1333
-
1334
- def compute_logits(
1335
- self,
1336
- hidden_states: torch.Tensor,
1337
- sampling_metadata: SamplingMetadata,
1338
- ) -> Optional[torch.Tensor]:
1339
- logits = self.logits_processor(self.language_model.lm_head,
1340
- hidden_states, sampling_metadata)
1341
- return logits
1342
-
1343
- def unpack_data(self,
1344
- image_data: Union[list[torch.Tensor], torch.Tensor],
1345
- padding_value=0) -> torch.Tensor:
1346
- if isinstance(image_data, torch.Tensor):
1347
- # torch.Tensor
1348
- return image_data
1349
- else:
1350
- assert isinstance(
1351
- image_data[0],
1352
- torch.Tensor), "Image data is not properly batched."
1353
- # list[torch.Tensor]
1354
- bsz = len(image_data)
1355
- max_length = max(t.size(0) for t in image_data)
1356
- trailing_dims = image_data[0].shape[1:]
1357
- for data in image_data:
1358
- cur_trailing_dims = data.shape[1:]
1359
- assert cur_trailing_dims == trailing_dims
1360
- output_tensor = torch.full((bsz, max_length, *trailing_dims),
1361
- padding_value,
1362
- dtype=image_data[0].dtype,
1363
- device=image_data[0].device)
1364
- for i, t in enumerate(image_data):
1365
- output_tensor[i, :t.size(0)] = t
1366
- return output_tensor
1367
-
1368
- def _parse_and_validate_image_input(
1369
- self, **kwargs: object) -> Optional[MllamaImagePixelInputs]:
1370
- # tensor with the same shape will be batched together by
1371
- # MultiModalKwargs.batch, so pixel_values here can be:
1372
- # - list[torch.Tensor]:
1373
- # with shape (num_image, num_tiles, 3, image_res, image_res)
1374
- # - torch.Tensor:
1375
- # with shape (bs, num_image, num_tiles, 3, image_res, image_res)
1376
- pixel_values: Optional[Union[list[list[torch.Tensor]],
1377
- list[torch.Tensor],
1378
- torch.Tensor]] = kwargs.pop(
1379
- "pixel_values", None)
1380
- image_embeds: Optional[Union[list[list[torch.Tensor]],
1381
- list[torch.Tensor],
1382
- torch.Tensor]] = kwargs.pop(
1383
- "image_embeds", None)
1384
- aspect_ratio_ids: Optional[Union[list[list[torch.Tensor]],
1385
- list[torch.Tensor],
1386
- torch.Tensor]] = kwargs.pop(
1387
- "aspect_ratio_ids", None)
1388
- aspect_ratio_mask: Optional[Union[list[list[torch.Tensor]],
1389
- list[torch.Tensor],
1390
- torch.Tensor]] = kwargs.pop(
1391
- "aspect_ratio_mask", None)
1392
-
1393
- if pixel_values is None and image_embeds is None:
1394
- return None
1395
-
1396
- if pixel_values is not None and image_embeds is not None:
1397
- raise ValueError(
1398
- "Both pixel values and image embeds are provided.")
1399
-
1400
- if pixel_values is not None:
1401
- assert aspect_ratio_ids is not None
1402
- assert aspect_ratio_mask is not None
1403
-
1404
- return MllamaImagePixelInputs(
1405
- type="pixel_values",
1406
- data=self.unpack_data(pixel_values),
1407
- aspect_ratio_ids=self.unpack_data(aspect_ratio_ids),
1408
- aspect_ratio_mask=self.unpack_data(aspect_ratio_mask))
1409
-
1410
- if image_embeds is not None:
1411
- raise NotImplementedError
1412
-
1413
- raise AssertionError("This line should be unreachable.")
1414
-
1415
- def _get_and_validate_encoder_lens(
1416
- self,
1417
- encoder_seq_lens: list[int],
1418
- num_tiles: list[list[int]],
1419
- num_tokens_per_tile: int,
1420
- ) -> list[int]:
1421
- # Get the actual number of encoder tokens for each sample.
1422
- # Because attn_metadata.encoder_seq_lens only counts the last
1423
- # group of images for each sample, which is used to cheat the
1424
- # block manager to allocate blocks for those images only.
1425
- # See MllamaMultiModalProcessor for more details.
1426
- actual_encoder_seq_lens = [
1427
- sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
1428
- ]
1429
-
1430
- # remove 0 encoder len entries for text-only requests for these
1431
- # assertions
1432
- attn_metadata_lens = [x for x in encoder_seq_lens if x > 0]
1433
- assert len(actual_encoder_seq_lens) == len(attn_metadata_lens)
1434
- for actual_len, last_group_len in zip(actual_encoder_seq_lens,
1435
- attn_metadata_lens):
1436
- assert actual_len >= last_group_len
1437
-
1438
- return actual_encoder_seq_lens
1439
-
1440
- def flat_encoder_result(self, cross_attention_states: torch.Tensor,
1441
- attn_metadata: AttentionMetadata,
1442
- actual_encoder_seq_lens: list[int]):
1443
-
1444
- cross_attention_states_flat = torch.zeros(
1445
- sum(actual_encoder_seq_lens),
1446
- cross_attention_states.shape[-1],
1447
- device=cross_attention_states.device,
1448
- dtype=cross_attention_states.dtype)
1449
- start_pos = 0
1450
- for seq_len, vision_token_in_batch in zip(actual_encoder_seq_lens,
1451
- cross_attention_states):
1452
- end_pos = start_pos + seq_len
1453
- cross_attention_states_flat[
1454
- start_pos:end_pos] = vision_token_in_batch[:seq_len]
1455
- start_pos = end_pos
1456
- cross_attention_states = cross_attention_states_flat
1457
- return cross_attention_states
1458
-
1459
- def get_language_model(self) -> torch.nn.Module:
1460
- return self.language_model
1461
-
1462
- def get_cross_attention_states(
1463
- self,
1464
- image_inputs: MllamaImagePixelInputs,
1465
- attn_metadata: AttentionMetadata,
1466
- actual_encoder_seq_lens: list[int],
1467
- ) -> tuple[torch.Tensor]:
1468
- # NOTE: llama's reference implementation runs vision model on CPU
1469
- pixel_values = image_inputs['data']
1470
- aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1471
- aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1472
- cross_attention_states = self.vision_model(pixel_values,
1473
- aspect_ratio_ids,
1474
- aspect_ratio_mask)
1475
- cross_attention_states, _ = self.multi_modal_projector(
1476
- cross_attention_states)
1477
-
1478
- bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1479
- cross_attention_states = cross_attention_states.view(
1480
- bsz, -1, image_token_dim)
1481
-
1482
- cross_attention_states = self.flat_encoder_result(
1483
- cross_attention_states, attn_metadata, actual_encoder_seq_lens)
1484
-
1485
- return cross_attention_states
1486
-
1487
- def get_cross_attention_mask(
1488
- self,
1489
- input_ids: torch.Tensor,
1490
- attn_metadata: AttentionMetadata,
1491
- num_tiles: list[list[int]],
1492
- num_tokens_per_tile: int,
1493
- dtype: torch.dtype,
1494
- ) -> tuple[torch.Tensor, torch.Tensor]:
1495
- token_ids = input_ids.tolist()
1496
- start = 0
1497
- batch_token_ids = []
1498
- for seq_len in attn_metadata.seq_lens:
1499
- batch_token_ids.append(token_ids[start:start + seq_len])
1500
- start += seq_len
1501
- sparse_mask = [
1502
- get_cross_attention_token_mask(t, self.image_token_id)
1503
- for t in batch_token_ids
1504
- ]
1505
-
1506
- # Skip generating cross-attention mask if all samples
1507
- # are text-only or have only 1 leading image.
1508
- if skip_attention_mask(sparse_mask):
1509
- return None, None
1510
-
1511
- dense_mask, tile_range_for_decode = \
1512
- convert_sparse_cross_attention_mask_to_dense(
1513
- sparse_mask, num_tiles, attn_metadata.seq_lens)
1514
- cross_attention_mask = \
1515
- convert_dense_cross_attention_mask_to_tensor(
1516
- dense_mask, num_tokens_per_tile, input_ids.device, dtype)
1517
- kv_range_for_decode = [[
1518
- t[0] * num_tokens_per_tile, t[1] * num_tokens_per_tile
1519
- ] for t in tile_range_for_decode]
1520
-
1521
- return cross_attention_mask, kv_range_for_decode
1522
-
1523
- def get_full_text_row_masked_out_mask(
1524
- self,
1525
- attn_metadata: AttentionMetadata,
1526
- device: torch.device,
1527
- ) -> torch.Tensor:
1528
- full_text_row_masked_out_mask = torch.ones(
1529
- (attn_metadata.num_prefill_tokens, 1), dtype=torch.bool)
1530
- start_pos = 0
1531
- for seq_len, encoder_seq_len in zip(attn_metadata.seq_lens,
1532
- attn_metadata.encoder_seq_lens):
1533
- if encoder_seq_len == 0:
1534
- full_text_row_masked_out_mask[start_pos:start_pos +
1535
- seq_len] = False
1536
- start_pos += seq_len
1537
- full_text_row_masked_out_mask = full_text_row_masked_out_mask.to(
1538
- device)
1539
- return full_text_row_masked_out_mask
1540
-
1541
- def forward(
1542
- self,
1543
- input_ids: torch.Tensor,
1544
- positions: torch.Tensor,
1545
- **kwargs: object,
1546
- ) -> Union[CausalLMOutputWithPast]:
1547
- attn_metadata = get_forward_context().attn_metadata
1548
- if attn_metadata.num_prefill_tokens > 0 and \
1549
- attn_metadata.num_decode_tokens > 0:
1550
- raise ValueError("Chunk prefill not supported")
1551
- image_inputs = self._parse_and_validate_image_input(**kwargs)
1552
- cross_attention_states = None
1553
- cross_attention_mask = None
1554
- kv_range_for_decode = None
1555
-
1556
- # For 1) text-only prefill and decode, 2) image-present decode.
1557
- if image_inputs is None:
1558
- full_text_row_masked_out_mask = (
1559
- attn_metadata.encoder_seq_lens_tensor
1560
- != 0).reshape(-1, 1).to(input_ids.device)
1561
- skip_cross_attention = attn_metadata.max_encoder_seq_len == 0
1562
-
1563
- # For image-present prefill.
1564
- else:
1565
- skip_cross_attention = False
1566
-
1567
- num_tiles = [t.tolist() for t in kwargs.pop("num_tiles")]
1568
- num_tokens_per_tile = calc_token_per_chunk(self.image_size)
1569
-
1570
- actual_encoder_seq_lens = self._get_and_validate_encoder_lens(
1571
- attn_metadata.encoder_seq_lens,
1572
- num_tiles,
1573
- num_tokens_per_tile,
1574
- )
1575
-
1576
- cross_attention_states = self.get_cross_attention_states(
1577
- image_inputs, attn_metadata, actual_encoder_seq_lens)
1578
-
1579
- full_text_row_masked_out_mask = \
1580
- self.get_full_text_row_masked_out_mask(
1581
- attn_metadata, input_ids.device)
1582
-
1583
- cross_attention_mask, kv_range_for_decode = \
1584
- self.get_cross_attention_mask(
1585
- input_ids, attn_metadata, num_tiles,
1586
- num_tokens_per_tile, cross_attention_states.dtype)
1587
-
1588
- outputs = self.language_model(
1589
- input_ids=input_ids,
1590
- positions=positions,
1591
- cross_attention_states=cross_attention_states,
1592
- cross_attention_mask=cross_attention_mask,
1593
- kv_range_for_decode=kv_range_for_decode,
1594
- full_text_row_masked_out_mask=full_text_row_masked_out_mask,
1595
- skip_cross_attention=skip_cross_attention,
1596
- )
1597
-
1598
- return outputs
1599
-
1600
- def load_weights(self, weights: Iterable[tuple[str,
1601
- torch.Tensor]]) -> set[str]:
1602
- loader = AutoWeightsLoader(self)
1603
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1604
-
1605
- def get_mm_mapping(self) -> MultiModelKeys:
1606
- """
1607
- Get the module prefix in multimodal models
1608
- """
1609
- return MultiModelKeys.from_string_field(
1610
- language_model="language_model",
1611
- connector="multi_modal_projector",
1612
- tower_model="vision_model")
1613
-
1614
-
1615
-def skip_attention_mask(sparse_mask: list[list[int]]) -> bool:
1616
- for mask in sparse_mask:
1617
- # Skip text-only samples.
1618
- if len(mask) == 0:
1619
- continue
1620
- # If the sample contains more than 1 images,
1621
- # we can't skip mask.
1622
- if len(mask) != 1:
1623
- return False
1624
- # If the sample contains only 1 image,
1625
- # but the image is not the leading one,
1626
- # we can't skip mask.
1627
- if mask[0][0] != 0 or mask[0][1] != -1:
1628
- return False
1629
- return True
1630
-
1631
-
1632
-def convert_sparse_cross_attention_mask_to_dense(
1633
- sparse_mask: list[list[list[int]]],
1634
- num_tiles: list[list[int]],
1635
- lengths: list[int],
1636
-) -> tuple[np.ndarray, list[tuple[int, int]]]:
1637
- total_length = sum(lengths)
1638
- total_tiles = sum([sum(tiles) for tiles in num_tiles])
1639
- dense_mask = np.zeros(shape=(total_length, total_tiles), dtype=np.int64)
1640
- # A list of ranges, range[i] = [start, end] means that the i-th image will
1641
- # use tiles[start, end] for cross-attention decoding.
1642
- tile_range_for_decode = []
1643
-
1644
- seq_start = 0
1645
- tile_start = 0
1646
-
1647
- # sparse_mask has an [] entry for each sequence that does not have images,
1648
- # but num_tiles does not have these entries...
1649
- num_tiles_idx = 0
1650
- for masks, length in zip(sparse_mask, lengths):
1651
- if len(masks) == 0:
1652
- # Text only
1653
- continue
1654
-
1655
- tiles = num_tiles[num_tiles_idx]
1656
- num_tiles_idx += 1
1657
- ts, td = -1, 0
1658
- for mask, tile in zip(masks, tiles):
1659
- if len(mask) != 2:
1660
- continue
1661
- start, end = mask
1662
- end = min(end, length)
1663
- if end == -1:
1664
- end = length
1665
- if end == length:
1666
- if ts == -1:
1667
- ts = tile_start
1668
- td += tile
1669
- dense_mask[seq_start + start:seq_start + end,
1670
- tile_start:tile_start + tile] = 1
1671
- tile_start += tile
1672
- assert ts != -1
1673
- assert td != 0
1674
- tile_range_for_decode.append((ts, ts + td))
1675
- seq_start += length
1676
- assert num_tiles_idx == len(num_tiles)
1677
-
1678
- return dense_mask, tile_range_for_decode
1679
-
1680
-
1681
-def convert_dense_cross_attention_mask_to_tensor(
1682
- cross_attention_token_mask: np.ndarray,
1683
- num_tokens_per_tile: int,
1684
- device: torch.device,
1685
- dtype: torch.dtype,
1686
-) -> torch.Tensor:
1687
- mask = torch.tensor(cross_attention_token_mask, dtype=dtype, device=device)
1688
- mask = mask.repeat_interleave(num_tokens_per_tile, dim=1)
1689
-
1690
- mask = 1.0 - mask
1691
- mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(dtype).min)
1692
-
1693
- ninf = torch.finfo(dtype).min
1694
- full_text_mask = ((mask != ninf).any(dim=-1).type_as(mask)[..., None])
1695
- mask *= full_text_mask
1696
- # (num_prompt_tokens, num_encoder_tokens)
1697
- return mask
vllm/model_executor/models/registry.py
+2
-9
@@ -147,10 +147,6 @@ _TEXT_GENERATION_MODELS = {
147
"TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
148
"XverseForCausalLM": ("llama", "LlamaForCausalLM"),
149
"Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
150
- # [Encoder-decoder]
151
- "BartModel": ("bart", "BartForConditionalGeneration"),
152
- "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
153
- "MBartForConditionalGeneration": ("bart", "MBartForConditionalGeneration"),
150
}
151
152
_EMBEDDING_MODELS = {
@@ -237,6 +233,7 @@ _MULTIMODAL_MODELS = {
233
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
234
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"), # noqa: E501
235
"Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
236
+ "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"), # noqa: E501
237
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
238
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
239
"LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
@@ -263,16 +260,12 @@ _MULTIMODAL_MODELS = {
260
"Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
261
"Qwen2_5OmniForConditionalGeneration": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
262
"UltravoxModel": ("ultravox", "UltravoxModel"),
263
+ "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
264
"Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"), # noqa: E501
265
"TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501
266
"Tarsier2ForConditionalGeneration": ("qwen2_vl", "Tarsier2ForConditionalGeneration"), # noqa: E501
267
"VoxtralForConditionalGeneration": ("voxtral", "VoxtralForConditionalGeneration"), # noqa: E501
268
# [Encoder-decoder]
271
- "DonutForConditionalGeneration": ("donut", "DonutForConditionalGeneration"),
272
- "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
273
- "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
274
- "Llama4ForConditionalGeneration": ("mllama4", "Llama4ForConditionalGeneration"), # noqa: E501
275
- "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
269
"WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501
270
}
271
vllm/multimodal/profiling.py
+1
-1
@@ -209,7 +209,7 @@ class MultiModalProfiler(Generic[_I]):
209
if processor.pad_dummy_encoder_prompt:
210
num_tokens_to_pad = max(total_len, seq_len) - total_len
211
encoder_prompt_token_ids.extend([0] * num_tokens_to_pad)
212
- # NOTE: Whisper and Donut allows total_len > seq_len.
212
+ # NOTE: Whisper allows total_len > seq_len.
213
elif total_len > seq_len and not envs.VLLM_USE_V1:
214
# `max_num_batched_tokens` is defined by `SchedulerConfig`
215
logger.warning_once(
vllm/test_utils.py
-1
@@ -36,7 +36,6 @@ MODELS_ON_S3 = [
36
"llava-hf/llava-v1.6-mistral-7b-hf",
37
"llava-hf/LLaVA-NeXT-Video-7B-hf",
38
# "meta-llama/Llama-2-7b-hf",
39
- "meta-llama/Llama-3.2-11B-Vision-Instruct",
39
"meta-llama/Llama-3.2-1B",
40
"meta-llama/Llama-3.2-1B-Instruct",
41
"meta-llama/Meta-Llama-3-8B",
vllm/transformers_utils/chat_templates/registry.py
-1
@@ -35,7 +35,6 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
35
"blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
36
"chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
37
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
38
- "florence2": CHAT_TEMPLATES_DIR / "template_basic.jinja",
38
"fuyu": CHAT_TEMPLATES_DIR / "template_fuyu.jinja",
39
"minicpmv": _get_minicpmv_chat_template_fallback,
40
"paligemma": CHAT_TEMPLATES_DIR / "template_basic.jinja",
vllm/transformers_utils/config.py
-5
@@ -90,11 +90,6 @@ _AUTO_CONFIG_KWARGS_OVERRIDES: dict[str, dict[str, Any]] = {
90
"internvl_chat": {
91
"has_no_defaults_at_init": True
92
},
93
- # transformers regards mllama as is_encoder_decoder=False
94
- # vllm needs is_encoder_decoder=True to enable cross-attention
95
- "mllama": {
96
- "is_encoder_decoder": True
97
- },
93
"NVLM_D": {
94
"has_no_defaults_at_init": True
95
},
vllm/v1/engine/processor.py
+1
-1
@@ -498,7 +498,7 @@ class Processor:
498
assert isinstance(mm_processor, EncDecMultiModalProcessor)
499
500
if mm_processor.pad_dummy_encoder_prompt:
501
- return # Skip encoder length check for Whisper and Donut
501
+ return # Skip encoder length check for Whisper
502
503
if model_config.is_multimodal_model:
504
suggestion = (
vllm/worker/enc_dec_model_runner.py
deleted
-553
@@ -1,553 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import dataclasses
5
-import itertools
6
-from typing import Any, Dict, List, Optional, Tuple, Type, cast
7
-
8
-import torch
9
-import torch.distributed
10
-
11
-from vllm.attention.backends.abstract import (AttentionBackend,
12
- AttentionMetadata)
13
-from vllm.attention.backends.utils import PAD_SLOT_ID
14
-from vllm.attention.selector import (get_env_variable_attn_backend,
15
- get_global_forced_attn_backend)
16
-from vllm.config import VllmConfig
17
-from vllm.forward_context import set_forward_context
18
-from vllm.inputs import INPUT_REGISTRY, InputRegistry
19
-from vllm.logger import init_logger
20
-from vllm.lora.request import LoRARequest
21
-from vllm.model_executor import SamplingMetadata
22
-from vllm.model_executor.layers.sampler import SamplerOutput
23
-from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
24
- MultiModalRegistry)
25
-from vllm.platforms import _Backend
26
-from vllm.sampling_params import SamplingParams
27
-from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
28
-from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
29
-from vllm.worker.model_runner import (GPUModelRunnerBase,
30
- ModelInputForGPUBuilder,
31
- ModelInputForGPUWithSamplingMetadata)
32
-from vllm.worker.model_runner_base import (
33
- _add_attn_metadata_broadcastable_dict,
34
- _add_sampling_metadata_broadcastable_dict)
35
-from vllm.worker.utils import assert_enc_dec_mr_supported_scenario
36
-
37
-logger = init_logger(__name__)
38
-LORA_WARMUP_RANK = 8
39
-
40
-
41
-@dataclasses.dataclass(frozen=True)
42
-class EncoderDecoderModelInput(ModelInputForGPUWithSamplingMetadata):
43
- """
44
- Used by the EncoderDecoderModelRunner.
45
- """
46
- encoder_input_tokens: Optional[torch.Tensor] = None
47
- encoder_input_positions: Optional[torch.Tensor] = None
48
-
49
- def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
50
- tensor_dict = {
51
- "input_tokens": self.input_tokens,
52
- "inputs_embeds": self.inputs_embeds,
53
- "input_positions": self.input_positions,
54
- "encoder_input_tokens": self.encoder_input_tokens,
55
- "encoder_input_positions": self.encoder_input_positions,
56
- "virtual_engine": self.virtual_engine,
57
- "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
58
- "finished_requests_ids": self.finished_requests_ids,
59
- "multi_modal_kwargs": self.multi_modal_kwargs,
60
- }
61
- _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
62
- _add_sampling_metadata_broadcastable_dict(tensor_dict,
63
- self.sampling_metadata)
64
- return tensor_dict
65
-
66
- @classmethod
67
- def from_broadcasted_tensor_dict(
68
- cls,
69
- tensor_dict: Dict[str, Any],
70
- attn_backend: Optional["AttentionBackend"] = None,
71
- ) -> "EncoderDecoderModelInput":
72
- return cast(
73
- EncoderDecoderModelInput,
74
- super().from_broadcasted_tensor_dict(tensor_dict, attn_backend))
75
-
76
-
77
-class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
78
- _model_input_cls: Type[EncoderDecoderModelInput] = (
79
- EncoderDecoderModelInput)
80
- _builder_cls: Type[ModelInputForGPUBuilder] = (ModelInputForGPUBuilder)
81
-
82
- def __init__(
83
- self,
84
- vllm_config: VllmConfig,
85
- kv_cache_dtype: Optional[str] = "auto",
86
- is_driver_worker: bool = False,
87
- input_registry: InputRegistry = INPUT_REGISTRY,
88
- mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
89
- ):
90
- '''
91
- EncoderDecoderModelRunner constructor.
92
-
93
- `lora_config` is unused (since these features are not yet supported
94
- for encoder/decoder models) but these arguments are present here for
95
- compatibility with the base-class constructor.
96
- '''
97
- self._maybe_force_supported_attention_backend()
98
-
99
- super().__init__(
100
- vllm_config=vllm_config,
101
- kv_cache_dtype=kv_cache_dtype,
102
- is_driver_worker=is_driver_worker,
103
- input_registry=input_registry,
104
- mm_registry=mm_registry,
105
- )
106
-
107
- # Crash for unsupported encoder/scenarios
108
- assert_enc_dec_mr_supported_scenario(self)
109
-
110
- def _maybe_force_supported_attention_backend(self):
111
- '''
112
- Force vLLM to use the XFormers attention backend,
113
- which is currently the only supported option.
114
- '''
115
-
116
- def raise_backend_err():
117
- # The user has specified an attention backend override
118
- # which is invalid for encoder/decoder models
119
- raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_BACKEND)
120
-
121
- maybe_env_var_forced_backend = get_env_variable_attn_backend()
122
- maybe_global_forced_backend = get_global_forced_attn_backend()
123
- is_forced_by_global = maybe_global_forced_backend is not None
124
- is_forced_by_env_var = maybe_env_var_forced_backend is not None
125
- if is_forced_by_global: # noqa: SIM102
126
- # Backend override enforced by global variable takes
127
- # precedence over vLLM backend environment variable.
128
- if maybe_global_forced_backend not in\
129
- [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
130
- raise_backend_err()
131
- elif is_forced_by_env_var: # noqa: SIM102
132
- # Backend override enforced by vLLM backend
133
- # environment variable
134
- if maybe_env_var_forced_backend not in\
135
- [_Backend.XFORMERS, _Backend.FLASH_ATTN]:
136
- raise_backend_err()
137
-
138
- def _list_to_int32_tensor(
139
- self,
140
- _list: List[int],
141
- ) -> torch.Tensor:
142
- return torch.tensor(_list, dtype=torch.int32, device=self.device)
143
-
144
- def _list_to_long_tensor(
145
- self,
146
- _list: List[int],
147
- ) -> torch.Tensor:
148
- return torch.tensor(_list, dtype=torch.long, device=self.device)
149
-
150
- def _empty_int32_tensor(self) -> torch.Tensor:
151
- return self._list_to_int32_tensor([])
152
-
153
- def _empty_long_tensor(self) -> torch.Tensor:
154
- return self._list_to_long_tensor([])
155
-
156
- @torch.inference_mode()
157
- def execute_model(
158
- self,
159
- model_input: EncoderDecoderModelInput,
160
- kv_caches: List[torch.Tensor],
161
- intermediate_tensors: Optional[IntermediateTensors] = None,
162
- num_steps: int = 1,
163
- ) -> Optional[List[SamplerOutput]]:
164
- if num_steps > 1:
165
- raise ValueError("num_steps > 1 is not supported in "
166
- "EncoderDecoderModelRunner")
167
- if self.lora_config:
168
- assert model_input.lora_requests is not None
169
- assert model_input.lora_mapping is not None
170
- self.set_active_loras(model_input.lora_requests,
171
- model_input.lora_mapping)
172
- if (model_input.attn_metadata is not None
173
- and model_input.attn_metadata.prefill_metadata is None
174
- and model_input.attn_metadata.decode_metadata.use_cuda_graph):
175
- if model_input.inputs_embeds is None:
176
- assert model_input.input_tokens is not None
177
- graph_batch_size = model_input.input_tokens.shape[0]
178
- model_executable = (
179
- self.graph_runners[model_input.virtual_engine][(
180
- graph_batch_size, False)])
181
- else:
182
- graph_batch_size = model_input.inputs_embeds.shape[0]
183
- model_executable = (
184
- self.graph_runners[model_input.virtual_engine][(
185
- graph_batch_size, True)])
186
- else:
187
- model_executable = self.model
188
-
189
- seqlen_agnostic_kwargs = {
190
- "finished_requests_ids": model_input.finished_requests_ids,
191
- "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
192
- } if self.has_inner_state else {}
193
-
194
- multi_modal_kwargs = model_input.multi_modal_kwargs or {}
195
- with set_forward_context(model_input.attn_metadata, self.vllm_config,
196
- model_input.virtual_engine):
197
- hidden_or_intermediate_states = model_executable(
198
- input_ids=model_input.input_tokens,
199
- inputs_embeds=model_input.inputs_embeds,
200
- positions=model_input.input_positions,
201
- encoder_input_ids=model_input.encoder_input_tokens,
202
- encoder_positions=model_input.encoder_input_positions,
203
- intermediate_tensors=intermediate_tensors,
204
- **MultiModalKwargs.as_kwargs(
205
- multi_modal_kwargs,
206
- device=self.device,
207
- ),
208
- **seqlen_agnostic_kwargs,
209
- )
210
-
211
- logits = self.model.compute_logits(hidden_or_intermediate_states,
212
- model_input.sampling_metadata)
213
-
214
- if not self.is_driver_worker:
215
- return []
216
-
217
- if model_input.async_callback is not None:
218
- model_input.async_callback()
219
-
220
- # Sample the next token.
221
- output: SamplerOutput = self.sampler(
222
- logits=logits,
223
- sampling_metadata=model_input.sampling_metadata,
224
- )
225
-
226
- return [output]
227
-
228
- def make_model_input_from_broadcasted_tensor_dict(
229
- self, tensor_dict: Dict[str, Any]) -> EncoderDecoderModelInput:
230
- return EncoderDecoderModelInput.from_broadcasted_tensor_dict(
231
- tensor_dict,
232
- attn_backend=self.attn_backend,
233
- )
234
-
235
- def prepare_model_input(
236
- self,
237
- seq_group_metadata_list: List[SequenceGroupMetadata],
238
- virtual_engine: int = 0,
239
- finished_requests_ids: Optional[List[str]] = None
240
- ) -> EncoderDecoderModelInput:
241
- """Prepare the model input based on a given sequence group, including
242
- metadata for the sampling step.
243
-
244
- Since chunked prefill is not supported for encoder/decoder models,
245
- `input_tokens` is assumed to be either entirely prefill tokens or
246
- entirely decode tokens.
247
-
248
- """
249
- model_input = self._prepare_model_input_tensors(
250
- seq_group_metadata_list, finished_requests_ids)
251
- (
252
- attn_metadata,
253
- encoder_input_tokens_tensor,
254
- encoder_input_positions_tensor,
255
- ) = (self._prepare_encoder_model_input_tensors(seq_group_metadata_list,
256
- model_input))
257
- # Inject attn_metadata encoder/cross-attention fields &
258
- # encoder input tokens/positions into model_input.
259
- # Frozen dataclass fields cannot be modified, so use
260
- # dataclasses.replace to construct a new model input
261
- # instance.
262
- model_input = dataclasses.replace(
263
- model_input,
264
- attn_metadata=attn_metadata,
265
- encoder_input_tokens=encoder_input_tokens_tensor,
266
- encoder_input_positions=encoder_input_positions_tensor,
267
- )
268
-
269
- generators = self.get_generators(finished_requests_ids)
270
- sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
271
- model_input.seq_lens,
272
- model_input.query_lens,
273
- self.device,
274
- self.pin_memory,
275
- generators=generators)
276
- is_prompt = (seq_group_metadata_list[0].is_prompt
277
- if seq_group_metadata_list else None)
278
- return dataclasses.replace(model_input,
279
- sampling_metadata=sampling_metadata,
280
- is_prompt=is_prompt,
281
- virtual_engine=virtual_engine)
282
-
283
- @torch.inference_mode()
284
- def profile_run(self) -> None:
285
- # Enable top-k sampling to reflect the accurate memory usage.
286
- sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
287
- max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
288
- max_num_seqs = self.scheduler_config.max_num_seqs
289
-
290
- # This represents the maximum number of different requests
291
- # that will have unique loras, and therefore the max amount of
292
- # memory consumption. Create dummy lora request copies from the
293
- # lora request passed in, which contains a lora from the lora
294
- # warmup path.
295
- dummy_lora_requests: List[LoRARequest] = []
296
- dummy_lora_requests_per_seq: List[LoRARequest] = []
297
- if self.lora_config:
298
- dummy_lora_requests = self._add_dummy_loras(
299
- self.lora_config.max_loras)
300
- assert len(dummy_lora_requests) == self.lora_config.max_loras
301
- dummy_lora_requests_per_seq = [
302
- dummy_lora_requests[idx % len(dummy_lora_requests)]
303
- for idx in range(max_num_seqs)
304
- ]
305
-
306
- # Profile memory usage with max_num_sequences sequences and the total
307
- # number of tokens equal to max_num_batched_tokens.
308
- seqs: List[SequenceGroupMetadata] = []
309
-
310
- max_mm_tokens = self.mm_registry.get_max_multimodal_tokens(
311
- self.model_config)
312
- if max_mm_tokens > 0:
313
- logger.info("Starting profile run for multi-modal models.")
314
-
315
- batch_size = 0
316
- for group_id in range(max_num_seqs):
317
- seq_len = (max_num_batched_tokens // max_num_seqs +
318
- (group_id < max_num_batched_tokens % max_num_seqs))
319
- batch_size += seq_len
320
-
321
- decoder_dummy_data = self.input_registry \
322
- .dummy_data_for_profiling(self.model_config,
323
- seq_len,
324
- self.mm_registry,
325
- is_encoder_data=False)
326
- encoder_dummy_data = self.input_registry \
327
- .dummy_data_for_profiling(self.model_config,
328
- seq_len,
329
- self.mm_registry,
330
- is_encoder_data=True)
331
-
332
- # Having more tokens is over-conservative but otherwise fine
333
- assert len(
334
- decoder_dummy_data.seq_data.prompt_token_ids
335
- ) >= seq_len, (
336
- f"Expected at least {seq_len} dummy tokens for profiling, "
337
- f"but got: {len(decoder_dummy_data.seq_data.prompt_token_ids)}"
338
- )
339
-
340
- assert decoder_dummy_data.multi_modal_data is None or \
341
- encoder_dummy_data.multi_modal_data is None, (
342
- "Multi-modal data can't be provided in both encoder and decoder"
343
- )
344
-
345
- seq = SequenceGroupMetadata(
346
- request_id=str(group_id),
347
- is_prompt=True,
348
- seq_data={group_id: decoder_dummy_data.seq_data},
349
- sampling_params=sampling_params,
350
- block_tables=None,
351
- encoder_seq_data=encoder_dummy_data.seq_data,
352
- cross_block_table=None,
353
- lora_request=dummy_lora_requests_per_seq[group_id]
354
- if dummy_lora_requests_per_seq else None,
355
- multi_modal_data=decoder_dummy_data.multi_modal_data
356
- or encoder_dummy_data.multi_modal_data,
357
- multi_modal_placeholders=decoder_dummy_data.
358
- multi_modal_placeholders
359
- or encoder_dummy_data.multi_modal_placeholders)
360
- seqs.append(seq)
361
-
362
- finished_requests_ids = [seq.request_id for seq in seqs]
363
- model_input = self.prepare_model_input(
364
- seqs, finished_requests_ids=finished_requests_ids)
365
- intermediate_tensors = None
366
- self.execute_model(model_input, None, intermediate_tensors)
367
- torch.cuda.synchronize()
368
- return
369
-
370
- def _prepare_encoder_model_input_tensors(
371
- self,
372
- seq_group_metadata_list: List[SequenceGroupMetadata],
373
- model_input: EncoderDecoderModelInput,
374
- ) -> Tuple[AttentionMetadata, Optional[torch.Tensor],
375
- Optional[torch.Tensor]]:
376
- """Helper method to prepare the encoder- and cross-attn-related
377
- model inputs based on a given sequence group. These additional inputs
378
- are used to augment an already-computed `EncoderDecoderModelInput`
379
- data structure which already has decoder-related model inputs
380
- populated.
381
-
382
- Sets the following attn_metadata fields:
383
- * `num_encoder_tokens`
384
- * `encoder_seq_lens`
385
- * `encoder_seq_lens_tensor`
386
- * `max_encoder_seq_len`
387
- * `cross_slot_mapping`
388
- * `cross_block_tables`
389
-
390
- Constructs a new model inputs data structure, based on
391
- (1) the existing fields in the `model_inputs` argument,
392
- and (2) the following additional fields which are
393
- computed (or in the case of `attn_metadata`, updated)
394
- by this function:
395
- * attn_metadata
396
- * encoder_input_tokens
397
- * encoder_input_positions
398
-
399
- Arguments:
400
-
401
- * seq_group_metadata_list: list of sequence groups for which to
402
- compute inputs
403
- * model_inputs: model inputs data structure with decoder-oriented
404
- fields already computed.
405
-
406
- Return:
407
-
408
- * Updated model inputs data structure
409
- """
410
-
411
- if len(seq_group_metadata_list) == 0:
412
- return (model_input.attn_metadata, None, None)
413
-
414
- # Since we are not supporting chunked prefill either the entire
415
- # batch is prefill or it is decode
416
- is_prompt = seq_group_metadata_list[0].is_prompt
417
-
418
- # Build encoder inputs
419
- encoder_seq_lens: List[int] = []
420
- if is_prompt:
421
- # Prefill phase.
422
- cross_block_tables = self._empty_int32_tensor().view(
423
- len(seq_group_metadata_list), -1)
424
-
425
- # Extract input tokens/positions, cross-attention slot-mapping,
426
- # & seq len from each sequence group metadata
427
- (
428
- encoder_input_tokens,
429
- encoder_input_positions,
430
- cross_slot_mapping,
431
- ) = (
432
- [],
433
- [],
434
- [],
435
- )
436
- for seq_group_metadata in seq_group_metadata_list:
437
- # Build seq lens
438
- seq_len = seq_group_metadata.encoder_seq_data.get_len()
439
- token_ids = seq_group_metadata.encoder_seq_data.get_token_ids()
440
- encoder_seq_lens.append(seq_len)
441
-
442
- # Build slot mapping
443
- is_profile_run = (seq_group_metadata.block_tables is None)
444
- if is_profile_run:
445
- # During memory profiling, the block tables are not
446
- # initialized yet. In this case, we just use a dummy
447
- # slot mapping.
448
- # In embeddings, the block tables are {seq_id: None}.
449
- cross_slot_mapping.extend([PAD_SLOT_ID] * seq_len)
450
- else:
451
- for i in range(0, seq_len):
452
- block_number = seq_group_metadata.cross_block_table[
453
- i // self.block_size]
454
- block_offset = i % self.block_size
455
- slot = block_number * self.block_size + block_offset
456
- cross_slot_mapping.append(slot)
457
-
458
- # Build encoder input tokens
459
- encoder_input_tokens.extend(token_ids)
460
- encoder_input_positions.extend(list(range(0, seq_len)))
461
-
462
- # Convert tokens/positions & cross-attention
463
- # slot-mapping to encoder input tensors
464
- encoder_input_tokens_tensor = self._list_to_long_tensor(
465
- encoder_input_tokens)
466
- encoder_input_positions_tensor = self._list_to_long_tensor(
467
- encoder_input_positions)
468
- cross_slot_mapping_tensor = self._list_to_long_tensor(
469
- cross_slot_mapping)
470
-
471
- else:
472
- # Decode phase.
473
- encoder_input_tokens_tensor = self._empty_long_tensor()
474
- encoder_input_positions_tensor = self._empty_long_tensor()
475
- cross_slot_mapping_tensor = self._empty_long_tensor()
476
- # Extract cross-attention block tables &
477
- # seq len from each sequence group metadata.
478
- # Cross-attention block tables are empty
479
- # during vLLM memory profiling.
480
- cross_block_tables = []
481
- for seq_group_metadata in seq_group_metadata_list:
482
- for _ in range(len(seq_group_metadata.seq_data)):
483
- encoder_seq_lens.append(
484
- seq_group_metadata.encoder_seq_data.get_len())
485
- cross_block_table = seq_group_metadata.cross_block_table
486
- cross_block_tables.append([] if (
487
- cross_block_table is None) else cross_block_table)
488
-
489
- if (model_input.attn_metadata is not None
490
- and model_input.attn_metadata.use_cuda_graph):
491
- # We will be using CUDA graph replay for this decode.
492
- max_len_of_block_table = self.get_max_block_per_batch()
493
- batch_size = len(encoder_seq_lens)
494
- graph_batch_size = self.vllm_config.pad_for_cudagraph(
495
- batch_size)
496
- assert graph_batch_size >= batch_size
497
- cuda_graph_pad_size = graph_batch_size - batch_size
498
- # extend the cross_block_tables and encoder_seq_lens to match
499
- # the graph_batch_size.
500
- cross_block_tables.extend([[]
501
- for _ in range(cuda_graph_pad_size)
502
- ])
503
- encoder_seq_lens.extend(
504
- itertools.repeat(1, cuda_graph_pad_size))
505
-
506
- else:
507
- max_len_of_block_table = max(
508
- len(block_table) for block_table in cross_block_tables)
509
-
510
- cross_block_tables = make_tensor_with_pad(
511
- cross_block_tables,
512
- max_len=max_len_of_block_table,
513
- pad=0,
514
- dtype=torch.int32,
515
- device=self.device,
516
- )
517
-
518
- # Compute encoder sequence lengths & encoder
519
- # sequence starting offset tensors
520
- max_encoder_seq_len = max(encoder_seq_lens, default=0)
521
- encoder_seq_lens_tensor = self._list_to_int32_tensor(encoder_seq_lens)
522
- encoder_seq_start_loc = torch.zeros(encoder_seq_lens_tensor.shape[0] +
523
- 1,
524
- dtype=torch.int32,
525
- device=self.device)
526
- torch.cumsum(encoder_seq_lens_tensor,
527
- dim=0,
528
- dtype=encoder_seq_start_loc.dtype,
529
- out=encoder_seq_start_loc[1:])
530
-
531
- # Update attention metadata with encoder-oriented attributes
532
- attn_metadata = model_input.attn_metadata
533
- assert attn_metadata is not None
534
- (
535
- attn_metadata.num_encoder_tokens,
536
- attn_metadata.encoder_seq_lens,
537
- attn_metadata.encoder_seq_lens_tensor,
538
- attn_metadata.max_encoder_seq_len,
539
- attn_metadata.encoder_seq_start_loc,
540
- attn_metadata.cross_slot_mapping,
541
- attn_metadata.cross_block_tables,
542
- ) = (
543
- sum(encoder_seq_lens),
544
- encoder_seq_lens,
545
- encoder_seq_lens_tensor,
546
- max_encoder_seq_len,
547
- encoder_seq_start_loc,
548
- cross_slot_mapping_tensor,
549
- cross_block_tables,
550
- )
551
-
552
- return (attn_metadata, encoder_input_tokens_tensor,
553
- encoder_input_positions_tensor)
vllm/worker/utils.py
deleted
-49
@@ -1,49 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-'''
4
-Worker-related helper functions.
5
-'''
6
-
7
-from vllm.utils import STR_NOT_IMPL_ENC_DEC_ERR_STRS
8
-from vllm.worker.model_runner import GPUModelRunnerBase
9
-
10
-
11
-def assert_enc_dec_mr_supported_scenario(
12
- enc_dec_mr: GPUModelRunnerBase) -> None:
13
- '''
14
- Asserted that the provided encoder/decoder model runner instance reflects
15
- a supported scenario.
16
- '''
17
-
18
- # Reminder: Please update docs/features/compatibility_matrix.md
19
- # If the feature combo become valid
20
-
21
- if enc_dec_mr.cache_config.enable_prefix_caching:
22
- raise NotImplementedError(
23
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PREFIX_CACHE'])
24
-
25
- if enc_dec_mr.sliding_window is not None:
26
- raise NotImplementedError(
27
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SWA'])
28
-
29
- if enc_dec_mr.scheduler_config.chunked_prefill_enabled:
30
- raise NotImplementedError(STR_NOT_IMPL_ENC_DEC_ERR_STRS[
31
- 'STR_NOT_IMPL_ENC_DEC_CHUNKED_PREFILL'])
32
-
33
- if getattr(enc_dec_mr.model_config.hf_config, 'attn_logit_softcapping',
34
- None) is not None:
35
- raise NotImplementedError(
36
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_LOGIT_SOFTCAP']
37
- )
38
-
39
- if enc_dec_mr.lora_config is not None:
40
- raise NotImplementedError(
41
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_LORA'])
42
-
43
- if enc_dec_mr.parallel_config.pipeline_parallel_size > 1:
44
- raise NotImplementedError(
45
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_PP'])
46
-
47
- if enc_dec_mr.scheduler_config.num_lookahead_slots > 0:
48
- raise NotImplementedError(
49
- STR_NOT_IMPL_ENC_DEC_ERR_STRS['STR_NOT_IMPL_ENC_DEC_SPEC_DEC'])
vllm/worker/worker.py
+1
-5
@@ -28,7 +28,6 @@ from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
28
from vllm.utils import (GiB_bytes, MemorySnapshot, bind_kv_cache,
29
memory_profiling)
30
from vllm.worker.cache_engine import CacheEngine
31
-from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
31
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
32
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
33
WorkerInput)
@@ -82,10 +81,7 @@ class Worker(LocalOrDistributedWorkerBase):
81
"qwen3_next_mtp")) \
82
else {"return_hidden_states": True}
83
85
- ModelRunnerClass: Type[GPUModelRunnerBase] = ModelRunner
86
- if self.model_config.is_encoder_decoder:
87
- ModelRunnerClass = EncoderDecoderModelRunner
88
- self.model_runner: GPUModelRunnerBase = ModelRunnerClass(
84
+ self.model_runner: GPUModelRunnerBase = ModelRunner(
85
vllm_config=self.vllm_config,
86
kv_cache_dtype=self.cache_config.cache_dtype,
87
is_driver_worker=is_driver_worker,