68ea76e780 · tduncan/Nvidia-Optimization-Stack-VLLM

@@ -1,438 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -"""

4 -Consolidated test for ViT attention backend functionality across multiple models.

5 -

6 -This test validates that each multimodal model can successfully generate outputs

7 -using different ViT attention backends. Tests are parametrized by model and backend.

8 -"""

9 -

10 -from typing import Any

11 -

12 -import pytest

13 -from transformers import AutoProcessor

14 -

15 -from vllm import LLM, SamplingParams

16 -from vllm.multimodal.utils import encode_image_url

17 -from vllm.multimodal.video import sample_frames_from_video

18 -from vllm.platforms import current_platform

19 -from vllm.v1.attention.backends.registry import AttentionBackendEnum

20 -

21 -from ....utils import create_new_process_for_each_test

22 -from ...utils import dummy_hf_overrides

23 -

24 -# Dots.OCR prompt from official repository

25 -# https://github.com/rednote-hilab/dots.ocr/blob/d72d1d8c5bdd0362eb264f714cdbd1e5daa7cdff/dots_ocr/utils/prompts.py#L3

26 -# ruff: noqa: E501

27 -DOTS_OCR_PROMPT = """Please output the layout information from the PDF image, including each layout element's bbox, its category, and the corresponding text content within the bbox.

28 -

29 -1. Bbox format: [x1, y1, x2, y2]

30 -

31 -2. Layout Categories: The possible categories are ['Caption', 'Footnote', 'Formula', 'List-item', 'Page-footer', 'Page-header', 'Picture', 'Section-header', 'Table', 'Text', 'Title'].

32 -

33 -3. Text Extraction & Formatting Rules:

34 - - Picture: For the 'Picture' category, the text field should be omitted.

35 - - Formula: Format its text as LaTeX.

36 - - Table: Format its text as HTML.

37 - - All Others (Text, Title, etc.): Format their text as Markdown.

38 -

39 -4. Constraints:

40 - - The output text must be the original text from the image, with no translation.

41 - - All layout elements must be sorted according to human reading order.

42 -

43 -5. Final Output: The entire output must be a single JSON object.

44 -"""

45 -

46 -VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"

47 -

48 -

49 -# Model configurations

50 -MODEL_CONFIGS: dict[str, dict[str, Any]] = {

51 - "dots_ocr": {

52 - "model_name": "rednote-hilab/dots.ocr",

53 - "interface": "llm_chat",

54 - "max_model_len": 32768,

55 - "max_num_seqs": 1,

56 - "limit_mm_per_prompt": {"image": 1},

57 - "sampling_params": {

58 - "temperature": 0.1,

59 - "max_tokens": 16384,

60 - "top_p": 0.9,

61 - "stop_token_ids": None,

62 - },

63 - "use_specific_image": "stop_sign",

64 - "prompt_builder": "build_dots_ocr_prompt",

65 - "output_validator": lambda x: len(x) > 10 and "stop" in x.lower(),

66 - },

67 - "ernie45_vl": {

68 - "model_name": "baidu/ERNIE-4.5-VL-28B-A3B-PT",

69 - "interface": "llm_generate",

70 - "max_model_len": 16384,

71 - "max_num_seqs": 2,

72 - "sampling_params": {

73 - "temperature": 0.0,

74 - "max_tokens": 256,

75 - "stop_token_ids": None,

76 - },

77 - "use_processor": True,

78 - "question": "What is the content of each image?",

79 - },

80 - "glm4_1v": {

81 - "model_name": "zai-org/GLM-4.1V-9B-Thinking",

82 - "interface": "llm_generate",

83 - "max_model_len": 32768,

84 - "max_num_seqs": 2,

85 - "sampling_params": {

86 - "temperature": 0.0,

87 - "max_tokens": 256,

88 - "stop_token_ids": None,

89 - },

90 - "use_processor": True,

91 - "question": "What is the content of each image?",

92 - },

93 - "glm_ocr": {

94 - "model_name": "zai-org/GLM-OCR",

95 - "interface": "llm_generate",

96 - "max_model_len": 131072,

97 - "max_num_seqs": 2,

98 - "sampling_params": {

99 - "temperature": 0.0,

100 - "max_tokens": 256,

101 - "stop_token_ids": None,

102 - },

103 - "use_processor": True,

104 - "question": "Text Recognition:",

105 - },

106 - "keye_vl": {

107 - "model_name": "Kwai-Keye/Keye-VL-8B-Preview",

108 - "interface": "llm_generate",

109 - "max_model_len": 8192,

110 - "max_num_seqs": 5,

111 - "sampling_params": {

112 - "temperature": 0.0,

113 - "max_tokens": 256,

114 - "stop_token_ids": None,

115 - },

116 - "supported_backends": {

117 - AttentionBackendEnum.FLASH_ATTN,

118 - AttentionBackendEnum.ROCM_AITER_FA,

119 - },

120 - "use_processor": True,

121 - "question": "What is the content of each image?",

122 - },

123 - "ovis2_5": {

124 - "model_name": "AIDC-AI/Ovis2.5-2B",

125 - "interface": "llm_generate",

126 - "max_model_len": 8192,

127 - "max_num_seqs": 2,

128 - "sampling_params": {

129 - "temperature": 0.0,

130 - "max_tokens": 256,

131 - "stop_token_ids": None,

132 - },

133 - "prompt_builder": "build_ovis_prompt",

134 - "question": "What is the content of each image?",

135 - },

136 - "qwen2_5_vl": {

137 - "model_name": "Qwen/Qwen2.5-VL-3B-Instruct",

138 - "interface": "vllm_runner",

139 - "media_type": "video",

140 - "max_model_len": 4000,

141 - "max_num_seqs": 1,

142 - "limit_mm_per_prompt": {"video": 1},

143 - "sampling_params": {

144 - "max_tokens": 128,

145 - },

146 - "runner_kwargs": {

147 - "runner": "generate",

148 - "dtype": "bfloat16",

149 - },

150 - "video_params": {

151 - "num_frames": 16,

152 - "pruning_rates": [0.0, 0.75],

153 - },

154 - },

155 - "qwen2_5_omni": {

156 - "model_name": "Qwen/Qwen2.5-Omni-3B",

157 - "interface": "llm_generate",

158 - "max_model_len": 32768,

159 - "max_num_seqs": 2,

160 - "limit_mm_per_prompt": {"image": 3, "video": 3, "audio": 3},

161 - "sampling_params": {

162 - "temperature": 0.6,

163 - "top_p": 0.95,

164 - "top_k": 20,

165 - "max_tokens": 16384,

166 - },

167 - "use_processor": True,

168 - "question": "What is the content of each image?",

169 - },

170 - "qwen3_omni": {

171 - "model_name": "Qwen/Qwen3-Omni-30B-A3B-Instruct",

172 - "interface": "llm_generate",

173 - "max_model_len": 32768,

174 - "max_num_seqs": 2,

175 - "limit_mm_per_prompt": {"image": 3, "video": 3, "audio": 3},

176 - "sampling_params": {

177 - "temperature": 0.6,

178 - "top_p": 0.95,

179 - "top_k": 20,

180 - "max_tokens": 16384,

181 - },

182 - "use_processor": True,

183 - "question": "What is the content of each image?",

184 - },

185 -}

186 -

187 -

188 -# Prompt builder functions

189 -def build_dots_ocr_prompt(images, config):

190 - """Build Dots.OCR specific prompt with OCR instructions."""

191 - # Use only stop_sign image for Dots.OCR

192 - image = images[0] # Already filtered to stop_sign

193 - image_url = encode_image_url(image)

194 -

195 - placeholders = [{"type": "image_url", "image_url": {"url": image_url}}]

196 - messages = [

197 - {

198 - "role": "user",

199 - "content": [

200 - *placeholders,

201 - {

202 - "type": "text",

203 - "text": f"<|img|><|imgpad|><|endofimg|>{DOTS_OCR_PROMPT}",

204 - },

205 - ],

206 - },

207 - ]

208 -

209 - return messages

210 -

211 -

212 -def build_processor_prompt(images, config):

213 - """Build prompt using AutoProcessor.apply_chat_template()."""

214 - processor = AutoProcessor.from_pretrained(

215 - config["model_name"], trust_remote_code=True

216 - )

217 -

218 - image_urls = [encode_image_url(img) for img in images]

219 - placeholders = [{"type": "image", "image": url} for url in image_urls]

220 - messages = [

221 - {

222 - "role": "user",

223 - "content": [

224 - *placeholders,

225 - {"type": "text", "text": config["question"]},

226 - ],

227 - },

228 - ]

229 -

230 - return processor.apply_chat_template(

231 - messages, tokenize=False, add_generation_prompt=True

232 - )

233 -

234 -

235 -def build_ovis_prompt(images, config):

236 - """Build Ovis2.5 specific prompt with custom format."""

237 - image_urls = [encode_image_url(img) for img in images]

238 -

239 - placeholders = "\n".join(

240 - f"Image-{i}: <image>\n" for i, _ in enumerate(image_urls, start=1)

241 - )

242 -

243 - return (

244 - f"<|im_start|>user\n\n{placeholders}\n{config['question']}<|im_end|>\n"

245 - "<|im_start|>assistant\n"

246 - )

247 -

248 -

249 -def build_qwen2_5_video_prompt():

250 - """Build Qwen2.5-VL video prompt with EVS placeholder."""

251 - return (

252 - f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"

253 - f"<|im_start|>user\n{VIDEO_PLACEHOLDER}"

254 - "Describe this video with a short sentence (no more than 20 words)"

255 - "<|im_end|><|im_start|>assistant\n"

256 - )

257 -

258 -

259 -# Handler functions

260 -def run_llm_generate_test(config, mm_encoder_attn_backend, image_assets):

261 - """Standard LLM.generate() interface handler."""

262 - images = [asset.pil_image for asset in image_assets]

263 -

264 - # Build prompt

265 - if config.get("use_processor"):

266 - prompt = build_processor_prompt(images, config)

267 - else:

268 - prompt_builder_name = config.get("prompt_builder", "build_ovis_prompt")

269 - prompt_builder = globals()[prompt_builder_name]

270 - prompt = prompt_builder(images, config)

271 -

272 - # Determine limit_mm_per_prompt

273 - limit_mm_per_prompt = config.get("limit_mm_per_prompt", {"image": len(images)})

274 -

275 - # Create engine

276 - llm = LLM(

277 - model=config["model_name"],

278 - trust_remote_code=True,

279 - max_model_len=config["max_model_len"],

280 - max_num_seqs=config["max_num_seqs"],

281 - limit_mm_per_prompt=limit_mm_per_prompt,

282 - mm_encoder_attn_backend=mm_encoder_attn_backend,

283 - hf_overrides=dummy_hf_overrides,

284 - load_format="dummy",

285 - seed=42,

286 - )

287 -

288 - # Generate

289 - sampling_params = SamplingParams(**config["sampling_params"])

290 - outputs = llm.generate(

291 - {

292 - "prompt": prompt,

293 - "multi_modal_data": {"image": images},

294 - },

295 - sampling_params=sampling_params,

296 - )

297 -

298 - # Validate

299 - for o in outputs:

300 - generated_text = o.outputs[0].text

301 - validator = config.get("output_validator", lambda x: len(x) > 10)

302 - assert validator(generated_text), (

303 - f"Validation failed for {config['model_name']}: {generated_text}"

304 - )

305 -

306 -

307 -def run_llm_chat_test(config, mm_encoder_attn_backend, image_assets):

308 - """LLM.chat() interface handler for Dots.OCR."""

309 - # Filter to stop_sign image only

310 - stop_sign_image = [

311 - asset.pil_image for asset in image_assets if asset.name == "stop_sign"

312 - ][0]

313 -

314 - # Build messages

315 - messages = build_dots_ocr_prompt([stop_sign_image], config)

316 -

317 - # Create engine

318 - llm = LLM(

319 - model=config["model_name"],

320 - trust_remote_code=True,

321 - max_model_len=config["max_model_len"],

322 - max_num_seqs=config["max_num_seqs"],

323 - limit_mm_per_prompt=config["limit_mm_per_prompt"],

324 - mm_encoder_attn_backend=mm_encoder_attn_backend,

325 - hf_overrides=dummy_hf_overrides,

326 - load_format="dummy",

327 - seed=42,

328 - )

329 -

330 - # Generate using chat

331 - sampling_params = SamplingParams(**config["sampling_params"])

332 - outputs = llm.chat(messages=messages, sampling_params=sampling_params)

333 -

334 - # Validate

335 - for o in outputs:

336 - generated_text = o.outputs[0].text

337 - validator = config.get("output_validator", lambda x: len(x) > 10)

338 - assert validator(generated_text), (

339 - f"Validation failed for {config['model_name']}: {generated_text}"

340 - )

341 -

342 -

343 -def run_video_test(config, mm_encoder_attn_backend, video_assets, vllm_runner):

344 - """Video test with EVS (Efficient Video Sampling) handler."""

345 - for pruning_rate in config["video_params"]["pruning_rates"]:

346 - num_frames = config["video_params"]["num_frames"]

347 -

348 - # Sample frames from video

349 - sampled_vids = [

350 - sample_frames_from_video(asset.np_ndarrays, num_frames)

351 - for asset in video_assets

352 - ]

353 -

354 - # Build prompt and prepare video

355 - prompt = build_qwen2_5_video_prompt()

356 - prompts = [prompt]

357 - videos = [sampled_vids[0]]

358 -

359 - # Run with vllm_runner context manager

360 - with vllm_runner(

361 - config["model_name"],

362 - max_model_len=config["max_model_len"],

363 - max_num_seqs=config["max_num_seqs"],

364 - limit_mm_per_prompt=config["limit_mm_per_prompt"],

365 - tensor_parallel_size=1,

366 - video_pruning_rate=pruning_rate,

367 - mm_encoder_attn_backend=mm_encoder_attn_backend,

368 - hf_overrides=dummy_hf_overrides,

369 - load_format="dummy",

370 - **config["runner_kwargs"],

371 - ) as vllm_model:

372 - outputs = vllm_model.generate_greedy(

373 - prompts,

374 - config["sampling_params"]["max_tokens"],

375 - videos=videos,

376 - )

377 -

378 - # Validate output

379 - assert len(outputs) == 1, f"Expected 1 output, got {len(outputs)}"

380 - output_ids, output_text = outputs[0]

381 - assert len(output_ids) > 0, "Generated no output IDs"

382 - assert len(output_text) > 0, "Generated empty text"

383 - assert isinstance(output_text, str), (

384 - f"Output is not string: {type(output_text)}"

385 - )

386 -

387 -

388 -# Main test function

389 -@pytest.mark.parametrize("model_key", list(MODEL_CONFIGS.keys()))

390 -@pytest.mark.parametrize(

391 - "mm_encoder_attn_backend",

392 - [None] + current_platform.get_supported_vit_attn_backends(),

393 -)

394 -@pytest.mark.skip(reason="Broken test due to memory segmentation fault")

395 -@create_new_process_for_each_test()

396 -def test_vit_backend_functionality(

397 - model_key: str,

398 - mm_encoder_attn_backend: AttentionBackendEnum | None,

399 - image_assets,

400 - video_assets,

401 - vllm_runner,

402 - request,

403 -):

404 - """Test ViT attention backend functionality for multimodal models.

405 -

406 - This test validates that each model can successfully generate outputs

407 - using different ViT attention backends. The test:

408 - 1. Filters unsupported backends per model

409 - 2. Applies appropriate GPU marks

410 - 3. Routes to the correct test handler based on interface

411 - 4. Validates output meets minimum requirements

412 - """

413 - config = MODEL_CONFIGS[model_key]

414 -

415 - # Step 1: Backend filtering

416 - if (

417 - "supported_backends" in config

418 - and mm_encoder_attn_backend is not None

419 - and mm_encoder_attn_backend not in config["supported_backends"]

420 - ):

421 - pytest.skip(

422 - f"{model_key} does not support {mm_encoder_attn_backend} backend now."

423 - )

424 -

425 - # Step 2: Apply GPU marks dynamically

426 - if "gpu_marks" in config:

427 - for mark in config["gpu_marks"]:

428 - request.applymarker(mark)

429 -

430 - # Step 3: Route to appropriate handler

431 - if config.get("media_type") == "video":

432 - run_video_test(config, mm_encoder_attn_backend, video_assets, vllm_runner)

433 - elif config["interface"] == "llm_chat":

434 - run_llm_chat_test(config, mm_encoder_attn_backend, image_assets)

435 - elif config["interface"] == "llm_generate":

436 - run_llm_generate_test(config, mm_encoder_attn_backend, image_assets)

437 - else:

438 - raise ValueError(f"Unknown interface: {config['interface']}")

[Misc] Remove dead code in ViT functionality test (#48220)