@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e56bf27741
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix InternVL2 inference with various num_patches (#8375)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Isotr0py committed Sep 13, 2024 at 01:10 UTC e56bf2774158dca80637a1b8309bbc4d308774b1
2 files changed +39 -3
tests/models/test_internvl.py
+35
@@ -331,6 +331,41 @@ def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
331 )
332
333
334 +@pytest.mark.parametrize("model", ["OpenGVLab/InternVL2-2B"])
335 +@pytest.mark.parametrize("size_factors", [[0.5, 1.0]])
336 +@pytest.mark.parametrize("dtype", [target_dtype])
337 +@pytest.mark.parametrize("max_tokens", [128])
338 +@pytest.mark.parametrize("num_logprobs", [5])
339 +@torch.inference_mode()
340 +def test_different_num_patches(hf_runner, vllm_runner, image_assets, model,
341 + size_factors, dtype: str, max_tokens: int,
342 + num_logprobs: int) -> None:
343 + images = [asset.pil_image.resize((896, 896)) for asset in image_assets]
344 +
345 + inputs_batching = [(
346 + [prompt for _ in size_factors],
347 + [rescale_image_size(image, factor) for factor in size_factors],
348 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
349 +
350 + inputs_multi_images = [
351 + ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
352 + [[rescale_image_size(image, factor) for image in images]
353 + for factor in size_factors])
354 + ]
355 + for inputs in [inputs_batching, inputs_multi_images]:
356 + run_test(
357 + hf_runner,
358 + vllm_runner,
359 + inputs,
360 + model,
361 + dtype=dtype,
362 + max_tokens=max_tokens,
363 + num_logprobs=num_logprobs,
364 + mm_limit=2,
365 + tensor_parallel_size=1,
366 + )
367 +
368 +
369 @pytest.mark.parametrize(
370 "models", [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")])
371 @pytest.mark.parametrize(
vllm/model_executor/models/internvl.py
+4 -3
@@ -270,6 +270,7 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
270 # Add an N dimension for number of images per prompt (currently 1).
271 data = data.unsqueeze(0)
272 elif is_list_of(data, Image.Image):
273 + # we can't stack here because the images may have different num_patches
274 data = [
275 image_to_pixel_values(img,
276 image_size,
@@ -277,7 +278,6 @@ def input_mapper_for_internvl(ctx: InputContext, data: object):
278 max_num,
279 use_thumbnail=use_thumbnail) for img in data
280 ]
280 - data = torch.stack(data)
281 model_config = ctx.model_config
282 tokenizer = cached_get_tokenizer(model_config.tokenizer,
283 trust_remote_code=True)
@@ -449,11 +449,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal):
449 if not isinstance(pixel_values, (torch.Tensor, list)):
450 raise ValueError("Incorrect type of pixel values. "
451 f"Got type: {type(pixel_values)}")
452 -
452 + # We need to flatten (B, N, P) to (B*N*P),
453 + # so we call flatten_bn twice.
454 return InternVLImagePixelInputs(
455 type="pixel_values",
456 data=self._validate_pixel_values(
456 - flatten_bn(pixel_values, concat=True).flatten(0, 1)),
457 + flatten_bn(flatten_bn(pixel_values), concat=True)),
458 )
459
460 raise AssertionError("This line should be unreachable.")