@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aae6927be0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[VLM][Model] Add test for InternViT vision encoder (#7409)

Isotr0py committed Aug 20, 2024 at 23:10 UTC aae6927be06dedbda39c6b0c30f6aa3242b84388
2 files changed +98 -1
tests/conftest.py
+18 -1
@@ -24,7 +24,9 @@ from vllm.assets.image import ImageAsset
24 from vllm.config import TokenizerPoolConfig
25 from vllm.connections import global_http_connection
26 from vllm.distributed import (destroy_distributed_environment,
27 - destroy_model_parallel)
27 + destroy_model_parallel,
28 + init_distributed_environment,
29 + initialize_model_parallel)
30 from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
31 to_enc_dec_tuple_list, zip_enc_dec_prompts)
32 from vllm.logger import init_logger
@@ -90,6 +92,21 @@ def init_test_http_connection():
92 global_http_connection.reuse_client = False
93
94
95 +@pytest.fixture
96 +def dist_init():
97 + temp_file = tempfile.mkstemp()[1]
98 + init_distributed_environment(
99 + world_size=1,
100 + rank=0,
101 + distributed_init_method=f"file://{temp_file}",
102 + local_rank=0,
103 + backend="nccl",
104 + )
105 + initialize_model_parallel(1, 1)
106 + yield
107 + cleanup()
108 +
109 +
110 def cleanup():
111 destroy_model_parallel()
112 destroy_distributed_environment()
tests/models/test_intern_vit.py new
+80
@@ -0,0 +1,80 @@
1 +from typing import Optional
2 +
3 +import pytest
4 +import torch
5 +import torch.nn as nn
6 +from huggingface_hub import snapshot_download
7 +from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8 +
9 +from vllm.model_executor.models.intern_vit import InternVisionModel
10 +
11 +from ..conftest import _ImageAssets, cleanup
12 +
13 +pytestmark = pytest.mark.vlm
14 +
15 +# we use snapshot_download to prevent conflicts between
16 +# dynamic_module and trust_remote_code for hf_runner
17 +DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
18 +models = [
19 + snapshot_download("OpenGVLab/InternViT-300M-448px",
20 + allow_patterns=DOWNLOAD_PATTERN),
21 + snapshot_download("OpenGVLab/InternViT-6B-448px-V1-5",
22 + allow_patterns=DOWNLOAD_PATTERN),
23 +]
24 +
25 +
26 +def run_intern_vit_test(
27 + image_assets: _ImageAssets,
28 + model: str,
29 + *,
30 + dtype: str,
31 + distributed_executor_backend: Optional[str] = None,
32 +):
33 + img_processor = CLIPImageProcessor.from_pretrained(model)
34 + images = [asset.pil_image for asset in image_assets]
35 + pixel_values = [
36 + img_processor(images, return_tensors='pt').pixel_values.to(dtype)
37 + for images in images
38 + ]
39 +
40 + config = AutoConfig.from_pretrained(model, trust_remote_code=True)
41 + if not getattr(config, "norm_type", None):
42 + config.norm_type = "rms_norm"
43 +
44 + hf_model = AutoModel.from_pretrained(model,
45 + torch_dtype=dtype,
46 + trust_remote_code=True).to("cuda")
47 + hf_outputs_per_image = [
48 + hf_model(pixel_value.to("cuda")).last_hidden_state
49 + for pixel_value in pixel_values
50 + ]
51 +
52 + vllm_model = InternVisionModel(config)
53 + vllm_model.load_weights(hf_model.state_dict().items())
54 +
55 + del hf_model
56 + cleanup()
57 +
58 + vllm_model = vllm_model.to("cuda", dtype)
59 + vllm_outputs_per_image = [
60 + vllm_model(pixel_values=pixel_value.to("cuda"))
61 + for pixel_value in pixel_values
62 + ]
63 + del vllm_model
64 + cleanup()
65 +
66 + cos_similar = nn.CosineSimilarity(dim=-1)
67 + for vllm_output, hf_output in zip(vllm_outputs_per_image,
68 + hf_outputs_per_image):
69 + assert cos_similar(vllm_output, hf_output).mean() > 0.99
70 +
71 +
72 +@pytest.mark.parametrize("model", models)
73 +@pytest.mark.parametrize("dtype", [torch.half])
74 +@torch.inference_mode()
75 +def test_models(dist_init, image_assets, model, dtype: str) -> None:
76 + run_intern_vit_test(
77 + image_assets,
78 + model,
79 + dtype=dtype,
80 + )