A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[VLM][Model] Add test for InternViT vision encoder (#7409)
Isotr0py committed
Aug 20, 2024 at 23:10 UTC
aae6927be06dedbda39c6b0c30f6aa3242b84388
2 files changed
+98
-1
tests/conftest.py
+18
-1
@@ -24,7 +24,9 @@ from vllm.assets.image import ImageAsset
24
from vllm.config import TokenizerPoolConfig
25
from vllm.connections import global_http_connection
26
from vllm.distributed import (destroy_distributed_environment,
27
- destroy_model_parallel)
27
+ destroy_model_parallel,
28
+ init_distributed_environment,
29
+ initialize_model_parallel)
30
from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
31
to_enc_dec_tuple_list, zip_enc_dec_prompts)
32
from vllm.logger import init_logger
@@ -90,6 +92,21 @@ def init_test_http_connection():
92
global_http_connection.reuse_client = False
93
94
95
+@pytest.fixture
96
+def dist_init():
97
+ temp_file = tempfile.mkstemp()[1]
98
+ init_distributed_environment(
99
+ world_size=1,
100
+ rank=0,
101
+ distributed_init_method=f"file://{temp_file}",
102
+ local_rank=0,
103
+ backend="nccl",
104
+ )
105
+ initialize_model_parallel(1, 1)
106
+ yield
107
+ cleanup()
108
+
109
+
110
def cleanup():
111
destroy_model_parallel()
112
destroy_distributed_environment()
tests/models/test_intern_vit.py
new
+80
@@ -0,0 +1,80 @@
1
+from typing import Optional
2
+
3
+import pytest
4
+import torch
5
+import torch.nn as nn
6
+from huggingface_hub import snapshot_download
7
+from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8
+
9
+from vllm.model_executor.models.intern_vit import InternVisionModel
10
+
11
+from ..conftest import _ImageAssets, cleanup
12
+
13
+pytestmark = pytest.mark.vlm
14
+
15
+# we use snapshot_download to prevent conflicts between
16
+# dynamic_module and trust_remote_code for hf_runner
17
+DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
18
+models = [
19
+ snapshot_download("OpenGVLab/InternViT-300M-448px",
20
+ allow_patterns=DOWNLOAD_PATTERN),
21
+ snapshot_download("OpenGVLab/InternViT-6B-448px-V1-5",
22
+ allow_patterns=DOWNLOAD_PATTERN),
23
+]
24
+
25
+
26
+def run_intern_vit_test(
27
+ image_assets: _ImageAssets,
28
+ model: str,
29
+ *,
30
+ dtype: str,
31
+ distributed_executor_backend: Optional[str] = None,
32
+):
33
+ img_processor = CLIPImageProcessor.from_pretrained(model)
34
+ images = [asset.pil_image for asset in image_assets]
35
+ pixel_values = [
36
+ img_processor(images, return_tensors='pt').pixel_values.to(dtype)
37
+ for images in images
38
+ ]
39
+
40
+ config = AutoConfig.from_pretrained(model, trust_remote_code=True)
41
+ if not getattr(config, "norm_type", None):
42
+ config.norm_type = "rms_norm"
43
+
44
+ hf_model = AutoModel.from_pretrained(model,
45
+ torch_dtype=dtype,
46
+ trust_remote_code=True).to("cuda")
47
+ hf_outputs_per_image = [
48
+ hf_model(pixel_value.to("cuda")).last_hidden_state
49
+ for pixel_value in pixel_values
50
+ ]
51
+
52
+ vllm_model = InternVisionModel(config)
53
+ vllm_model.load_weights(hf_model.state_dict().items())
54
+
55
+ del hf_model
56
+ cleanup()
57
+
58
+ vllm_model = vllm_model.to("cuda", dtype)
59
+ vllm_outputs_per_image = [
60
+ vllm_model(pixel_values=pixel_value.to("cuda"))
61
+ for pixel_value in pixel_values
62
+ ]
63
+ del vllm_model
64
+ cleanup()
65
+
66
+ cos_similar = nn.CosineSimilarity(dim=-1)
67
+ for vllm_output, hf_output in zip(vllm_outputs_per_image,
68
+ hf_outputs_per_image):
69
+ assert cos_similar(vllm_output, hf_output).mean() > 0.99
70
+
71
+
72
+@pytest.mark.parametrize("model", models)
73
+@pytest.mark.parametrize("dtype", [torch.half])
74
+@torch.inference_mode()
75
+def test_models(dist_init, image_assets, model, dtype: str) -> None:
76
+ run_intern_vit_test(
77
+ image_assets,
78
+ model,
79
+ dtype=dtype,
80
+ )