A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/Build] Simplify model loading for `HfRunner` (#5251)
Cyrus Leung committed
Jun 5, 2024 at 01:09 UTC
9ba093b4f4f914a8557eb7e4bf961d84420671a5
3 files changed
+18
-13
tests/conftest.py
+16
-11
@@ -1,14 +1,15 @@
1
import contextlib
2
import gc
3
import os
4
-from typing import Any, Dict, List, Optional, Tuple
4
+from typing import Any, Dict, List, Optional, Tuple, TypeVar
5
6
import pytest
7
import torch
8
+import torch.nn as nn
9
import torch.nn.functional as F
10
from PIL import Image
10
-from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
11
- LlavaConfig, LlavaForConditionalGeneration)
11
+from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
12
+ AutoProcessor, AutoTokenizer, BatchEncoding)
13
14
from vllm import LLM, SamplingParams
15
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
@@ -144,16 +145,12 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
145
"float": torch.float,
146
}
147
147
-AutoModelForCausalLM.register(LlavaConfig, LlavaForConditionalGeneration)
148
-
149
-_EMBEDDING_MODELS = [
150
- "intfloat/e5-mistral-7b-instruct",
151
-]
148
+_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding)
149
150
151
class HfRunner:
152
156
- def wrap_device(self, input: any):
153
+ def wrap_device(self, input: _T) -> _T:
154
if not is_cpu():
155
return input.to("cuda")
156
else:
@@ -163,13 +160,16 @@ class HfRunner:
160
self,
161
model_name: str,
162
dtype: str = "half",
163
+ *,
164
+ is_embedding_model: bool = False,
165
+ is_vision_model: bool = False,
166
) -> None:
167
assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
168
torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
169
170
self.model_name = model_name
171
172
- if model_name in _EMBEDDING_MODELS:
172
+ if is_embedding_model:
173
# Lazy init required for AMD CI
174
from sentence_transformers import SentenceTransformer
175
self.model = self.wrap_device(
@@ -178,8 +178,13 @@ class HfRunner:
178
device="cpu",
179
).to(dtype=torch_dtype))
180
else:
181
+ if is_vision_model:
182
+ auto_cls = AutoModelForVision2Seq
183
+ else:
184
+ auto_cls = AutoModelForCausalLM
185
+
186
self.model = self.wrap_device(
182
- AutoModelForCausalLM.from_pretrained(
187
+ auto_cls.from_pretrained(
188
model_name,
189
torch_dtype=torch_dtype,
190
trust_remote_code=True,
tests/models/test_embedding.py
+1
-1
@@ -28,7 +28,7 @@ def test_models(
28
model: str,
29
dtype: str,
30
) -> None:
31
- hf_model = hf_runner(model, dtype=dtype)
31
+ hf_model = hf_runner(model, dtype=dtype, is_embedding_model=True)
32
hf_outputs = hf_model.encode(example_prompts)
33
del hf_model
34
tests/models/test_llava.py
+1
-1
@@ -94,7 +94,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
94
"""
95
model_id, vision_language_config = model_and_config
96
97
- hf_model = hf_runner(model_id, dtype=dtype)
97
+ hf_model = hf_runner(model_id, dtype=dtype, is_vision_model=True)
98
hf_outputs = hf_model.generate_greedy(hf_image_prompts,
99
max_tokens,
100
images=hf_images)