A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CI/Build] Further decouple HuggingFace implementation from ours during tests (#4166)
Cyrus Leung committed
May 15, 2024 at 14:38 UTC
e9cdd2b1e20beb1c21c55441d0e6a4ed86f4e292
1 file changed
+41
-36
tests/conftest.py
+41
-36
@@ -1,19 +1,21 @@
1
import contextlib
2
import gc
3
import os
4
-from typing import List, Optional, Tuple
4
+from typing import Any, Dict, List, Optional, Tuple
5
6
import pytest
7
import torch
8
from PIL import Image
9
-from transformers import (AutoModelForCausalLM, AutoProcessor,
10
- LlavaForConditionalGeneration)
9
+from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
10
+ LlavaConfig, LlavaForConditionalGeneration)
11
12
from vllm import LLM, SamplingParams
13
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
14
from vllm.distributed import destroy_model_parallel
15
+from vllm.logger import init_logger
16
from vllm.sequence import MultiModalData
16
-from vllm.transformers_utils.tokenizer import get_tokenizer
17
+
18
+logger = init_logger(__name__)
19
20
_TEST_DIR = os.path.dirname(__file__)
21
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
@@ -129,9 +131,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
131
"float": torch.float,
132
}
133
132
-_VISION_LANGUAGE_MODELS = {
133
- "llava-hf/llava-1.5-7b-hf": LlavaForConditionalGeneration,
134
-}
134
+AutoModelForCausalLM.register(LlavaConfig, LlavaForConditionalGeneration)
135
136
_EMBEDDING_MODELS = [
137
"intfloat/e5-mistral-7b-instruct",
@@ -143,23 +143,14 @@ class HfRunner:
143
def __init__(
144
self,
145
model_name: str,
146
- tokenizer_name: Optional[str] = None,
146
dtype: str = "half",
147
) -> None:
148
assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
149
torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
150
+
151
self.model_name = model_name
152
- if model_name in _VISION_LANGUAGE_MODELS:
153
- self.model = _VISION_LANGUAGE_MODELS[model_name].from_pretrained(
154
- model_name,
155
- torch_dtype=torch_dtype,
156
- trust_remote_code=True,
157
- ).cuda()
158
- self.processor = AutoProcessor.from_pretrained(
159
- model_name,
160
- torch_dtype=torch_dtype,
161
- )
162
- elif model_name in _EMBEDDING_MODELS:
152
+
153
+ if model_name in _EMBEDDING_MODELS:
154
# Lazy init required for AMD CI
155
from sentence_transformers import SentenceTransformer
156
self.model = SentenceTransformer(
@@ -172,10 +163,24 @@ class HfRunner:
163
torch_dtype=torch_dtype,
164
trust_remote_code=True,
165
).cuda()
175
- self.processor = None
176
- if tokenizer_name is None:
177
- tokenizer_name = model_name
178
- self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
166
+
167
+ self.tokenizer = AutoTokenizer.from_pretrained(
168
+ model_name,
169
+ torch_dtype=torch_dtype,
170
+ trust_remote_code=True,
171
+ )
172
+
173
+ try:
174
+ self.processor = AutoProcessor.from_pretrained(
175
+ model_name,
176
+ torch_dtype=torch_dtype,
177
+ trust_remote_code=True,
178
+ )
179
+ except Exception:
180
+ logger.warning(
181
+ "Unable to auto-load processor from HuggingFace for "
182
+ "model %s. Using tokenizer instead.", model_name)
183
+ self.processor = self.tokenizer
184
185
def generate(
186
self,
@@ -187,19 +192,19 @@ class HfRunner:
192
if images:
193
assert len(prompts) == len(images)
194
for i, prompt in enumerate(prompts):
190
- if self.model_name not in _VISION_LANGUAGE_MODELS:
191
- input_ids = self.tokenizer(prompt,
192
- return_tensors="pt").input_ids
193
- inputs = {"input_ids": input_ids.cuda()}
194
- else:
195
- image = images[i] if images else None
196
- inputs = self.processor(text=prompt,
197
- images=image,
198
- return_tensors="pt")
199
- inputs = {
200
- key: value.cuda() if value is not None else None
201
- for key, value in inputs.items()
202
- }
195
+ processor_kwargs: Dict[str, Any] = {
196
+ "text": prompt,
197
+ "return_tensors": "pt",
198
+ }
199
+ if images is not None and images[i] is not None:
200
+ processor_kwargs["images"] = images[i]
201
+
202
+ inputs = self.processor(**processor_kwargs)
203
+ inputs = {
204
+ key: value.cuda() if value is not None else None
205
+ for key, value in inputs.items()
206
+ }
207
+
208
output_ids = self.model.generate(
209
**inputs,
210
use_cache=True,