@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e9cdd2b1e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI/Build] Further decouple HuggingFace implementation from ours during tests (#4166)

Cyrus Leung committed May 15, 2024 at 14:38 UTC e9cdd2b1e20beb1c21c55441d0e6a4ed86f4e292
1 file changed +41 -36
tests/conftest.py
+41 -36
@@ -1,19 +1,21 @@
1 import contextlib
2 import gc
3 import os
4 -from typing import List, Optional, Tuple
4 +from typing import Any, Dict, List, Optional, Tuple
5
6 import pytest
7 import torch
8 from PIL import Image
9 -from transformers import (AutoModelForCausalLM, AutoProcessor,
10 - LlavaForConditionalGeneration)
9 +from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
10 + LlavaConfig, LlavaForConditionalGeneration)
11
12 from vllm import LLM, SamplingParams
13 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
14 from vllm.distributed import destroy_model_parallel
15 +from vllm.logger import init_logger
16 from vllm.sequence import MultiModalData
16 -from vllm.transformers_utils.tokenizer import get_tokenizer
17 +
18 +logger = init_logger(__name__)
19
20 _TEST_DIR = os.path.dirname(__file__)
21 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
@@ -129,9 +131,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
131 "float": torch.float,
132 }
133
132 -_VISION_LANGUAGE_MODELS = {
133 - "llava-hf/llava-1.5-7b-hf": LlavaForConditionalGeneration,
134 -}
134 +AutoModelForCausalLM.register(LlavaConfig, LlavaForConditionalGeneration)
135
136 _EMBEDDING_MODELS = [
137 "intfloat/e5-mistral-7b-instruct",
@@ -143,23 +143,14 @@ class HfRunner:
143 def __init__(
144 self,
145 model_name: str,
146 - tokenizer_name: Optional[str] = None,
146 dtype: str = "half",
147 ) -> None:
148 assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
149 torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
150 +
151 self.model_name = model_name
152 - if model_name in _VISION_LANGUAGE_MODELS:
153 - self.model = _VISION_LANGUAGE_MODELS[model_name].from_pretrained(
154 - model_name,
155 - torch_dtype=torch_dtype,
156 - trust_remote_code=True,
157 - ).cuda()
158 - self.processor = AutoProcessor.from_pretrained(
159 - model_name,
160 - torch_dtype=torch_dtype,
161 - )
162 - elif model_name in _EMBEDDING_MODELS:
152 +
153 + if model_name in _EMBEDDING_MODELS:
154 # Lazy init required for AMD CI
155 from sentence_transformers import SentenceTransformer
156 self.model = SentenceTransformer(
@@ -172,10 +163,24 @@ class HfRunner:
163 torch_dtype=torch_dtype,
164 trust_remote_code=True,
165 ).cuda()
175 - self.processor = None
176 - if tokenizer_name is None:
177 - tokenizer_name = model_name
178 - self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
166 +
167 + self.tokenizer = AutoTokenizer.from_pretrained(
168 + model_name,
169 + torch_dtype=torch_dtype,
170 + trust_remote_code=True,
171 + )
172 +
173 + try:
174 + self.processor = AutoProcessor.from_pretrained(
175 + model_name,
176 + torch_dtype=torch_dtype,
177 + trust_remote_code=True,
178 + )
179 + except Exception:
180 + logger.warning(
181 + "Unable to auto-load processor from HuggingFace for "
182 + "model %s. Using tokenizer instead.", model_name)
183 + self.processor = self.tokenizer
184
185 def generate(
186 self,
@@ -187,19 +192,19 @@ class HfRunner:
192 if images:
193 assert len(prompts) == len(images)
194 for i, prompt in enumerate(prompts):
190 - if self.model_name not in _VISION_LANGUAGE_MODELS:
191 - input_ids = self.tokenizer(prompt,
192 - return_tensors="pt").input_ids
193 - inputs = {"input_ids": input_ids.cuda()}
194 - else:
195 - image = images[i] if images else None
196 - inputs = self.processor(text=prompt,
197 - images=image,
198 - return_tensors="pt")
199 - inputs = {
200 - key: value.cuda() if value is not None else None
201 - for key, value in inputs.items()
202 - }
195 + processor_kwargs: Dict[str, Any] = {
196 + "text": prompt,
197 + "return_tensors": "pt",
198 + }
199 + if images is not None and images[i] is not None:
200 + processor_kwargs["images"] = images[i]
201 +
202 + inputs = self.processor(**processor_kwargs)
203 + inputs = {
204 + key: value.cuda() if value is not None else None
205 + for key, value in inputs.items()
206 + }
207 +
208 output_ids = self.model.generate(
209 **inputs,
210 use_cache=True,