@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ba093b4f4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/Build] Simplify model loading for `HfRunner` (#5251)

Cyrus Leung committed Jun 5, 2024 at 01:09 UTC 9ba093b4f4f914a8557eb7e4bf961d84420671a5
3 files changed +18 -13
tests/conftest.py
+16 -11
@@ -1,14 +1,15 @@
1 import contextlib
2 import gc
3 import os
4 -from typing import Any, Dict, List, Optional, Tuple
4 +from typing import Any, Dict, List, Optional, Tuple, TypeVar
5
6 import pytest
7 import torch
8 +import torch.nn as nn
9 import torch.nn.functional as F
10 from PIL import Image
10 -from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
11 - LlavaConfig, LlavaForConditionalGeneration)
11 +from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
12 + AutoProcessor, AutoTokenizer, BatchEncoding)
13
14 from vllm import LLM, SamplingParams
15 from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
@@ -144,16 +145,12 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
145 "float": torch.float,
146 }
147
147 -AutoModelForCausalLM.register(LlavaConfig, LlavaForConditionalGeneration)
148 -
149 -_EMBEDDING_MODELS = [
150 - "intfloat/e5-mistral-7b-instruct",
151 -]
148 +_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding)
149
150
151 class HfRunner:
152
156 - def wrap_device(self, input: any):
153 + def wrap_device(self, input: _T) -> _T:
154 if not is_cpu():
155 return input.to("cuda")
156 else:
@@ -163,13 +160,16 @@ class HfRunner:
160 self,
161 model_name: str,
162 dtype: str = "half",
163 + *,
164 + is_embedding_model: bool = False,
165 + is_vision_model: bool = False,
166 ) -> None:
167 assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
168 torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
169
170 self.model_name = model_name
171
172 - if model_name in _EMBEDDING_MODELS:
172 + if is_embedding_model:
173 # Lazy init required for AMD CI
174 from sentence_transformers import SentenceTransformer
175 self.model = self.wrap_device(
@@ -178,8 +178,13 @@ class HfRunner:
178 device="cpu",
179 ).to(dtype=torch_dtype))
180 else:
181 + if is_vision_model:
182 + auto_cls = AutoModelForVision2Seq
183 + else:
184 + auto_cls = AutoModelForCausalLM
185 +
186 self.model = self.wrap_device(
182 - AutoModelForCausalLM.from_pretrained(
187 + auto_cls.from_pretrained(
188 model_name,
189 torch_dtype=torch_dtype,
190 trust_remote_code=True,
tests/models/test_embedding.py
+1 -1
@@ -28,7 +28,7 @@ def test_models(
28 model: str,
29 dtype: str,
30 ) -> None:
31 - hf_model = hf_runner(model, dtype=dtype)
31 + hf_model = hf_runner(model, dtype=dtype, is_embedding_model=True)
32 hf_outputs = hf_model.encode(example_prompts)
33 del hf_model
34
tests/models/test_llava.py
+1 -1
@@ -94,7 +94,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
94 """
95 model_id, vision_language_config = model_and_config
96
97 - hf_model = hf_runner(model_id, dtype=dtype)
97 + hf_model = hf_runner(model_id, dtype=dtype, is_vision_model=True)
98 hf_outputs = hf_model.generate_greedy(hf_image_prompts,
99 max_tokens,
100 images=hf_images)