@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ff57ebfca
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Initialize Florence-2 language backbone support (#9555)

Isotr0py committed Oct 23, 2024 at 18:42 UTC 3ff57ebfcacdd4f7690ed8f5693657de2bdedea8
5 files changed +428 -8
examples/florence2_inference.py new
+44
@@ -0,0 +1,44 @@
1 +'''
2 +Demonstrate prompting of text-to-text
3 +encoder/decoder models, specifically Florence-2
4 +'''
5 +# TODO(Isotr0py):
6 +# Move to offline_inference_vision_language.py after porting vision backbone
7 +from vllm import LLM, SamplingParams
8 +
9 +dtype = "float"
10 +
11 +# Create a Florence-2 encoder/decoder model instance
12 +llm = LLM(
13 + model="microsoft/Florence-2-base",
14 + tokenizer="facebook/bart-base",
15 + dtype=dtype,
16 + trust_remote_code=True,
17 +)
18 +
19 +prompts = [
20 + "<CAPTION>", "<DETAILED_CAPTION>", "<MORE_DETAILED_CAPTION>",
21 + "<CAPTION_TO_PHRASE_GROUNDING>", "<OD>", "<DENSE_REGION_CAPTION>",
22 + "<REGION_PROPOSAL>", "<OCR>", "<OCR_WITH_REGION>"
23 +]
24 +# Create a sampling params object.
25 +sampling_params = SamplingParams(
26 + temperature=0,
27 + top_p=1.0,
28 + min_tokens=0,
29 + max_tokens=20,
30 +)
31 +
32 +# Generate output tokens from the prompts. The output is a list of
33 +# RequestOutput objects that contain the prompt, generated
34 +# text, and other information.
35 +outputs = llm.generate(prompts, sampling_params)
36 +
37 +# Print the outputs.
38 +for output in outputs:
39 + prompt = output.prompt
40 + encoder_prompt = output.encoder_prompt
41 + generated_text = output.outputs[0].text
42 + print(f"Encoder prompt: {encoder_prompt!r}, "
43 + f"Decoder prompt: {prompt!r}, "
44 + f"Generated text: {generated_text!r}")
tests/conftest.py
+20 -8
@@ -253,7 +253,9 @@ class HfRunner:
253 dtype: str = "half",
254 *,
255 model_kwargs: Optional[Dict[str, Any]] = None,
256 + is_embedding_model: bool = False,
257 is_sentence_transformer: bool = False,
258 + skip_tokenizer_init: bool = False,
259 auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM,
260 postprocess_inputs: Callable[[BatchEncoding],
261 BatchEncoding] = identity,
@@ -281,11 +283,12 @@ class HfRunner:
283 **model_kwargs,
284 ))
285
284 - self.tokenizer = AutoTokenizer.from_pretrained(
285 - model_name,
286 - torch_dtype=torch_dtype,
287 - trust_remote_code=True,
288 - )
286 + if not skip_tokenizer_init:
287 + self.tokenizer = AutoTokenizer.from_pretrained(
288 + model_name,
289 + torch_dtype=torch_dtype,
290 + trust_remote_code=True,
291 + )
292
293 # don't put this import at the top level
294 # it will call torch.cuda.device_count()
@@ -295,6 +298,8 @@ class HfRunner:
298 torch_dtype=torch_dtype,
299 trust_remote_code=True,
300 )
301 + if skip_tokenizer_init:
302 + self.tokenizer = self.processor.tokenizer
303
304 self.postprocess_inputs = postprocess_inputs
305
@@ -535,6 +540,7 @@ class HfRunner:
540 encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
541 max_tokens: int,
542 num_logprobs: int,
543 + images: Optional[PromptImageInput] = None,
544 **kwargs: Any,
545 ) -> List[TokensTextLogprobs]:
546 '''
@@ -545,11 +551,17 @@ class HfRunner:
551 all_output_ids: List[List[int]] = []
552 all_output_strs: List[str] = []
553
548 - for (encoder_prompt,
549 - decoder_prompt) in to_enc_dec_tuple_list(encoder_decoder_prompts):
554 + for i, (encoder_prompt, decoder_prompt) in enumerate(
555 + to_enc_dec_tuple_list(encoder_decoder_prompts)):
556 + processor_kwargs: Dict[str, Any] = {
557 + "text": encoder_prompt,
558 + "return_tensors": "pt",
559 + }
560 + if images is not None and images[i] is not None:
561 + processor_kwargs["images"] = images[i]
562
563 encoder_input_ids = self.wrap_device(
552 - self.tokenizer(encoder_prompt, return_tensors="pt").input_ids,
564 + self.processor(**processor_kwargs).input_ids,
565 device=self.model.device.type,
566 )
567
tests/models/encoder_decoder/vision_language/test_florence2.py new
+102
@@ -0,0 +1,102 @@
1 +from functools import partial
2 +from typing import List, Optional, Tuple, Type
3 +
4 +import pytest
5 +from PIL import Image
6 +
7 +from vllm.inputs.data import ExplicitEncoderDecoderPrompt
8 +from vllm.sequence import SampleLogprobs
9 +
10 +from ....conftest import HfRunner, VllmRunner
11 +from ...utils import check_logprobs_close
12 +
13 +Florence2Prompt = partial(ExplicitEncoderDecoderPrompt,
14 + decoder_prompt=None,
15 + mm_processor_kwargs=None)
16 +
17 +MODELS = ["microsoft/Florence-2-base"]
18 +# Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
19 +# Therefore, we borrow the BartTokenizer from the original Bart model
20 +TOKENIZER = "facebook/bart-base"
21 +PROMPTS = [
22 + Florence2Prompt(encoder_prompt="<CAPTION>"),
23 + Florence2Prompt(encoder_prompt="<DETAILED_CAPTION>"),
24 + Florence2Prompt(encoder_prompt="<MORE_DETAILED_CAPTION>"),
25 + Florence2Prompt(encoder_prompt="<CAPTION_TO_PHRASE_GROUNDING>"),
26 + Florence2Prompt(encoder_prompt="<DENSE_REGION_CAPTION>"),
27 + Florence2Prompt(encoder_prompt="<REGION_PROPOSAL>"),
28 + Florence2Prompt(encoder_prompt="<OCR_WITH_REGION>"),
29 + Florence2Prompt(encoder_prompt="<OCR>"),
30 + Florence2Prompt(encoder_prompt="<OD>"),
31 +]
32 +
33 +
34 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
35 + Optional[SampleLogprobs]], ):
36 + """Sanitize vllm output to be comparable with hf output."""
37 + output_ids, output_str, out_logprobs = vllm_output
38 +
39 + hf_output_str = "</s><s>" + output_str + "</s>"
40 +
41 + return output_ids, hf_output_str, out_logprobs
42 +
43 +
44 +def run_test(
45 + hf_runner: Type[HfRunner],
46 + vllm_runner: Type[VllmRunner],
47 + prompts: List[ExplicitEncoderDecoderPrompt],
48 + model: str,
49 + *,
50 + dtype: str,
51 + max_tokens: int,
52 + num_logprobs: int,
53 + tensor_parallel_size: int,
54 + distributed_executor_backend: Optional[str] = None,
55 +) -> None:
56 + with vllm_runner(model,
57 + tokenizer_name=TOKENIZER,
58 + dtype=dtype,
59 + tensor_parallel_size=tensor_parallel_size,
60 + distributed_executor_backend=distributed_executor_backend,
61 + enforce_eager=True) as vllm_model:
62 + vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
63 + prompts, max_tokens, num_logprobs)
64 +
65 + # Florence-2 processors require image inputs
66 + dummy_image = Image.new(mode="RGB", size=(2, 2))
67 + with hf_runner(model, dtype=dtype, skip_tokenizer_init=True) as hf_model:
68 + hf_model.model.get_output_embeddings = lambda: \
69 + hf_model.model.language_model.lm_head
70 + hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
71 + prompts,
72 + max_tokens,
73 + num_logprobs,
74 + images=[dummy_image] * len(prompts),
75 + ))
76 +
77 + check_logprobs_close(
78 + outputs_0_lst=hf_outputs,
79 + outputs_1_lst=[
80 + vllm_to_hf_output(vllm_output) for vllm_output in vllm_outputs
81 + ],
82 + name_0="hf",
83 + name_1="vllm",
84 + )
85 +
86 +
87 +@pytest.mark.parametrize("model", MODELS)
88 +@pytest.mark.parametrize("dtype", ["float"])
89 +@pytest.mark.parametrize("max_tokens", [64])
90 +@pytest.mark.parametrize("num_logprobs", [5])
91 +def test_models(hf_runner, vllm_runner, model, dtype, max_tokens,
92 + num_logprobs) -> None:
93 + run_test(
94 + hf_runner,
95 + vllm_runner,
96 + PROMPTS,
97 + model,
98 + dtype=dtype,
99 + max_tokens=max_tokens,
100 + num_logprobs=num_logprobs,
101 + tensor_parallel_size=1,
102 + )
vllm/model_executor/models/florence2.py new
+261
@@ -0,0 +1,261 @@
1 +import math
2 +from typing import Iterable, List, Optional, Tuple
3 +
4 +import torch
5 +import torch.nn as nn
6 +from transformers import PretrainedConfig
7 +
8 +from vllm.attention import AttentionMetadata
9 +from vllm.config import CacheConfig
10 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
11 +from vllm.model_executor.layers.quantization.base_config import (
12 + QuantizationConfig)
13 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
14 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15 +from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
16 + BartParallelLMHead,
17 + BartScaledWordEmbedding)
18 +from vllm.model_executor.sampling_metadata import SamplingMetadata
19 +from vllm.sequence import IntermediateTensors
20 +
21 +from .utils import AutoWeightsLoader
22 +
23 +
24 +class Florence2LanguageModel(nn.Module):
25 +
26 + def __init__(self,
27 + config: PretrainedConfig,
28 + cache_config: Optional[CacheConfig] = None,
29 + quant_config: Optional[QuantizationConfig] = None):
30 + super().__init__()
31 + self.config = config
32 +
33 + self.padding_idx = config.pad_token_id
34 + self.vocab_size = config.vocab_size
35 +
36 + self.shared = BartScaledWordEmbedding(self.vocab_size, config.d_model)
37 + self.encoder = BartEncoder(config,
38 + cache_config=cache_config,
39 + quant_config=quant_config)
40 + self.decoder = BartDecoder(config,
41 + cache_config=cache_config,
42 + quant_config=quant_config)
43 +
44 + if self.config.tie_word_embeddings:
45 + self.encoder.embed_tokens.weight = self.shared.weight
46 + self.decoder.embed_tokens.weight = self.shared.weight
47 +
48 + def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
49 + encoder_input_ids: torch.Tensor,
50 + encoder_positions: torch.Tensor, kv_caches: List[torch.Tensor],
51 + attn_metadata: AttentionMetadata) -> torch.Tensor:
52 + r"""
53 + Args:
54 + input_ids
55 + Indices of *decoder* input sequence tokens in the vocabulary.
56 + Padding will be ignored by default should you
57 + provide it.
58 + positions
59 + Positions of *decoder* input sequence tokens.
60 + encoder_input_ids
61 + Indices of *encoder* input sequence tokens in the vocabulary.
62 + encoder_positions:
63 + Positions of *encoder* input sequence tokens.
64 + kv_caches:
65 + Layer-wise list of KV cache tensors
66 + attn_metadata:
67 + vLLM Attention metadata structure
68 + Returns:
69 + Model output torch.Tensor
70 + """
71 +
72 + encoder_hidden_states = None
73 +
74 + if encoder_input_ids.numel() > 0:
75 + # Run encoder attention if a non-zero number of encoder tokens
76 + # are provided as input
77 + encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
78 + positions=encoder_positions,
79 + kv_caches=kv_caches,
80 + attn_metadata=attn_metadata)
81 +
82 + # decoder outputs consists of
83 + # (dec_features, past_key_value, dec_hidden, dec_attn)
84 + decoder_outputs = self.decoder(
85 + decoder_input_ids=input_ids,
86 + decoder_positions=positions,
87 + encoder_hidden_states=encoder_hidden_states,
88 + kv_caches=kv_caches,
89 + attn_metadata=attn_metadata)
90 +
91 + return decoder_outputs
92 +
93 +
94 +class Florence2LanguageForConditionalGeneration(nn.Module):
95 +
96 + def __init__(self,
97 + config: PretrainedConfig,
98 + cache_config: Optional[CacheConfig] = None,
99 + quant_config: Optional[QuantizationConfig] = None):
100 + super().__init__()
101 + self.config = config
102 + self.model = Florence2LanguageModel(config,
103 + cache_config=cache_config,
104 + quant_config=quant_config)
105 + embed_scale = math.sqrt(
106 + config.d_model) if config.scale_embedding else 1.0
107 +
108 + self.vocab_size = config.vocab_size
109 + self.lm_head = BartParallelLMHead(self.vocab_size,
110 + config.d_model,
111 + embed_scale=embed_scale)
112 +
113 + self.logits_processor = LogitsProcessor(self.vocab_size,
114 + config.vocab_size)
115 + self.sampler = Sampler()
116 +
117 + def forward(
118 + self,
119 + input_ids: torch.Tensor,
120 + positions: torch.Tensor,
121 + encoder_input_ids: torch.Tensor,
122 + encoder_positions: torch.Tensor,
123 + kv_caches: List[torch.Tensor],
124 + attn_metadata: AttentionMetadata,
125 + **kwargs,
126 + ) -> torch.Tensor:
127 + r"""
128 + Args:
129 + input_ids
130 + torch.Tensor of *decoder* input token ids.
131 + positions
132 + torch.Tensor of *decoder* position indices.
133 + encoder_input_ids
134 + torch.Tensor of *encoder* input token ids.
135 + encoder_positions
136 + torch.Tensor of *encoder* position indices
137 + kv_caches:
138 + Layer-wise list of KV cache tensors
139 + attn_metadata:
140 + vLLM Attention metadata structure
141 + Returns:
142 + Output torch.Tensor
143 + """
144 + return self.model(input_ids, positions, encoder_input_ids,
145 + encoder_positions, kv_caches, attn_metadata)
146 +
147 + def compute_logits(
148 + self,
149 + hidden_states: torch.Tensor,
150 + sampling_metadata: SamplingMetadata,
151 + ) -> Optional[torch.Tensor]:
152 + logits = self.logits_processor(self.lm_head, hidden_states,
153 + sampling_metadata)
154 + return logits
155 +
156 + def sample(self, logits: torch.Tensor,
157 + sampling_metadata: SamplingMetadata) -> SamplerOutput:
158 + next_tokens = self.sampler(logits, sampling_metadata)
159 + return next_tokens
160 +
161 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
162 + stacked_params_mapping = [
163 + # (param_name, shard_name, shard_id)
164 + ("qkv_proj", "q_proj", "q"),
165 + ("qkv_proj", "k_proj", "k"),
166 + ("qkv_proj", "v_proj", "v"),
167 + ]
168 +
169 + params_dict = dict(self.named_parameters())
170 + for name, loaded_weight in weights:
171 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
172 + if weight_name not in name:
173 + continue
174 +
175 + param = params_dict[name.replace(weight_name, param_name)]
176 + weight_loader = param.weight_loader
177 + weight_loader(param, loaded_weight, shard_id)
178 + break
179 + else:
180 + if "final_logits_bias" in name:
181 + continue
182 + if self.config.tie_word_embeddings and "embed_tokens" in name:
183 + continue
184 + param = params_dict[name]
185 + weight_loader = getattr(param, "weight_loader",
186 + default_weight_loader)
187 + weight_loader(param, loaded_weight)
188 +
189 +
190 +class Florence2ForConditionalGeneration(nn.Module):
191 +
192 + def __init__(self,
193 + config: PretrainedConfig,
194 + cache_config: Optional[CacheConfig] = None,
195 + quant_config: Optional[QuantizationConfig] = None):
196 + super().__init__()
197 +
198 + # TODO(Isotr0py): Add vision backbone
199 + self.language_model = Florence2LanguageForConditionalGeneration(
200 + config=config.text_config,
201 + cache_config=cache_config,
202 + quant_config=quant_config)
203 +
204 + @property
205 + def sampler(self):
206 + return self.language_model.sampler
207 +
208 + def forward(
209 + self,
210 + input_ids: torch.Tensor,
211 + positions: torch.Tensor,
212 + kv_caches: List[torch.Tensor],
213 + attn_metadata: AttentionMetadata,
214 + intermediate_tensors: Optional[IntermediateTensors] = None,
215 + *,
216 + encoder_input_ids: torch.Tensor,
217 + encoder_positions: torch.Tensor,
218 + **kwargs,
219 + ) -> torch.Tensor:
220 + r"""
221 + Args:
222 + input_ids
223 + torch.Tensor of *decoder* input token ids.
224 + positions
225 + torch.Tensor of *decoder* position indices.
226 + encoder_input_ids
227 + torch.Tensor of *encoder* input token ids.
228 + encoder_positions
229 + torch.Tensor of *encoder* position indices
230 + kv_caches:
231 + Layer-wise list of KV cache tensors
232 + attn_metadata:
233 + vLLM Attention metadata structure
234 + Returns:
235 + Output torch.Tensor
236 + """
237 + return self.language_model(input_ids, positions, encoder_input_ids,
238 + encoder_positions, kv_caches, attn_metadata)
239 +
240 + def compute_logits(
241 + self,
242 + hidden_states: torch.Tensor,
243 + sampling_metadata: SamplingMetadata,
244 + ) -> Optional[torch.Tensor]:
245 + return self.language_model.compute_logits(hidden_states,
246 + sampling_metadata)
247 +
248 + def sample(
249 + self,
250 + logits: torch.Tensor,
251 + sampling_metadata: SamplingMetadata,
252 + ) -> SamplerOutput:
253 + return self.language_model.sample(logits, sampling_metadata)
254 +
255 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
256 + skip_prefixes = [
257 + 'image_projection', "vision_tower", "image_proj_norm",
258 + "image_pos_embed", "visual_temporal_embed"
259 + ]
260 + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
261 + loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -85,6 +85,7 @@ _TEXT_GENERATION_MODELS = {
85 # [Encoder-decoder]
86 "BartModel": ("bart", "BartForConditionalGeneration"),
87 "BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
88 + "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
89 }
90
91 _EMBEDDING_MODELS = {