A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Initialize Florence-2 language backbone support (#9555)
Isotr0py committed
Oct 23, 2024 at 18:42 UTC
3ff57ebfcacdd4f7690ed8f5693657de2bdedea8
5 files changed
+428
-8
examples/florence2_inference.py
new
+44
@@ -0,0 +1,44 @@
1
+'''
2
+Demonstrate prompting of text-to-text
3
+encoder/decoder models, specifically Florence-2
4
+'''
5
+# TODO(Isotr0py):
6
+# Move to offline_inference_vision_language.py after porting vision backbone
7
+from vllm import LLM, SamplingParams
8
+
9
+dtype = "float"
10
+
11
+# Create a Florence-2 encoder/decoder model instance
12
+llm = LLM(
13
+ model="microsoft/Florence-2-base",
14
+ tokenizer="facebook/bart-base",
15
+ dtype=dtype,
16
+ trust_remote_code=True,
17
+)
18
+
19
+prompts = [
20
+ "<CAPTION>", "<DETAILED_CAPTION>", "<MORE_DETAILED_CAPTION>",
21
+ "<CAPTION_TO_PHRASE_GROUNDING>", "<OD>", "<DENSE_REGION_CAPTION>",
22
+ "<REGION_PROPOSAL>", "<OCR>", "<OCR_WITH_REGION>"
23
+]
24
+# Create a sampling params object.
25
+sampling_params = SamplingParams(
26
+ temperature=0,
27
+ top_p=1.0,
28
+ min_tokens=0,
29
+ max_tokens=20,
30
+)
31
+
32
+# Generate output tokens from the prompts. The output is a list of
33
+# RequestOutput objects that contain the prompt, generated
34
+# text, and other information.
35
+outputs = llm.generate(prompts, sampling_params)
36
+
37
+# Print the outputs.
38
+for output in outputs:
39
+ prompt = output.prompt
40
+ encoder_prompt = output.encoder_prompt
41
+ generated_text = output.outputs[0].text
42
+ print(f"Encoder prompt: {encoder_prompt!r}, "
43
+ f"Decoder prompt: {prompt!r}, "
44
+ f"Generated text: {generated_text!r}")
tests/conftest.py
+20
-8
@@ -253,7 +253,9 @@ class HfRunner:
253
dtype: str = "half",
254
*,
255
model_kwargs: Optional[Dict[str, Any]] = None,
256
+ is_embedding_model: bool = False,
257
is_sentence_transformer: bool = False,
258
+ skip_tokenizer_init: bool = False,
259
auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM,
260
postprocess_inputs: Callable[[BatchEncoding],
261
BatchEncoding] = identity,
@@ -281,11 +283,12 @@ class HfRunner:
283
**model_kwargs,
284
))
285
284
- self.tokenizer = AutoTokenizer.from_pretrained(
285
- model_name,
286
- torch_dtype=torch_dtype,
287
- trust_remote_code=True,
288
- )
286
+ if not skip_tokenizer_init:
287
+ self.tokenizer = AutoTokenizer.from_pretrained(
288
+ model_name,
289
+ torch_dtype=torch_dtype,
290
+ trust_remote_code=True,
291
+ )
292
293
# don't put this import at the top level
294
# it will call torch.cuda.device_count()
@@ -295,6 +298,8 @@ class HfRunner:
298
torch_dtype=torch_dtype,
299
trust_remote_code=True,
300
)
301
+ if skip_tokenizer_init:
302
+ self.tokenizer = self.processor.tokenizer
303
304
self.postprocess_inputs = postprocess_inputs
305
@@ -535,6 +540,7 @@ class HfRunner:
540
encoder_decoder_prompts: List[ExplicitEncoderDecoderPrompt[str, str]],
541
max_tokens: int,
542
num_logprobs: int,
543
+ images: Optional[PromptImageInput] = None,
544
**kwargs: Any,
545
) -> List[TokensTextLogprobs]:
546
'''
@@ -545,11 +551,17 @@ class HfRunner:
551
all_output_ids: List[List[int]] = []
552
all_output_strs: List[str] = []
553
548
- for (encoder_prompt,
549
- decoder_prompt) in to_enc_dec_tuple_list(encoder_decoder_prompts):
554
+ for i, (encoder_prompt, decoder_prompt) in enumerate(
555
+ to_enc_dec_tuple_list(encoder_decoder_prompts)):
556
+ processor_kwargs: Dict[str, Any] = {
557
+ "text": encoder_prompt,
558
+ "return_tensors": "pt",
559
+ }
560
+ if images is not None and images[i] is not None:
561
+ processor_kwargs["images"] = images[i]
562
563
encoder_input_ids = self.wrap_device(
552
- self.tokenizer(encoder_prompt, return_tensors="pt").input_ids,
564
+ self.processor(**processor_kwargs).input_ids,
565
device=self.model.device.type,
566
)
567
tests/models/encoder_decoder/vision_language/test_florence2.py
new
+102
@@ -0,0 +1,102 @@
1
+from functools import partial
2
+from typing import List, Optional, Tuple, Type
3
+
4
+import pytest
5
+from PIL import Image
6
+
7
+from vllm.inputs.data import ExplicitEncoderDecoderPrompt
8
+from vllm.sequence import SampleLogprobs
9
+
10
+from ....conftest import HfRunner, VllmRunner
11
+from ...utils import check_logprobs_close
12
+
13
+Florence2Prompt = partial(ExplicitEncoderDecoderPrompt,
14
+ decoder_prompt=None,
15
+ mm_processor_kwargs=None)
16
+
17
+MODELS = ["microsoft/Florence-2-base"]
18
+# Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
19
+# Therefore, we borrow the BartTokenizer from the original Bart model
20
+TOKENIZER = "facebook/bart-base"
21
+PROMPTS = [
22
+ Florence2Prompt(encoder_prompt="<CAPTION>"),
23
+ Florence2Prompt(encoder_prompt="<DETAILED_CAPTION>"),
24
+ Florence2Prompt(encoder_prompt="<MORE_DETAILED_CAPTION>"),
25
+ Florence2Prompt(encoder_prompt="<CAPTION_TO_PHRASE_GROUNDING>"),
26
+ Florence2Prompt(encoder_prompt="<DENSE_REGION_CAPTION>"),
27
+ Florence2Prompt(encoder_prompt="<REGION_PROPOSAL>"),
28
+ Florence2Prompt(encoder_prompt="<OCR_WITH_REGION>"),
29
+ Florence2Prompt(encoder_prompt="<OCR>"),
30
+ Florence2Prompt(encoder_prompt="<OD>"),
31
+]
32
+
33
+
34
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
35
+ Optional[SampleLogprobs]], ):
36
+ """Sanitize vllm output to be comparable with hf output."""
37
+ output_ids, output_str, out_logprobs = vllm_output
38
+
39
+ hf_output_str = "</s><s>" + output_str + "</s>"
40
+
41
+ return output_ids, hf_output_str, out_logprobs
42
+
43
+
44
+def run_test(
45
+ hf_runner: Type[HfRunner],
46
+ vllm_runner: Type[VllmRunner],
47
+ prompts: List[ExplicitEncoderDecoderPrompt],
48
+ model: str,
49
+ *,
50
+ dtype: str,
51
+ max_tokens: int,
52
+ num_logprobs: int,
53
+ tensor_parallel_size: int,
54
+ distributed_executor_backend: Optional[str] = None,
55
+) -> None:
56
+ with vllm_runner(model,
57
+ tokenizer_name=TOKENIZER,
58
+ dtype=dtype,
59
+ tensor_parallel_size=tensor_parallel_size,
60
+ distributed_executor_backend=distributed_executor_backend,
61
+ enforce_eager=True) as vllm_model:
62
+ vllm_outputs = vllm_model.generate_encoder_decoder_greedy_logprobs(
63
+ prompts, max_tokens, num_logprobs)
64
+
65
+ # Florence-2 processors require image inputs
66
+ dummy_image = Image.new(mode="RGB", size=(2, 2))
67
+ with hf_runner(model, dtype=dtype, skip_tokenizer_init=True) as hf_model:
68
+ hf_model.model.get_output_embeddings = lambda: \
69
+ hf_model.model.language_model.lm_head
70
+ hf_outputs = (hf_model.generate_encoder_decoder_greedy_logprobs_limit(
71
+ prompts,
72
+ max_tokens,
73
+ num_logprobs,
74
+ images=[dummy_image] * len(prompts),
75
+ ))
76
+
77
+ check_logprobs_close(
78
+ outputs_0_lst=hf_outputs,
79
+ outputs_1_lst=[
80
+ vllm_to_hf_output(vllm_output) for vllm_output in vllm_outputs
81
+ ],
82
+ name_0="hf",
83
+ name_1="vllm",
84
+ )
85
+
86
+
87
+@pytest.mark.parametrize("model", MODELS)
88
+@pytest.mark.parametrize("dtype", ["float"])
89
+@pytest.mark.parametrize("max_tokens", [64])
90
+@pytest.mark.parametrize("num_logprobs", [5])
91
+def test_models(hf_runner, vllm_runner, model, dtype, max_tokens,
92
+ num_logprobs) -> None:
93
+ run_test(
94
+ hf_runner,
95
+ vllm_runner,
96
+ PROMPTS,
97
+ model,
98
+ dtype=dtype,
99
+ max_tokens=max_tokens,
100
+ num_logprobs=num_logprobs,
101
+ tensor_parallel_size=1,
102
+ )
vllm/model_executor/models/florence2.py
new
+261
@@ -0,0 +1,261 @@
1
+import math
2
+from typing import Iterable, List, Optional, Tuple
3
+
4
+import torch
5
+import torch.nn as nn
6
+from transformers import PretrainedConfig
7
+
8
+from vllm.attention import AttentionMetadata
9
+from vllm.config import CacheConfig
10
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
11
+from vllm.model_executor.layers.quantization.base_config import (
12
+ QuantizationConfig)
13
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
14
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
15
+from vllm.model_executor.models.bart import (BartDecoder, BartEncoder,
16
+ BartParallelLMHead,
17
+ BartScaledWordEmbedding)
18
+from vllm.model_executor.sampling_metadata import SamplingMetadata
19
+from vllm.sequence import IntermediateTensors
20
+
21
+from .utils import AutoWeightsLoader
22
+
23
+
24
+class Florence2LanguageModel(nn.Module):
25
+
26
+ def __init__(self,
27
+ config: PretrainedConfig,
28
+ cache_config: Optional[CacheConfig] = None,
29
+ quant_config: Optional[QuantizationConfig] = None):
30
+ super().__init__()
31
+ self.config = config
32
+
33
+ self.padding_idx = config.pad_token_id
34
+ self.vocab_size = config.vocab_size
35
+
36
+ self.shared = BartScaledWordEmbedding(self.vocab_size, config.d_model)
37
+ self.encoder = BartEncoder(config,
38
+ cache_config=cache_config,
39
+ quant_config=quant_config)
40
+ self.decoder = BartDecoder(config,
41
+ cache_config=cache_config,
42
+ quant_config=quant_config)
43
+
44
+ if self.config.tie_word_embeddings:
45
+ self.encoder.embed_tokens.weight = self.shared.weight
46
+ self.decoder.embed_tokens.weight = self.shared.weight
47
+
48
+ def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
49
+ encoder_input_ids: torch.Tensor,
50
+ encoder_positions: torch.Tensor, kv_caches: List[torch.Tensor],
51
+ attn_metadata: AttentionMetadata) -> torch.Tensor:
52
+ r"""
53
+ Args:
54
+ input_ids
55
+ Indices of *decoder* input sequence tokens in the vocabulary.
56
+ Padding will be ignored by default should you
57
+ provide it.
58
+ positions
59
+ Positions of *decoder* input sequence tokens.
60
+ encoder_input_ids
61
+ Indices of *encoder* input sequence tokens in the vocabulary.
62
+ encoder_positions:
63
+ Positions of *encoder* input sequence tokens.
64
+ kv_caches:
65
+ Layer-wise list of KV cache tensors
66
+ attn_metadata:
67
+ vLLM Attention metadata structure
68
+ Returns:
69
+ Model output torch.Tensor
70
+ """
71
+
72
+ encoder_hidden_states = None
73
+
74
+ if encoder_input_ids.numel() > 0:
75
+ # Run encoder attention if a non-zero number of encoder tokens
76
+ # are provided as input
77
+ encoder_hidden_states = self.encoder(input_ids=encoder_input_ids,
78
+ positions=encoder_positions,
79
+ kv_caches=kv_caches,
80
+ attn_metadata=attn_metadata)
81
+
82
+ # decoder outputs consists of
83
+ # (dec_features, past_key_value, dec_hidden, dec_attn)
84
+ decoder_outputs = self.decoder(
85
+ decoder_input_ids=input_ids,
86
+ decoder_positions=positions,
87
+ encoder_hidden_states=encoder_hidden_states,
88
+ kv_caches=kv_caches,
89
+ attn_metadata=attn_metadata)
90
+
91
+ return decoder_outputs
92
+
93
+
94
+class Florence2LanguageForConditionalGeneration(nn.Module):
95
+
96
+ def __init__(self,
97
+ config: PretrainedConfig,
98
+ cache_config: Optional[CacheConfig] = None,
99
+ quant_config: Optional[QuantizationConfig] = None):
100
+ super().__init__()
101
+ self.config = config
102
+ self.model = Florence2LanguageModel(config,
103
+ cache_config=cache_config,
104
+ quant_config=quant_config)
105
+ embed_scale = math.sqrt(
106
+ config.d_model) if config.scale_embedding else 1.0
107
+
108
+ self.vocab_size = config.vocab_size
109
+ self.lm_head = BartParallelLMHead(self.vocab_size,
110
+ config.d_model,
111
+ embed_scale=embed_scale)
112
+
113
+ self.logits_processor = LogitsProcessor(self.vocab_size,
114
+ config.vocab_size)
115
+ self.sampler = Sampler()
116
+
117
+ def forward(
118
+ self,
119
+ input_ids: torch.Tensor,
120
+ positions: torch.Tensor,
121
+ encoder_input_ids: torch.Tensor,
122
+ encoder_positions: torch.Tensor,
123
+ kv_caches: List[torch.Tensor],
124
+ attn_metadata: AttentionMetadata,
125
+ **kwargs,
126
+ ) -> torch.Tensor:
127
+ r"""
128
+ Args:
129
+ input_ids
130
+ torch.Tensor of *decoder* input token ids.
131
+ positions
132
+ torch.Tensor of *decoder* position indices.
133
+ encoder_input_ids
134
+ torch.Tensor of *encoder* input token ids.
135
+ encoder_positions
136
+ torch.Tensor of *encoder* position indices
137
+ kv_caches:
138
+ Layer-wise list of KV cache tensors
139
+ attn_metadata:
140
+ vLLM Attention metadata structure
141
+ Returns:
142
+ Output torch.Tensor
143
+ """
144
+ return self.model(input_ids, positions, encoder_input_ids,
145
+ encoder_positions, kv_caches, attn_metadata)
146
+
147
+ def compute_logits(
148
+ self,
149
+ hidden_states: torch.Tensor,
150
+ sampling_metadata: SamplingMetadata,
151
+ ) -> Optional[torch.Tensor]:
152
+ logits = self.logits_processor(self.lm_head, hidden_states,
153
+ sampling_metadata)
154
+ return logits
155
+
156
+ def sample(self, logits: torch.Tensor,
157
+ sampling_metadata: SamplingMetadata) -> SamplerOutput:
158
+ next_tokens = self.sampler(logits, sampling_metadata)
159
+ return next_tokens
160
+
161
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
162
+ stacked_params_mapping = [
163
+ # (param_name, shard_name, shard_id)
164
+ ("qkv_proj", "q_proj", "q"),
165
+ ("qkv_proj", "k_proj", "k"),
166
+ ("qkv_proj", "v_proj", "v"),
167
+ ]
168
+
169
+ params_dict = dict(self.named_parameters())
170
+ for name, loaded_weight in weights:
171
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
172
+ if weight_name not in name:
173
+ continue
174
+
175
+ param = params_dict[name.replace(weight_name, param_name)]
176
+ weight_loader = param.weight_loader
177
+ weight_loader(param, loaded_weight, shard_id)
178
+ break
179
+ else:
180
+ if "final_logits_bias" in name:
181
+ continue
182
+ if self.config.tie_word_embeddings and "embed_tokens" in name:
183
+ continue
184
+ param = params_dict[name]
185
+ weight_loader = getattr(param, "weight_loader",
186
+ default_weight_loader)
187
+ weight_loader(param, loaded_weight)
188
+
189
+
190
+class Florence2ForConditionalGeneration(nn.Module):
191
+
192
+ def __init__(self,
193
+ config: PretrainedConfig,
194
+ cache_config: Optional[CacheConfig] = None,
195
+ quant_config: Optional[QuantizationConfig] = None):
196
+ super().__init__()
197
+
198
+ # TODO(Isotr0py): Add vision backbone
199
+ self.language_model = Florence2LanguageForConditionalGeneration(
200
+ config=config.text_config,
201
+ cache_config=cache_config,
202
+ quant_config=quant_config)
203
+
204
+ @property
205
+ def sampler(self):
206
+ return self.language_model.sampler
207
+
208
+ def forward(
209
+ self,
210
+ input_ids: torch.Tensor,
211
+ positions: torch.Tensor,
212
+ kv_caches: List[torch.Tensor],
213
+ attn_metadata: AttentionMetadata,
214
+ intermediate_tensors: Optional[IntermediateTensors] = None,
215
+ *,
216
+ encoder_input_ids: torch.Tensor,
217
+ encoder_positions: torch.Tensor,
218
+ **kwargs,
219
+ ) -> torch.Tensor:
220
+ r"""
221
+ Args:
222
+ input_ids
223
+ torch.Tensor of *decoder* input token ids.
224
+ positions
225
+ torch.Tensor of *decoder* position indices.
226
+ encoder_input_ids
227
+ torch.Tensor of *encoder* input token ids.
228
+ encoder_positions
229
+ torch.Tensor of *encoder* position indices
230
+ kv_caches:
231
+ Layer-wise list of KV cache tensors
232
+ attn_metadata:
233
+ vLLM Attention metadata structure
234
+ Returns:
235
+ Output torch.Tensor
236
+ """
237
+ return self.language_model(input_ids, positions, encoder_input_ids,
238
+ encoder_positions, kv_caches, attn_metadata)
239
+
240
+ def compute_logits(
241
+ self,
242
+ hidden_states: torch.Tensor,
243
+ sampling_metadata: SamplingMetadata,
244
+ ) -> Optional[torch.Tensor]:
245
+ return self.language_model.compute_logits(hidden_states,
246
+ sampling_metadata)
247
+
248
+ def sample(
249
+ self,
250
+ logits: torch.Tensor,
251
+ sampling_metadata: SamplingMetadata,
252
+ ) -> SamplerOutput:
253
+ return self.language_model.sample(logits, sampling_metadata)
254
+
255
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
256
+ skip_prefixes = [
257
+ 'image_projection', "vision_tower", "image_proj_norm",
258
+ "image_pos_embed", "visual_temporal_embed"
259
+ ]
260
+ loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
261
+ loader.load_weights(weights)
vllm/model_executor/models/registry.py
+1
@@ -85,6 +85,7 @@ _TEXT_GENERATION_MODELS = {
85
# [Encoder-decoder]
86
"BartModel": ("bart", "BartForConditionalGeneration"),
87
"BartForConditionalGeneration": ("bart", "BartForConditionalGeneration"),
88
+ "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
89
}
90
91
_EMBEDDING_MODELS = {