A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix] Update Florence-2 tokenizer to make grounding tasks work (#16734)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Apr 17, 2025 at 12:17 UTC
cb072ce93b1d5267967e6f97daf9d8b545fd7bc0
5 files changed
+16
-10
examples/offline_inference/encoder_decoder_multimodal.py
+2
-1
@@ -22,7 +22,7 @@ class ModelRequestData(NamedTuple):
22
def run_florence2():
23
engine_args = EngineArgs(
24
model="microsoft/Florence-2-large",
25
- tokenizer="facebook/bart-large",
25
+ tokenizer="Isotr0py/Florence-2-tokenizer",
26
max_num_seqs=8,
27
trust_remote_code=True,
28
limit_mm_per_prompt={"image": 1},
@@ -165,6 +165,7 @@ def main(args):
165
temperature=0,
166
top_p=1.0,
167
max_tokens=64,
168
+ skip_special_tokens=False,
169
)
170
171
start = time.time()
examples/offline_inference/vision_language.py
+1
-1
@@ -150,7 +150,7 @@ def run_florence2(questions: list[str], modality: str) -> ModelRequestData:
150
151
engine_args = EngineArgs(
152
model="microsoft/Florence-2-large",
153
- tokenizer="facebook/bart-large",
153
+ tokenizer="Isotr0py/Florence-2-tokenizer",
154
max_model_len=4096,
155
max_num_seqs=2,
156
trust_remote_code=True,
tests/conftest.py
+2
@@ -925,6 +925,7 @@ class VllmRunner:
925
max_tokens: int,
926
num_logprobs: int,
927
num_prompt_logprobs: Optional[int] = None,
928
+ skip_special_tokens: bool = True,
929
) -> Union[list[TokensTextLogprobs],
930
list[TokensTextLogprobsPromptLogprobs]]:
931
greedy_logprobs_params = SamplingParams(
@@ -932,6 +933,7 @@ class VllmRunner:
933
max_tokens=max_tokens,
934
logprobs=num_logprobs,
935
prompt_logprobs=(num_prompt_logprobs),
936
+ skip_special_tokens=skip_special_tokens,
937
)
938
'''
939
Greedy logprobs generation for vLLM encoder/decoder models
tests/models/encoder_decoder/vision_language/test_florence2.py
+10
-7
@@ -13,12 +13,12 @@ from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13
from ...utils import check_logprobs_close
14
15
MODELS = ["microsoft/Florence-2-base"]
16
-# Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
17
-# Therefore, we borrow the BartTokenizer from the original Bart model
18
-TOKENIZER = "facebook/bart-base"
16
+# Florence-2 model repo's tokenizer config is missing some special tokens.
17
+# Therefore, we use a converted tokenizer from a forked repo
18
+TOKENIZER = "Isotr0py/Florence-2-tokenizer"
19
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20
"stop_sign":
21
- "<CAPTION>", # special task token
21
+ "<OD>", # special task token which will output special tokens
22
"cherry_blossom":
23
"Describe in detail what is shown in the image.",
24
})
@@ -45,7 +45,6 @@ def hf_to_vllm_output(hf_output: tuple[list[int], str,
45
output_ids, output_str, out_logprobs = hf_output
46
47
output_str = output_str.replace("</s>", "").replace("<s>", "")
48
- output_ids = [ids for ids in output_ids if ids not in [0, 2]]
48
49
return output_ids, output_str, out_logprobs
50
@@ -71,8 +70,11 @@ def run_test(
70
enforce_eager=True) as vllm_model:
71
vllm_outputs_per_case = [
72
vllm_model.generate_encoder_decoder_greedy_logprobs(
74
- prompts, max_tokens, num_logprobs=num_logprobs)
75
- for prompts in inputs
73
+ prompts,
74
+ max_tokens,
75
+ num_logprobs=num_logprobs,
76
+ skip_special_tokens=False,
77
+ ) for prompts in inputs
78
]
79
80
hf_inputs = [get_hf_images_prompts(prompts) for prompts in inputs]
@@ -93,6 +95,7 @@ def run_test(
95
outputs_1_lst=vllm_outputs,
96
name_0="hf",
97
name_1="vllm",
98
+ num_outputs_0_skip_tokens=1,
99
)
100
101
tests/models/registry.py
+1
-1
@@ -366,7 +366,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
366
# Florence-2 uses BartFastTokenizer which can't be loaded from AutoTokenizer
367
# Therefore, we borrow the BartTokenizer from the original Bart model
368
"Florence2ForConditionalGeneration": _HfExamplesInfo("microsoft/Florence-2-base", # noqa: E501
369
- tokenizer="facebook/bart-base",
369
+ tokenizer="Isotr0py/Florence-2-tokenizer",
370
trust_remote_code=True), # noqa: E501
371
"MllamaForConditionalGeneration": _HfExamplesInfo("meta-llama/Llama-3.2-11B-Vision-Instruct"), # noqa: E501
372
"WhisperForConditionalGeneration": _HfExamplesInfo("openai/whisper-large-v3"), # noqa: E501