A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[VLM] Florence-2 supports online serving (#16164)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Apr 7, 2025 at 19:04 UTC
7c80368710755ab54147eaa0090b7782622fda48
3 files changed
+22
-3
examples/template_florence2.jinja
new
+7
@@ -0,0 +1,7 @@
1
+{%- for message in messages -%}
2
+ {%- if message['role'] == 'user' -%}
3
+ {{- message['content'] -}}
4
+ {%- elif message['role'] == 'assistant' -%}
5
+ {{- message['content'] -}}
6
+ {%- endif -%}
7
+{%- endfor -%}
vllm/entrypoints/chat_utils.py
+2
-2
@@ -487,8 +487,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
487
return "<|endoftext10|>" # 200010 (see vocab.json in hf model)
488
if model_type in ("minicpmo", "minicpmv"):
489
return "(<image>./</image>)"
490
- if model_type in ("blip-2", "fuyu", "paligemma", "pixtral",
491
- "mistral3"):
490
+ if model_type in ("blip-2", "florence2", "fuyu", "paligemma",
491
+ "pixtral", "mistral3"):
492
# These models do not use image tokens in the prompt
493
return None
494
if model_type == "qwen":
vllm/model_executor/models/florence2.py
+13
-1
@@ -10,7 +10,7 @@ import torch
10
import torch.nn as nn
11
import torch.nn.functional as F
12
from einops import rearrange
13
-from transformers import BatchFeature, PretrainedConfig
13
+from transformers import BartTokenizer, BatchFeature, PretrainedConfig
14
15
from vllm.config import VllmConfig
16
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -826,6 +826,18 @@ class Florence2MultiModalProcessor(
826
) -> Union[str, list[int]]:
827
return [self.info.get_hf_config().eos_token_id]
828
829
+ def _apply_hf_processor_tokens_only(
830
+ self,
831
+ prompt_tokens: list[int],
832
+ ) -> list[int]:
833
+ hf_processor = self.info.get_hf_processor()
834
+ tokenizer: BartTokenizer = hf_processor.tokenizer
835
+ prompt_text = tokenizer.decode(prompt_tokens)
836
+ # convert task tokens to prompt
837
+ prompt_text = hf_processor._construct_prompts([prompt_text])[0]
838
+ prompt_tokens = tokenizer.encode(prompt_text, add_special_tokens=False)
839
+ return prompt_tokens
840
+
841
def _call_hf_processor(
842
self,
843
prompt: str,