@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67abdbb42f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[VLM][Doc] Add `stop_token_ids` to InternVL example (#7354)

Isotr0py committed Aug 9, 2024 at 22:51 UTC 67abdbb42fdbb59c274130368981c0d0ac3539e3
1 file changed +17 -6
examples/offline_inference_vision_language.py
+17 -6
@@ -124,16 +124,27 @@ def run_minicpmv(question):
124
125 # InternVL
126 def run_internvl(question):
127 - # Generally, InternVL can use chatml template for conversation
128 - TEMPLATE = "<|im_start|>User\n{prompt}<|im_end|>\n<|im_start|>Assistant\n"
129 - prompt = f"<image>\n{question}\n"
130 - prompt = TEMPLATE.format(prompt=prompt)
127 + model_name = "OpenGVLab/InternVL2-2B"
128 +
129 llm = LLM(
132 - model="OpenGVLab/InternVL2-4B",
130 + model=model_name,
131 trust_remote_code=True,
132 max_num_seqs=5,
133 )
136 - stop_token_ids = None
134 +
135 + tokenizer = AutoTokenizer.from_pretrained(model_name,
136 + trust_remote_code=True)
137 + messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
138 + prompt = tokenizer.apply_chat_template(messages,
139 + tokenize=False,
140 + add_generation_prompt=True)
141 +
142 + # Stop tokens for InternVL
143 + # models variants may have different stop tokens
144 + # please refer to the model card for the correct "stop words":
145 + # https://huggingface.co/OpenGVLab/InternVL2-2B#service
146 + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
147 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
148 return llm, prompt, stop_token_ids
149
150