@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a60731247f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Doc] Update mllama example based on official doc (#11567)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Dec 28, 2024 at 08:31 UTC a60731247fba82fae5e71af7a19ea0df96de1caa
1 file changed +14 -1
examples/offline_inference_vision_language.py
+14 -1
@@ -308,7 +308,20 @@ def run_mllama(question: str, modality: str):
308 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
309 )
310
311 - prompt = f"<|image|><|begin_of_text|>{question}"
311 + tokenizer = AutoTokenizer.from_pretrained(model_name)
312 + messages = [{
313 + "role":
314 + "user",
315 + "content": [{
316 + "type": "image"
317 + }, {
318 + "type": "text",
319 + "text": f"{question}"
320 + }]
321 + }]
322 + prompt = tokenizer.apply_chat_template(messages,
323 + add_generation_prompt=True,
324 + tokenize=False)
325 stop_token_ids = None
326 return llm, prompt, stop_token_ids
327