A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Doc] Update mllama example based on official doc (#11567)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Dec 28, 2024 at 08:31 UTC
a60731247fba82fae5e71af7a19ea0df96de1caa
1 file changed
+14
-1
examples/offline_inference_vision_language.py
+14
-1
@@ -308,7 +308,20 @@ def run_mllama(question: str, modality: str):
308
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
309
)
310
311
- prompt = f"<|image|><|begin_of_text|>{question}"
311
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
312
+ messages = [{
313
+ "role":
314
+ "user",
315
+ "content": [{
316
+ "type": "image"
317
+ }, {
318
+ "type": "text",
319
+ "text": f"{question}"
320
+ }]
321
+ }]
322
+ prompt = tokenizer.apply_chat_template(messages,
323
+ add_generation_prompt=True,
324
+ tokenize=False)
325
stop_token_ids = None
326
return llm, prompt, stop_token_ids
327