@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 360ddbd37e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Update Pixtral example (#8431)

Roger Wang committed Sep 12, 2024 at 17:31 UTC 360ddbd37ec82d5a83fd02ee94d7401684bc3c92
1 file changed +3 -2
examples/offline_inference_pixtral.py
+3 -2
@@ -11,7 +11,7 @@ from vllm.sampling_params import SamplingParams
11 # - Server:
12 #
13 # ```bash
14 -# vllm serve mistralai/Pixtral-12B-2409 --tokenizer_mode mistral --limit_mm_per_prompt 'image=4' --max_num_batched_tokens 16384
14 +# vllm serve mistralai/Pixtral-12B-2409 --tokenizer-mode mistral --limit-mm-per-prompt 'image=4' --max-model-len 16384
15 # ```
16 #
17 # - Client:
@@ -45,6 +45,7 @@ def run_simple_demo():
45 model_name = "mistralai/Pixtral-12B-2409"
46 sampling_params = SamplingParams(max_tokens=8192)
47
48 + # Lower max_num_seqs or max_model_len on low-VRAM GPUs.
49 llm = LLM(model=model_name, tokenizer_mode="mistral")
50
51 prompt = "Describe this image in one sentence."
@@ -83,7 +84,7 @@ def run_advanced_demo():
84 model=model_name,
85 tokenizer_mode="mistral",
86 limit_mm_per_prompt={"image": max_img_per_msg},
86 - max_num_batched_tokens=max_img_per_msg * max_tokens_per_img,
87 + max_model_len=max_img_per_msg * max_tokens_per_img,
88 )
89
90 prompt = "Describe the following image."