A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Doc] Update Mistral Small 3.1/Pixtral example (#15184)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Mar 19, 2025 at 21:46 UTC
34868b106a8a1c3f2f1e1f0cff7e48d2d0d29a35
1 file changed
+8
-2
examples/offline_inference/mistral-small.py
renamed
+8
-2
@@ -6,14 +6,16 @@ import argparse
6
from vllm import LLM
7
from vllm.sampling_params import SamplingParams
8
9
-# This script is an offline demo for running Mistral-Small-3
9
+# This script is an offline demo for running Mistral-Small-3.1
10
#
11
# If you want to run a server/client setup, please follow this code:
12
#
13
# - Server:
14
#
15
# ```bash
16
-# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 --tokenizer-mode mistral --limit-mm-per-prompt 'image=4' --max-model-len 16384
16
+# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
17
+# --tokenizer-mode mistral --config-format mistral --load-format mistral \
18
+# --limit-mm-per-prompt 'image=4' --max-model-len 16384
19
# ```
20
#
21
# - Client:
@@ -51,6 +53,8 @@ def run_simple_demo(args: argparse.Namespace):
53
llm = LLM(
54
model=model_name,
55
tokenizer_mode="mistral",
56
+ config_format="mistral",
57
+ load_format="mistral",
58
max_model_len=4096,
59
max_num_seqs=2,
60
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
@@ -91,6 +95,8 @@ def run_advanced_demo(args: argparse.Namespace):
95
llm = LLM(
96
model=model_name,
97
tokenizer_mode="mistral",
98
+ config_format="mistral",
99
+ load_format="mistral",
100
limit_mm_per_prompt={"image": max_img_per_msg},
101
max_model_len=max_img_per_msg * max_tokens_per_img,
102
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,