A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Misc] Update Mistral-3.1 example (#16147)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 7, 2025 at 11:57 UTC
0a5738672158c07d5d66ac9f8c9e8876f2939bb9
1 file changed
+22
-8
examples/offline_inference/mistral-small.py
+22
-8
@@ -13,9 +13,14 @@ from vllm.sampling_params import SamplingParams
13
# - Server:
14
#
15
# ```bash
16
+# # Mistral format
17
# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
18
# --tokenizer-mode mistral --config-format mistral --load-format mistral \
19
# --limit-mm-per-prompt 'image=4' --max-model-len 16384
20
+#
21
+# # HF format
22
+# vllm serve mistralai/Mistral-Small-3.1-24B-Instruct-2503 \
23
+# --limit-mm-per-prompt 'image=4' --max-model-len 16384
24
# ```
25
#
26
# - Client:
@@ -44,19 +49,22 @@ from vllm.sampling_params import SamplingParams
49
# python demo.py simple
50
# python demo.py advanced
51
52
+# Lower max_model_len and/or max_num_seqs on low-VRAM GPUs.
53
+# These scripts have been tested on 2x L40 GPUs
54
+
55
56
def run_simple_demo(args: argparse.Namespace):
57
model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
58
sampling_params = SamplingParams(max_tokens=8192)
59
52
- # Lower max_model_len and/or max_num_seqs on low-VRAM GPUs.
60
llm = LLM(
61
model=model_name,
55
- tokenizer_mode="mistral",
56
- config_format="mistral",
57
- load_format="mistral",
62
+ tokenizer_mode="mistral" if args.format == "mistral" else "auto",
63
+ config_format="mistral" if args.format == "mistral" else "auto",
64
+ load_format="mistral" if args.format == "mistral" else "auto",
65
max_model_len=4096,
66
max_num_seqs=2,
67
+ tensor_parallel_size=2,
68
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
69
)
70
@@ -88,17 +96,18 @@ def run_simple_demo(args: argparse.Namespace):
96
97
def run_advanced_demo(args: argparse.Namespace):
98
model_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
91
- max_img_per_msg = 5
99
+ max_img_per_msg = 3
100
max_tokens_per_img = 4096
101
102
sampling_params = SamplingParams(max_tokens=8192, temperature=0.7)
103
llm = LLM(
104
model=model_name,
97
- tokenizer_mode="mistral",
98
- config_format="mistral",
99
- load_format="mistral",
105
+ tokenizer_mode="mistral" if args.format == "mistral" else "auto",
106
+ config_format="mistral" if args.format == "mistral" else "auto",
107
+ load_format="mistral" if args.format == "mistral" else "auto",
108
limit_mm_per_prompt={"image": max_img_per_msg},
109
max_model_len=max_img_per_msg * max_tokens_per_img,
110
+ tensor_parallel_size=2,
111
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
112
)
113
@@ -166,6 +175,11 @@ def main():
175
help="Specify the demo mode: 'simple' or 'advanced'",
176
)
177
178
+ parser.add_argument('--format',
179
+ choices=["mistral", "hf"],
180
+ default="mistral",
181
+ help='Specify the format of the model to load.')
182
+
183
parser.add_argument(
184
'--disable-mm-preprocessor-cache',
185
action='store_true',