A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[Bugfix] Fix Mistral-small `--format` (#36782)
Signed-off-by: 12010486 <silvia.colabrese@intel.com>
Silvia Colabrese committed
Mar 11, 2026 at 12:47 UTC
f33251ffc851405a36a95560975ea6963d8a2706
1 file changed
+6
-6
examples/offline_inference/mistral-small.py
+6
-6
@@ -62,9 +62,9 @@ def run_simple_demo(args: argparse.Namespace):
62
63
llm = LLM(
64
model=model_name,
65
- tokenizer_mode="mistral" if args.format == "mistral" else "auto",
66
- config_format="mistral" if args.format == "mistral" else "auto",
67
- load_format="mistral" if args.format == "mistral" else "auto",
65
+ tokenizer_mode="mistral" if args.format == "mistral" else "hf",
66
+ config_format="mistral" if args.format == "mistral" else "hf",
67
+ load_format="mistral" if args.format == "mistral" else "hf",
68
limit_mm_per_prompt={"image": 1},
69
max_model_len=4096,
70
max_num_seqs=2,
@@ -102,9 +102,9 @@ def run_advanced_demo(args: argparse.Namespace):
102
sampling_params = SamplingParams(max_tokens=8192, temperature=0.7)
103
llm = LLM(
104
model=model_name,
105
- tokenizer_mode="mistral" if args.format == "mistral" else "auto",
106
- config_format="mistral" if args.format == "mistral" else "auto",
107
- load_format="mistral" if args.format == "mistral" else "auto",
105
+ tokenizer_mode="mistral" if args.format == "mistral" else "hf",
106
+ config_format="mistral" if args.format == "mistral" else "hf",
107
+ load_format="mistral" if args.format == "mistral" else "hf",
108
limit_mm_per_prompt={"image": max_img_per_msg},
109
max_model_len=max_img_per_msg * max_tokens_per_img,
110
tensor_parallel_size=2,