57430fc95c · tduncan/Nvidia-Optimization-Stack-VLLM

docs/features/tool_calling.md

+18 -5

@@ -142,7 +142,7 @@ Flags: `--tool-call-parser hermes`

142 Supported models:

143

144 * `mistralai/Mistral-7B-Instruct-v0.3` (confirmed)

145 -* Additional mistral function-calling models are compatible as well.

145 +* Additional Mistral function-calling models are compatible as well.

146

147 Known issues:

148

@@ -158,12 +158,25 @@ Known issues:

158

159 Recommended flags:

160

161 -1. To use [mistral-common](https://github.com/mistralai/mistral-common) the official Mistral tokenization backend:

161 +1. To use the official Mistral AI's format:

162

163 - `--tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral`

163 + `--tool-call-parser mistral`

164

165 -2. To use the default Transformers tokenization backend:

166 - `--tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`

165 +2. To use the Transformers format when available:

166 +

167 + `--tokenizer_mode hf --config_format hf --load_format hf --tool-call-parser mistral --chat-template examples/tool_chat_template_mistral_parallel.jinja`

168 +

169 +!!! note

170 + Models officially released by Mistral AI have two possible formats:

171 +

172 + 1. The official format that is used by default with `auto` or `mistral` arguments:

173 +

174 + `--tokenizer_mode mistral --config_format mistral --load_format mistral`

175 + This format uses [mistral-common](https://github.com/mistralai/mistral-common), the Mistral AI's tokenizer backend.

176 +

177 + 2. The Transformers format, when available, that is used with `hf` arguments:

178 +

179 + `--tokenizer_mode hf --config_format hf --load_format hf --chat-template examples/tool_chat_template_mistral_parallel.jinja`

180

181 ### Llama Models (`llama3_json`)

182

tests/models/language/generation/test_mistral.py

+1 -1

@@ -208,7 +208,7 @@ def test_mistral_format(

208 with vllm_runner(

209 model,

210 dtype=dtype,

211 - tokenizer_mode="auto",

211 + tokenizer_mode="hf",

212 load_format="safetensors",

213 config_format="hf",

214 ) as hf_format_model:

tests/models/multimodal/test_mapping.py

+13 -1

@@ -50,12 +50,24 @@ def test_hf_model_weights_mapper(model_arch: str):

50 model_info.check_available_online(on_fail="skip")

51 model_info.check_transformers_version(on_fail="skip")

52

53 + is_mistral_model = model_arch in [

54 + "Mistral3ForConditionalGeneration",

55 + "PixtralForConditionalGeneration",

56 + "VoxtralForConditionalGeneration",

57 + ]

58 +

59 + if not is_mistral_model or model_info.tokenizer_mode == "mistral":

60 + tokenizer_mode = model_info.tokenizer_mode

61 + else:

62 + tokenizer_mode = "hf"

63 +

64 model_id = model_info.default

65

66 model_config = ModelConfig(

67 model_id,

68 tokenizer=model_info.tokenizer or model_id,

58 - tokenizer_mode=model_info.tokenizer_mode,

69 + tokenizer_mode=tokenizer_mode,

70 + config_format="hf",

71 revision=model_info.revision,

72 trust_remote_code=model_info.trust_remote_code,

73 hf_overrides=model_info.hf_overrides,

tests/models/quantization/test_bitsandbytes.py

+3

@@ -259,6 +259,9 @@ def validate_generated_texts(

259 tensor_parallel_size=vllm_tp_size,

260 enforce_eager=False,

261 default_torch_num_threads=1,

262 + tokenizer_mode="hf",

263 + load_format="hf",

264 + config_format="hf",

265 ) as llm:

266 vllm_outputs = llm.generate_greedy(prompts, max_tokens)

267 vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")

tests/tool_use/utils.py

+6

@@ -128,6 +128,12 @@ CONFIGS: dict[str, ServerConfig] = {

128 "arguments": [

129 "--enforce-eager",

130 "--no-enable-prefix-caching",

131 + "--tokenizer_mode",

132 + "hf",

133 + "--load_format",

134 + "hf",

135 + "--config_format",

136 + "hf",

137 "--tool-call-parser",

138 "mistral",

139 "--chat-template",

tests/transformers_utils/test_config.py new

+62

@@ -0,0 +1,62 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +

5 +import tempfile

6 +from pathlib import Path

7 +from unittest.mock import MagicMock, call, patch

8 +

9 +import pytest

10 +

11 +from vllm.transformers_utils.config import list_filtered_repo_files

12 +

13 +

14 +@pytest.mark.parametrize(

15 + "allow_patterns,expected_relative_files",

16 + [

17 + (

18 + ["*.json", "correct*.txt"],

19 + ["json_file.json", "subfolder/correct.txt", "correct_2.txt"],

20 + ),

21 + ],

22 +)

23 +def test_list_filtered_repo_files(

24 + allow_patterns: list[str], expected_relative_files: list[str]

25 +):

26 + with tempfile.TemporaryDirectory() as tmp_dir:

27 + # Prep folder and files

28 + path_tmp_dir = Path(tmp_dir)

29 + subfolder = path_tmp_dir / "subfolder"

30 + subfolder.mkdir()

31 + (path_tmp_dir / "json_file.json").touch()

32 + (path_tmp_dir / "correct_2.txt").touch()

33 + (path_tmp_dir / "uncorrect.txt").touch()

34 + (path_tmp_dir / "uncorrect.jpeg").touch()

35 + (subfolder / "correct.txt").touch()

36 + (subfolder / "uncorrect_sub.txt").touch()

37 +

38 + def _glob_path() -> list[str]:

39 + return [

40 + str(file.relative_to(path_tmp_dir))

41 + for file in path_tmp_dir.glob("**/*")

42 + if file.is_file()

43 + ]

44 +

45 + # Patch list_repo_files called by fn

46 + with patch(

47 + "vllm.transformers_utils.config.list_repo_files",

48 + MagicMock(return_value=_glob_path()),

49 + ) as mock_list_repo_files:

50 + out_files = sorted(

51 + list_filtered_repo_files(

52 + tmp_dir, allow_patterns, "revision", "model", "token"

53 + )

54 + )

55 + assert out_files == sorted(expected_relative_files)

56 + assert mock_list_repo_files.call_count == 1

57 + assert mock_list_repo_files.call_args_list[0] == call(

58 + repo_id=tmp_dir,

59 + revision="revision",

60 + repo_type="model",

61 + token="token",

62 + )

tests/transformers_utils/test_utils.py

+5 -1

@@ -2,7 +2,11 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4

5 -from vllm.transformers_utils.utils import is_cloud_storage, is_gcs, is_s3

5 +from vllm.transformers_utils.utils import (

6 + is_cloud_storage,

7 + is_gcs,

8 + is_s3,

9 +)

10

11

12 def test_is_gcs():

tests/v1/entrypoints/llm/test_struct_output_generate.py

+11 -3

@@ -46,11 +46,15 @@ EAGLE_SPEC_CONFIG = {

46

47 PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [

48 ("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "auto", None),

49 - ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", None),

49 + # FIXME: Since "auto" will use Mistral tokenizer and these backends do not support

50 + # it, we skip these tests for now.

51 + # ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", None),

52 + # ("mistralai/Ministral-8B-Instruct-2410", "lm-format-enforcer", "auto", None),

53 + ("mistralai/Ministral-8B-Instruct-2410", "guidance", "hf", None),

54 pytest.param(

55 "mistralai/Ministral-8B-Instruct-2410",

56 "lm-format-enforcer",

53 - "auto",

57 + "hf",

58 None,

59 marks=pytest.mark.skip(

60 reason=(

@@ -80,7 +84,7 @@ PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [

84 # ("mistralai/Ministral-8B-Instruct-2410", "outlines", "mistral", None),

85 # ("Qwen/Qwen2.5-1.5B-Instruct", "guidance", "auto"),

86 ("mistralai/Ministral-8B-Instruct-2410", "outlines", "auto", NGRAM_SPEC_CONFIG),

83 - ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto", NGRAM_SPEC_CONFIG),

87 + ("mistralai/Ministral-8B-Instruct-2410", "guidance", "hf", NGRAM_SPEC_CONFIG),

88 ("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar", "auto", NGRAM_SPEC_CONFIG),

89 ("meta-llama/Meta-Llama-3.1-8B-Instruct", "xgrammar", "auto", EAGLE_SPEC_CONFIG),

90 ]

@@ -151,6 +155,8 @@ def test_structured_output(

155 ),

156 seed=120,

157 tokenizer_mode=tokenizer_mode,

158 + load_format="auto" if not model_name.startswith("mistralai/") else "hf",

159 + config_format="auto" if not model_name.startswith("mistralai/") else "hf",

160 speculative_config=speculative_config,

161 )

162

@@ -720,6 +726,8 @@ def test_structured_output_auto_mode(

726 max_model_len=1024,

727 structured_outputs_config=dict(backend="auto"),

728 tokenizer_mode=tokenizer_mode,

729 + load_format="auto",

730 + config_format="auto",

731 )

732

733 sampling_params = SamplingParams(

vllm/config/model.py

+5 -4

@@ -81,7 +81,7 @@ TaskOption = Literal[

81 "transcription",

82 "draft",

83 ]

84 -TokenizerMode = Literal["auto", "slow", "mistral", "custom"]

84 +TokenizerMode = Literal["auto", "hf", "slow", "mistral", "custom"]

85 ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]

86 LogprobsMode = Literal[

87 "raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"

@@ -130,7 +130,8 @@ class ModelConfig:

130 name or path will be used."""

131 tokenizer_mode: TokenizerMode = "auto"

132 """Tokenizer mode:\n

133 - - "auto" will use the fast tokenizer if available.\n

133 + - "auto" will use "hf" tokenizer if Mistral's tokenizer is not available.\n

134 + - "hf" will use the fast tokenizer if available.\n

135 - "slow" will always use the slow tokenizer.\n

136 - "mistral" will always use the tokenizer from `mistral_common`.\n

137 - "custom" will use --tokenizer to select the preregistered tokenizer."""

@@ -241,8 +242,8 @@ class ModelConfig:

242 first one."""

243 config_format: str | ConfigFormat = "auto"

244 """The format of the model config to load:\n

244 - - "auto" will try to load the config in hf format if available else it

245 - will try to load in mistral format.\n

245 + - "auto" will try to load the config in hf format if available after trying

246 + to load in mistral format.\n

247 - "hf" will load the config in hf format.\n

248 - "mistral" will load the config in mistral format."""

249 hf_token: bool | str | None = None

vllm/model_executor/model_loader/__init__.py

+2

@@ -30,6 +30,7 @@ logger = init_logger(__name__)

30 # if a new load format is added here

31 LoadFormats = Literal[

32 "auto",

33 + "hf",

34 "bitsandbytes",

35 "dummy",

36 "fastsafetensors",

@@ -45,6 +46,7 @@ LoadFormats = Literal[

46 ]

47 _LOAD_FORMAT_TO_MODEL_LOADER: dict[str, type[BaseModelLoader]] = {

48 "auto": DefaultModelLoader,

49 + "hf": DefaultModelLoader,

50 "bitsandbytes": BitsAndBytesModelLoader,

51 "dummy": DummyModelLoader,

52 "fastsafetensors": DefaultModelLoader,

vllm/model_executor/model_loader/default_loader.py

+19 -1

@@ -31,6 +31,7 @@ from vllm.model_executor.model_loader.weight_utils import (

31 safetensors_weights_iterator,

32 )

33 from vllm.platforms import current_platform

34 +from vllm.transformers_utils.config import list_filtered_repo_files

35

36 logger = init_logger(__name__)

37

@@ -96,8 +97,25 @@ class DefaultModelLoader(BaseModelLoader):

97 load_format = self.load_config.load_format

98 use_safetensors = False

99 index_file = SAFE_WEIGHTS_INDEX_NAME

99 - # Some quantized models use .pt files for storing the weights.

100 +

101 + # First check for 'auto' format that mistral files format are present.

102 + # This is to load mistral models with official format by default.

103 if load_format == "auto":

104 + load_format = (

105 + "mistral"

106 + if len(

107 + list_filtered_repo_files(

108 + model_name_or_path=model_name_or_path,

109 + allow_patterns=["consolidated*.safetensors"],

110 + revision=revision,

111 + )

112 + )

113 + > 0

114 + else "hf"

115 + )

116 +

117 + # Some quantized models use .pt files for storing the weights.

118 + if load_format == "hf":

119 allow_patterns = ["*.safetensors", "*.bin"]

120 elif load_format == "safetensors" or load_format == "fastsafetensors":

121 use_safetensors = True

vllm/transformers_utils/config.py

+43 -3

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 +import fnmatch

5 import json

6 import os

7 import time

@@ -355,6 +356,41 @@ def list_repo_files(

356 return with_retry(lookup_files, "Error retrieving file list")

357

358

359 +def list_filtered_repo_files(

360 + model_name_or_path: str,

361 + allow_patterns: list[str],

362 + revision: str | None = None,

363 + repo_type: str | None = None,

364 + token: str | bool | None = None,

365 +) -> list[str]:

366 + try:

367 + all_files = list_repo_files(

368 + repo_id=model_name_or_path,

369 + revision=revision,

370 + token=token,

371 + repo_type=repo_type,

372 + )

373 + except Exception:

374 + logger.error(

375 + "Error retrieving file list. Please ensure your `model_name_or_path`"

376 + "`repo_type`, `token` and `revision` arguments are correctly set. "

377 + "Returning an empty list."

378 + )

379 + return []

380 +

381 + file_list = []

382 + # Filter patterns on filenames

383 + for pattern in allow_patterns:

384 + file_list.extend(

385 + [

386 + file

387 + for file in all_files

388 + if fnmatch.fnmatch(os.path.basename(file), pattern)

389 + ]

390 + )

391 + return file_list

392 +

393 +

394 def file_exists(

395 repo_id: str,

396 file_name: str,

@@ -619,10 +655,14 @@ def get_config(

655

656 if config_format == "auto":

657 try:

622 - if is_gguf or file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):

623 - config_format = "hf"

624 - elif file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):

658 + # First check for Mistral to avoid defaulting to

659 + # Transformers implementation.

660 + if file_or_path_exists(model, MISTRAL_CONFIG_NAME, revision=revision):

661 config_format = "mistral"

662 + elif is_gguf or file_or_path_exists(

663 + model, HF_CONFIG_NAME, revision=revision

664 + ):

665 + config_format = "hf"

666 else:

667 raise ValueError(

668 "Could not detect config format for no config file found. "

vllm/transformers_utils/configs/mistral.py

+1 -1

@@ -118,7 +118,7 @@ def _remap_general_mistral_args(config: dict) -> dict:

118 "model_type": ("model_type", "transformer"),

119 "hidden_act": ("activation", "silu"),

120 "tie_word_embeddings": ("tied_embeddings", False),

121 - "max_seq_len": ("max_seq_len", 128_000),

121 + "max_seq_len": ("max_seq_len", config.get("max_position_embeddings", 128_000)),

122 "max_position_embeddings": ("max_position_embeddings", 128_000),

123 }

124

vllm/transformers_utils/tokenizer.py

+19 -11

@@ -3,8 +3,8 @@

3

4 import contextlib

5 import copy

6 +import importlib.util

7 import os

7 -import warnings

8 from functools import lru_cache

9 from pathlib import Path

10 from typing import TYPE_CHECKING, Any, TypeAlias

@@ -15,7 +15,10 @@ from typing_extensions import assert_never

15

16 from vllm import envs

17 from vllm.logger import init_logger

18 -from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config

18 +from vllm.transformers_utils.config import (

19 + get_sentence_transformer_tokenizer_config,

20 + list_filtered_repo_files,

21 +)

22 from vllm.transformers_utils.tokenizers import MistralTokenizer

23 from vllm.transformers_utils.utils import check_gguf_file

24

@@ -182,25 +185,29 @@ def get_tokenizer(

185 kwargs["gguf_file"] = Path(tokenizer_name).name

186 tokenizer_name = Path(tokenizer_name).parent

187

185 - # if tokenizer is from official mistral org

186 - is_from_mistral_org = str(tokenizer_name).split("/")[0] == "mistralai"

187 - if is_from_mistral_org and tokenizer_mode != "mistral":

188 - warnings.warn(

189 - "It is strongly recommended to run mistral models with "

190 - '`--tokenizer-mode "mistral"` to ensure correct '

191 - "encoding and decoding.",

192 - FutureWarning,

193 - stacklevel=2,

188 + # if `tokenizer_mode` == "auto", check if tokenizer can be loaded via Mistral format

189 + # first to use official Mistral tokenizer if possible.

190 + mistral_common_installed = importlib.util.find_spec("mistral_common") is not None

191 + if tokenizer_mode == "auto" and mistral_common_installed:

192 + allow_patterns = ["tekken.json", "tokenizer.model.v*"]

193 + files_list = list_filtered_repo_files(

194 + model_name_or_path=str(tokenizer_name),

195 + allow_patterns=allow_patterns,

196 + revision=revision,

197 )

198 + if len(files_list) > 0:

199 + tokenizer_mode = "mistral"

200

201 tokenizer: AnyTokenizer

202 if tokenizer_mode == "mistral":

203 + logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}")

204 tokenizer = MistralTokenizer.from_pretrained(

205 str(tokenizer_name), revision=revision

206 )

207 elif tokenizer_mode == "custom":

208 from vllm.transformers_utils.tokenizer_base import TokenizerRegistry

209

210 + logger.debug_once(f"Loading CustomTokenizer from {tokenizer_name}")

211 tokenizer = TokenizerRegistry.get_tokenizer(

212 str(tokenizer_name),

213 *args,

@@ -210,6 +217,7 @@ def get_tokenizer(

217 )

218 else:

219 try:

220 + logger.debug_once(f"Loading AutoTokenizer from {tokenizer_name}")

221 tokenizer = AutoTokenizer.from_pretrained(

222 tokenizer_name,

223 *args,

vllm/v1/engine/processor.py

+22 -3

@@ -20,6 +20,7 @@ from vllm.multimodal.utils import argsort_mm_positions

20 from vllm.pooling_params import PoolingParams

21 from vllm.sampling_params import SamplingParams

22 from vllm.transformers_utils.tokenizer import AnyTokenizer

23 +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

24 from vllm.utils import length_from_prompt_token_ids_or_embeds

25 from vllm.v1.engine import EngineCoreRequest

26 from vllm.v1.metrics.stats import MultiModalCacheStats

@@ -300,12 +301,24 @@ class Processor:

301 # allows <|special_token|> and similar, see

302 # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens

303 # Without tokenizer these are disallowed in grammars.

304 + if isinstance(self.tokenizer, MistralTokenizer):

305 + raise ValueError(

306 + "Mistral tokenizer is not supported for the 'guidance' "

307 + "structured output backend. Please use ['xgrammar', 'outlines'] "

308 + "backends or tokenizer_mode='hf' instead."

309 + )

310 validate_guidance_grammar(params, tokenizer=None)

311 elif backend == "outlines":

312 # outlines backend

313 validate_structured_output_request_outlines(params)

314 elif backend == "lm-format-enforcer":

315 # lm format enforcer backend

316 + if isinstance(self.tokenizer, MistralTokenizer):

317 + raise ValueError(

318 + "Mistral tokenizer is not supported for the 'lm-format-enforcer' "

319 + "structured output backend. Please use ['xgrammar', 'outlines'] "

320 + "backends or tokenizer_mode='hf' instead."

321 + )

322 validate_structured_output_request_lm_format_enforcer(params)

323 else:

324 # NOTE: backend must be "auto" here, because we have

@@ -320,9 +333,15 @@ class Processor:

333 except ValueError:

334 # The request either failed validation

335 # or includes some jsonschema feature(s) that

323 - # are not supported in xgrammar. Fall back to guidance.

324 - validate_guidance_grammar(params, tokenizer=None)

325 - params.structured_outputs._backend = "guidance"

336 + # are not supported in xgrammar.

337 + if isinstance(self.tokenizer, MistralTokenizer):

338 + # Fall back to outlines if the tokenizer is Mistral

339 + validate_structured_output_request_outlines(params)

340 + params.structured_outputs._backend = "outlines"

341 + else:

342 + # Fall back to guidance by default.

343 + validate_guidance_grammar(params, tokenizer=None)

344 + params.structured_outputs._backend = "guidance"

345 # Remember that this backend was set automatically

346 params.structured_outputs._backend_was_auto = True

347

Default model load/config/tokenizer to `mistral` format if relevant files exist (#28659)