@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40a046cd82

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 22:30 UTC

[Bugfix]: Fix `TokenizerLike` interface (#30009)

Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>

Rohan Potdar committed Dec 5, 2025 at 22:56 UTC 40a046cd82af87e65d3be9db8bd27a4be65f1b00

8 files changed +78 -52

vllm/benchmarks/datasets.py

+32 -27

@@ -32,7 +32,6 @@ from typing import Any, cast

33 import numpy as np

34 from PIL import Image

35 -from transformers import PreTrainedTokenizerBase

35 from typing_extensions import deprecated

37 from vllm.lora.request import LoRARequest

@@ -189,7 +188,7 @@ class BenchmarkDataset(ABC):

188 @abstractmethod

189 def sample(

190 self,

192 - tokenizer: PreTrainedTokenizerBase,

191 + tokenizer: TokenizerLike,

192 num_requests: int,

193 request_id_prefix: str = "",

194 no_oversample: bool = False,

@@ -201,7 +200,7 @@ class BenchmarkDataset(ABC):

200 for generating a list of SampleRequest objects.

201

202 Args:

204 - tokenizer (PreTrainedTokenizerBase): The tokenizer to be used

203 + tokenizer (TokenizerLike): The tokenizer to be used

204 for processing the dataset's text.

205 num_requests (int): The number of sample requests to generate.

206 request_id_prefix (str): The prefix of request_id.

@@ -380,7 +379,7 @@ def process_video(video: Any) -> Mapping[str, Any]:

379

380

381 def gen_prompt_decode_to_target_len(

383 - tokenizer: PreTrainedTokenizerBase,

382 + tokenizer: TokenizerLike,

383 token_sequence: list[int],

384 target_token_len: int,

385 max_retry: int = 10,

@@ -468,7 +467,7 @@ class RandomDataset(BenchmarkDataset):

467

468 def sample(

469 self,

471 - tokenizer: PreTrainedTokenizerBase,

470 + tokenizer: TokenizerLike,

471 num_requests: int,

472 request_id_prefix: str = "",

473 no_oversample: bool = False,

@@ -580,7 +579,7 @@ class RandomDataset(BenchmarkDataset):

579 range_ratio: float,

580 input_len: int,

581 output_len: int,

583 - tokenizer: PreTrainedTokenizerBase,

582 + tokenizer: TokenizerLike,

583 ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:

584 """

585 Get the sampling parameters for the dataset.

@@ -626,7 +625,7 @@ class RandomDataset(BenchmarkDataset):

625 def generate_token_sequence(

626 self,

627 *,

629 - tokenizer: PreTrainedTokenizerBase,

628 + tokenizer: TokenizerLike,

629 prefix_token_ids: list[int],

630 prefix_len: int,

631 vocab_size: int,

@@ -686,7 +685,7 @@ class RandomDatasetForReranking(RandomDataset):

685

686 def sample(

687 self,

689 - tokenizer: PreTrainedTokenizerBase,

688 + tokenizer: TokenizerLike,

689 num_requests: int,

690 request_id_prefix: str = "",

691 range_ratio: float = RandomDataset.DEFAULT_RANGE_RATIO,

@@ -716,7 +715,11 @@ class RandomDatasetForReranking(RandomDataset):

715 doc_lens, _, doc_offsets = self.get_sampling_params(

716 num_requests, range_ratio, doc_len_param, 0, tokenizer

717 )

718 +

719 vocab_size = tokenizer.vocab_size

720 + prohibited_tokens = tokenizer.all_special_ids

721 + all_tokens = np.arange(vocab_size)

722 + allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))

723

724 query_prompt, query_input_len, token_mismatch_total = (

725 self.generate_token_sequence(

@@ -727,6 +730,7 @@ class RandomDatasetForReranking(RandomDataset):

730 input_len=query_len,

731 offset=int(query_offsets[0]),

732 index=0,

733 + allowed_tokens=allowed_tokens,

734 )

735 )

736

@@ -740,6 +744,7 @@ class RandomDatasetForReranking(RandomDataset):

744 input_len=int(doc_lens[i]),

745 offset=int(doc_offsets[i]),

746 index=i + 1,

747 + allowed_tokens=allowed_tokens,

748 )

749 token_mismatch_total += token_mismatch

750 requests.append((prompt, total_input_len))

@@ -1077,7 +1082,7 @@ class RandomMultiModalDataset(RandomDataset):

1082

1083 def sample(

1084 self,

1080 - tokenizer: PreTrainedTokenizerBase,

1085 + tokenizer: TokenizerLike,

1086 num_requests: int,

1087 request_id_prefix: str = "",

1088 no_oversample: bool = False,

@@ -1231,7 +1236,7 @@ class ShareGPTDataset(BenchmarkDataset):

1236

1237 def sample(

1238 self,

1234 - tokenizer: PreTrainedTokenizerBase,

1239 + tokenizer: TokenizerLike,

1240 num_requests: int,

1241 lora_path: str | None = None,

1242 max_loras: int | None = None,

@@ -1633,7 +1638,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1638 )

1639

1640

1636 -def get_samples(args, tokenizer) -> list[SampleRequest]:

1641 +def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:

1642 if not hasattr(args, "request_id_prefix"):

1643 args.request_id_prefix = ""

1644

@@ -1971,7 +1976,7 @@ class CustomDataset(BenchmarkDataset):

1976

1977 def sample(

1978 self,

1974 - tokenizer: PreTrainedTokenizerBase,

1979 + tokenizer: TokenizerLike,

1980 num_requests: int,

1981 lora_path: str | None = None,

1982 max_loras: int | None = None,

@@ -2101,7 +2106,7 @@ class SonnetDataset(BenchmarkDataset):

2106

2107 def sample(

2108 self,

2104 - tokenizer,

2109 + tokenizer: TokenizerLike,

2110 num_requests: int,

2111 prefix_len: int = DEFAULT_PREFIX_LEN,

2112 input_len: int = DEFAULT_INPUT_LEN,

@@ -2202,7 +2207,7 @@ class BurstGPTDataset(BenchmarkDataset):

2207

2208 def sample(

2209 self,

2205 - tokenizer: PreTrainedTokenizerBase,

2210 + tokenizer: TokenizerLike,

2211 num_requests: int,

2212 max_loras: int | None = None,

2213 lora_path: str | None = None,

@@ -2287,7 +2292,7 @@ class ConversationDataset(HuggingFaceDataset):

2292

2293 def sample(

2294 self,

2290 - tokenizer: PreTrainedTokenizerBase,

2295 + tokenizer: TokenizerLike,

2296 num_requests: int,

2297 output_len: int | None = None,

2298 enable_multimodal_chat: bool = False,

@@ -2347,7 +2352,7 @@ class MultiModalConversationDataset(HuggingFaceDataset):

2352

2353 def sample(

2354 self,

2350 - tokenizer: PreTrainedTokenizerBase,

2355 + tokenizer: TokenizerLike,

2356 num_requests: int,

2357 output_len: int | None = None,

2358 enable_multimodal_chat: bool = False,

@@ -2416,7 +2421,7 @@ class VisionArenaDataset(HuggingFaceDataset):

2421

2422 def sample(

2423 self,

2419 - tokenizer: PreTrainedTokenizerBase,

2424 + tokenizer: TokenizerLike,

2425 num_requests: int,

2426 output_len: int | None = None,

2427 enable_multimodal_chat: bool = False,

@@ -2470,7 +2475,7 @@ class MMVUDataset(HuggingFaceDataset):

2475

2476 def sample(

2477 self,

2473 - tokenizer: PreTrainedTokenizerBase,

2478 + tokenizer: TokenizerLike,

2479 num_requests: int,

2480 output_len: int | None = None,

2481 enable_multimodal_chat: bool = False,

@@ -2531,7 +2536,7 @@ class InstructCoderDataset(HuggingFaceDataset):

2536

2537 def sample(

2538 self,

2534 - tokenizer: PreTrainedTokenizerBase,

2539 + tokenizer: TokenizerLike,

2540 num_requests: int,

2541 output_len: int | None = None,

2542 enable_multimodal_chat: bool = False,

@@ -2595,7 +2600,7 @@ class MTBenchDataset(HuggingFaceDataset):

2600

2601 def sample(

2602 self,

2598 - tokenizer: PreTrainedTokenizerBase,

2603 + tokenizer: TokenizerLike,

2604 num_requests: int,

2605 output_len: int | None = None,

2606 enable_multimodal_chat: bool = False,

@@ -2661,7 +2666,7 @@ class BlazeditDataset(HuggingFaceDataset):

2666

2667 def sample(

2668 self,

2664 - tokenizer: PreTrainedTokenizerBase,

2669 + tokenizer: TokenizerLike,

2670 num_requests: int,

2671 output_len: int | None = None,

2672 skip_chat_template: bool = False,

@@ -2742,7 +2747,7 @@ class AIMODataset(HuggingFaceDataset):

2747

2748 def sample(

2749 self,

2745 - tokenizer: PreTrainedTokenizerBase,

2750 + tokenizer: TokenizerLike,

2751 num_requests: int,

2752 output_len: int | None = None,

2753 request_id_prefix: str = "",

@@ -2852,7 +2857,7 @@ class NextEditPredictionDataset(HuggingFaceDataset):

2857

2858 def sample(

2859 self,

2855 - tokenizer: PreTrainedTokenizerBase,

2860 + tokenizer: TokenizerLike,

2861 num_requests: int,

2862 request_id_prefix: str = "",

2863 no_oversample: bool = False,

@@ -2924,7 +2929,7 @@ class ASRDataset(HuggingFaceDataset):

2929

2930 def sample(

2931 self,

2927 - tokenizer: PreTrainedTokenizerBase,

2932 + tokenizer: TokenizerLike,

2933 num_requests: int,

2934 output_len: int | None = None,

2935 request_id_prefix: str = "",

@@ -3002,7 +3007,7 @@ class MLPerfDataset(HuggingFaceDataset):

3007

3008 def sample(

3009 self,

3005 - tokenizer: PreTrainedTokenizerBase,

3010 + tokenizer: TokenizerLike,

3011 num_requests: int,

3012 output_len: int | None = None,

3013 request_id_prefix: str = "",

@@ -3081,7 +3086,7 @@ class PrefixRepetitionRandomDataset(BenchmarkDataset):

3086

3087 def sample(

3088 self,

3084 - tokenizer: PreTrainedTokenizerBase,

3089 + tokenizer: TokenizerLike,

3090 num_requests: int,

3091 prefix_len: int = DEFAULT_PREFIX_LEN,

3092 suffix_len: int = DEFAULT_SUFFIX_LEN,

@@ -3167,7 +3172,7 @@ class MMStarDataset(HuggingFaceDataset):

3172

3173 def sample(

3174 self,

3170 - tokenizer: PreTrainedTokenizerBase,

3175 + tokenizer: TokenizerLike,

3176 num_requests: int,

3177 output_len: int | None = None,

3178 enable_multimodal_chat: bool = False,

vllm/benchmarks/serve.py

+16 -16

@@ -36,7 +36,6 @@ from typing import Any, Literal

36 import aiohttp

37 import numpy as np

38 from tqdm.asyncio import tqdm

39 -from transformers import PreTrainedTokenizerBase

40 from vllm.benchmarks.datasets import SampleRequest, add_dataset_parser, get_samples

41 from vllm.benchmarks.lib.endpoint_request_func import (

@@ -47,7 +46,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (

46 )

47 from vllm.benchmarks.lib.ready_checker import wait_for_endpoint

48 from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json

50 -from vllm.tokenizers import get_tokenizer

49 +from vllm.tokenizers import TokenizerLike, get_tokenizer

50 from vllm.utils.gc_utils import freeze_gc_heap

51 from vllm.utils.network_utils import join_host_port

@@ -286,7 +285,7 @@ def calculate_metrics(

285 input_requests: list[SampleRequest],

286 outputs: list[RequestFuncOutput],

287 dur_s: float,

289 - tokenizer: PreTrainedTokenizerBase,

288 + tokenizer: TokenizerLike,

289 selected_percentiles: list[float],

290 goodput_config_dict: dict[str, float],

291 ) -> tuple[BenchmarkMetrics, list[int]]:

@@ -489,7 +488,7 @@ async def benchmark(

488 base_url: str,

489 model_id: str,

490 model_name: str,

492 - tokenizer: PreTrainedTokenizerBase,

491 + tokenizer: TokenizerLike,

492 input_requests: list[SampleRequest],

493 logprobs: int | None,

494 request_rate: float,

@@ -1032,6 +1031,19 @@ def add_cli_args(parser: argparse.ArgumentParser):

1031 type=str,

1032 help="Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501

1033 )

1034 + parser.add_argument(

1035 + "--tokenizer-mode",

1036 + type=str,

1037 + default="auto",

1038 + help="""Tokenizer mode:\n

1039 + - "auto" will use the tokenizer from `mistral_common` for Mistral models

1040 + if available, otherwise it will use the "hf" tokenizer.\n

1041 + - "hf" will use the fast tokenizer if available.\n

1042 + - "slow" will always use the slow tokenizer.\n

1043 + - "mistral" will always use the tokenizer from `mistral_common`.\n

1044 + - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n

1045 + - Other custom values can be supported via plugins.""",

1046 + )

1047 parser.add_argument("--use-beam-search", action="store_true")

1048 parser.add_argument(

1049 "--logprobs",

@@ -1228,18 +1240,6 @@ def add_cli_args(parser: argparse.ArgumentParser):

1240 help="Common prefix length shared by all prompts (used by random dataset)",

1241 )

1242

1231 - parser.add_argument(

1232 - "--tokenizer-mode",

1233 - type=str,

1234 - default="auto",

1235 - choices=["auto", "slow", "mistral", "custom"],

1236 - help='The tokenizer mode.\n\n* "auto" will use the '

1237 - 'fast tokenizer if available.\n* "slow" will '

1238 - "always use the slow tokenizer. \n* "

1239 - '"mistral" will always use the `mistral_common` tokenizer. \n*'

1240 - '"custom" will use --tokenizer to select the preregistered tokenizer.',

1241 - )

1242 -

1243 parser.add_argument(

1244 "--served-model-name",

1245 type=str,

vllm/benchmarks/throughput.py

+16 -6

@@ -14,7 +14,7 @@ from typing import Any

14 import torch

15 import uvloop

16 from tqdm import tqdm

17 -from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizerBase

17 +from transformers import AutoModelForCausalLM, PreTrainedTokenizerBase

19 from vllm.benchmarks.datasets import (

20 AIMODataset,

@@ -35,6 +35,7 @@ from vllm.inputs import TextPrompt, TokensPrompt

35 from vllm.lora.request import LoRARequest

36 from vllm.outputs import RequestOutput

37 from vllm.sampling_params import BeamSearchParams

38 +from vllm.tokenizers import TokenizerLike, get_tokenizer

39 from vllm.utils.async_utils import merge_async_iterators

@@ -246,12 +247,15 @@ async def run_vllm_async(

247 def run_hf(

248 requests: list[SampleRequest],

249 model: str,

249 - tokenizer: PreTrainedTokenizerBase,

250 + tokenizer: TokenizerLike,

251 n: int,

252 max_batch_size: int,

253 trust_remote_code: bool,

254 disable_detokenize: bool = False,

255 ) -> float:

256 + assert isinstance(tokenizer, PreTrainedTokenizerBase), (

257 + "the hf backend only supports HF tokenizers"

258 + )

259 llm = AutoModelForCausalLM.from_pretrained(

260 model, dtype=torch.float16, trust_remote_code=trust_remote_code

261 )

@@ -692,15 +696,21 @@ def add_cli_args(parser: argparse.ArgumentParser):

696

697

698 def main(args: argparse.Namespace):

695 - if args.tokenizer is None:

696 - args.tokenizer = args.model

699 validate_args(args)

700 if args.seed is None:

701 args.seed = 0

702 random.seed(args.seed)

703 # Sample the requests.

702 - tokenizer = AutoTokenizer.from_pretrained(

703 - args.tokenizer, trust_remote_code=args.trust_remote_code

704 + if (

705 + args.backend == "hf" or args.backend == "mii"

706 + ) and args.tokenizer_mode == "auto":

707 + # mistral_common tokenizer is only supported on vllm and vllm-chat backends;

708 + # for hf and mii backends, we use hf tokenizer

709 + args.tokenizer_mode = "hf"

710 + tokenizer = get_tokenizer(

711 + args.tokenizer,

712 + tokenizer_mode=args.tokenizer_mode,

713 + trust_remote_code=args.trust_remote_code,

714 )

715 requests = get_requests(args, tokenizer)

716 is_multi_modal = any(request.multi_modal_data is not None for request in requests)

vllm/config/model.py

+2 -1

@@ -136,7 +136,8 @@ class ModelConfig:

136 name or path will be used."""

137 tokenizer_mode: TokenizerMode | str = "auto"

138 """Tokenizer mode:\n

139 - - "auto" will use "hf" tokenizer if Mistral's tokenizer is not available.\n

139 + - "auto" will use the tokenizer from `mistral_common` for Mistral models

140 + if available, otherwise it will use the "hf" tokenizer.\n

141 - "hf" will use the fast tokenizer if available.\n

142 - "slow" will always use the slow tokenizer.\n

143 - "mistral" will always use the tokenizer from `mistral_common`.\n

vllm/tokenizers/deepseekv32.py

@@ -54,6 +54,9 @@ class DeepseekV32Tokenizer(HfTokenizer):

54 prompt_str = encode_messages(messages, **encode_config) # type: ignore

55 return prompt_str

57 + def num_special_tokens_to_add(self) -> int:

58 + return len(self.encode(""))

59 +

60 @property

61 def all_special_tokens(self) -> list[str]:

62 return self.tokenizer.all_special_tokens

vllm/tokenizers/mistral.py

+5 -1

@@ -309,6 +309,9 @@ class MistralTokenizer(TokenizerLike):

309 for i in all_special_ids

310 ]

311

312 + def num_special_tokens_to_add(self) -> int:

313 + return len(self.encode(""))

314 +

315 # the following attributes are set to fit vLLM's design and are used

316 # by the structured output backends.

317 @property

@@ -421,6 +424,7 @@ class MistralTokenizer(TokenizerLike):

424 ) -> list[int]:

425 add_generation_prompt = kwargs.pop("add_generation_prompt", False)

426 continue_final_message = kwargs.get("continue_final_message", False)

427 + tokenize = kwargs.get("tokenize", True)

428 padding = kwargs.get("padding", False)

429 truncation = kwargs.get("truncation", False)

430 max_length = kwargs.get("max_length")

@@ -433,7 +437,7 @@ class MistralTokenizer(TokenizerLike):

437 conversation=messages,

438 tools=tools,

439 continue_final_message=continue_final_message,

436 - tokenize=True,

440 + tokenize=tokenize,

441 padding=padding,

442 truncation=truncation,

443 max_length=max_length,

vllm/tokenizers/protocol.py

@@ -22,6 +22,9 @@ class TokenizerLike(Protocol):

22 ) -> "TokenizerLike":

23 raise NotImplementedError

25 + def num_special_tokens_to_add(self) -> int:

26 + raise NotImplementedError

27 +

28 @property

29 def all_special_tokens(self) -> list[str]:

30 raise NotImplementedError

vllm/tokenizers/registry.py

+1 -1

@@ -183,7 +183,7 @@ def get_tokenizer(

183 "`tokenizer_mode='custom'` when initializing vLLM.",

184 tokenizer_args,

185 str(tokenizer_kwargs),

186 - tokenizer_mode,

186 + tokenizer_name,

187 )

188

189 tokenizer_mode = str(tokenizer_name)