[CI] Reuse loaded config for cached tokenizer (#49509)
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Andreas Karatzas committed
Jul 24, 2026 at 14:32 UTC
9863102ed94a91255cd6b0924027b3d3b985e913
2 files changed
+9
tests/tokenizers_/test_registry.py
+2
@@ -109,6 +109,8 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
109
try:
110
111
def fake_from_pretrained(path_or_repo_id: str, *args, **kwargs):
112
+ passed_config = kwargs.pop("config")
113
+ assert isinstance(passed_config, Qwen3_5MoeConfig)
114
loaded_config = AutoConfig.from_pretrained(
115
path_or_repo_id,
116
trust_remote_code=False,
vllm/tokenizers/registry.py
+7
@@ -18,6 +18,7 @@ from vllm.transformers_utils.repo_utils import (
18
)
19
from vllm.utils.import_utils import resolve_obj_by_qualname
20
21
+from .hf import CachedHfTokenizer
22
from .protocol import TokenizerLike
23
24
if TYPE_CHECKING:
@@ -233,6 +234,12 @@ def get_tokenizer(
234
else:
235
tokenizer_cls_ = tokenizer_cls
236
237
+ if config is not None and tokenizer_cls_ is CachedHfTokenizer:
238
+ # AutoTokenizer otherwise reloads config.json internally. Reuse the
239
+ # config that get_config just loaded successfully so a concurrent Hub
240
+ # cache refresh cannot invalidate the file between the two reads.
241
+ kwargs.setdefault("config", config)
242
+
243
tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
244
if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
245
from vllm.tokenizers.hf import get_cached_tokenizer