A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 23:30 UTC
[torch.compile] caching of config fields should be opt-out by default (#26468)
Signed-off-by: vnadathur <glvikramn@gmail.com> Signed-off-by: WorldExplored <srreyansh.sethi@gmail.com> Signed-off-by: Srreyansh Sethi <srreyansh.sethi@gmail.com> Signed-off-by: Srreyansh Sethi <107075589+WorldExplored@users.noreply.github.com> Co-authored-by: WorldExplored <srreyansh.sethi@gmail.com> Co-authored-by: Srreyansh Sethi <107075589+worldexplored@users.noreply.github.com> Co-authored-by: vnadathur <236933696+vnadathur@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
vnadathur committed
Nov 19, 2025 at 06:13 UTC
1ffe934c8ae978e5ed82559a1eaeca05e37f9b35
11 files changed
+602
-193
tests/config/test_config_utils.py
new
+166
@@ -0,0 +1,166 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from dataclasses import dataclass
5
+from enum import Enum
6
+
7
+import pytest
8
+
9
+from vllm.config.utils import get_hash_factors, hash_factors, normalize_value
10
+
11
+# Helpers
12
+
13
+
14
+def endswith_fqname(obj, suffix: str) -> bool:
15
+ # normalize_value(type) returns fully-qualified name
16
+ # Compare suffix to avoid brittle import paths.
17
+ out = normalize_value(obj)
18
+ return isinstance(out, str) and out.endswith(suffix)
19
+
20
+
21
+def expected_path(p_str: str = ".") -> str:
22
+ import pathlib
23
+
24
+ p = pathlib.Path(p_str)
25
+ return p.expanduser().resolve().as_posix()
26
+
27
+
28
+# Minimal dataclass to test get_hash_factors.
29
+# Avoid importing heavy vLLM configs.
30
+@dataclass
31
+class SimpleConfig:
32
+ a: object
33
+ b: object | None = None
34
+
35
+
36
+class DummyLogprobsMode(Enum):
37
+ RAW_LOGITS = "raw_logits"
38
+
39
+
40
+def test_hash_factors_deterministic():
41
+ """Test that hash_factors produces consistent SHA-256 hashes"""
42
+ factors = {"a": 1, "b": "test"}
43
+ hash1 = hash_factors(factors)
44
+ hash2 = hash_factors(factors)
45
+
46
+ assert hash1 == hash2
47
+ # Dict key insertion order should not affect the hash.
48
+ factors_reordered = {"b": "test", "a": 1}
49
+ assert hash_factors(factors_reordered) == hash1
50
+ assert len(hash1) == 64
51
+ assert all(c in "0123456789abcdef" for c in hash1)
52
+
53
+
54
+@pytest.mark.parametrize(
55
+ "inp, expected",
56
+ [
57
+ (None, None),
58
+ (True, True),
59
+ (1, 1),
60
+ (1.0, 1.0),
61
+ ("x", "x"),
62
+ (b"ab", "6162"),
63
+ (bytearray(b"ab"), "6162"),
64
+ ([1, 2], (1, 2)),
65
+ ({"b": 2, "a": 1}, (("a", 1), ("b", 2))),
66
+ ],
67
+)
68
+def test_normalize_value_matrix(inp, expected):
69
+ """Parametric input→expected normalization table."""
70
+ assert normalize_value(inp) == expected
71
+
72
+
73
+def test_normalize_value_enum():
74
+ # Enums normalize to (module.QualName, value).
75
+ # DummyLogprobsMode uses a string payload.
76
+ out = normalize_value(DummyLogprobsMode.RAW_LOGITS)
77
+ assert isinstance(out, tuple)
78
+ assert out[0].endswith("DummyLogprobsMode")
79
+ # Expect string payload 'raw_logits'.
80
+ assert out[1] == "raw_logits"
81
+
82
+
83
+def test_normalize_value_set_order_insensitive():
84
+ # Sets are unordered; normalize_value sorts elements for determinism.
85
+ assert normalize_value({3, 1, 2}) == normalize_value({1, 2, 3})
86
+
87
+
88
+def test_normalize_value_path_normalization():
89
+ from pathlib import Path # local import to avoid global dependency
90
+
91
+ # Paths expand/resolve to absolute strings.
92
+ # Stabilizes hashing across working dirs.
93
+ assert normalize_value(Path(".")) == expected_path(".")
94
+
95
+
96
+def test_normalize_value_uuid_and_to_json():
97
+ # Objects may normalize via uuid() or to_json_string().
98
+ class HasUUID:
99
+ def uuid(self):
100
+ return "test-uuid"
101
+
102
+ class ToJson:
103
+ def to_json_string(self):
104
+ return '{"x":1}'
105
+
106
+ assert normalize_value(HasUUID()) == "test-uuid"
107
+ assert normalize_value(ToJson()) == '{"x":1}'
108
+
109
+
110
+@pytest.mark.parametrize(
111
+ "bad",
112
+ [
113
+ (lambda x: x),
114
+ (type("CallableInstance", (), {"__call__": lambda self: 0}))(),
115
+ (lambda: (lambda: 0))(), # nested function instance
116
+ ],
117
+)
118
+def test_error_cases(bad):
119
+ """Inputs expected to raise TypeError."""
120
+ # Reject functions/lambdas/callable instances
121
+ # to avoid under-hashing.
122
+ with pytest.raises(TypeError):
123
+ normalize_value(bad)
124
+
125
+
126
+def test_enum_vs_int_disambiguation():
127
+ # int stays primitive
128
+ nf_int = normalize_value(1)
129
+ assert nf_int == 1
130
+
131
+ # enum becomes ("module.QualName", value)
132
+ nf_enum = normalize_value(DummyLogprobsMode.RAW_LOGITS)
133
+ assert isinstance(nf_enum, tuple) and len(nf_enum) == 2
134
+ enum_type, enum_val = nf_enum
135
+ assert enum_type.endswith(".DummyLogprobsMode")
136
+ assert enum_val == "raw_logits"
137
+
138
+ # Build factor dicts from configs with int vs enum
139
+ f_int = get_hash_factors(SimpleConfig(1), set())
140
+ f_enum = get_hash_factors(SimpleConfig(DummyLogprobsMode.RAW_LOGITS), set())
141
+ # The int case remains a primitive value
142
+ assert f_int["a"] == 1
143
+ # The enum case becomes a tagged tuple ("module.QualName", "raw_logits")
144
+ assert isinstance(f_enum["a"], tuple) and f_enum["a"][1] == "raw_logits"
145
+ # Factor dicts must differ so we don't collide primitives with Enums.
146
+ assert f_int != f_enum
147
+ # Hash digests must differ correspondingly
148
+ assert hash_factors(f_int) != hash_factors(f_enum)
149
+
150
+ # Hash functions produce stable hex strings
151
+ h_int = hash_factors(f_int)
152
+ h_enum = hash_factors(f_enum)
153
+ assert isinstance(h_int, str) and len(h_int) == 64
154
+ assert isinstance(h_enum, str) and len(h_enum) == 64
155
+
156
+
157
+def test_classes_are_types():
158
+ """Types normalize to FQNs; include real vLLM types."""
159
+ # Only classes allowed; functions/lambdas are rejected.
160
+ # Canonical form is the fully-qualified name.
161
+ assert isinstance(normalize_value(str), str)
162
+
163
+ class LocalDummy:
164
+ pass
165
+
166
+ assert endswith_fqname(LocalDummy, ".LocalDummy")
vllm/compilation/backends.py
+83
-22
@@ -4,12 +4,14 @@
4
import ast
5
import dataclasses
6
import hashlib
7
+import json
8
import operator
9
import os
10
import pprint
11
import time
12
from collections.abc import Callable, Sequence
13
from contextlib import contextmanager
14
+from functools import partial
15
from typing import Any
16
17
import torch
@@ -23,7 +25,9 @@ from vllm.compilation.partition_rules import (
25
should_split,
26
)
27
from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig
28
+from vllm.config.utils import hash_factors
29
from vllm.logger import init_logger
30
+from vllm.logging_utils import lazy
31
from vllm.platforms import current_platform
32
from vllm.utils.import_utils import resolve_obj_by_qualname
33
from vllm.utils.torch_utils import is_torch_equal_or_newer
@@ -580,35 +584,47 @@ class VllmBackend:
584
def __call__(
585
self, graph: fx.GraphModule, example_inputs
586
) -> VllmSerializableFunction:
583
- from .caching import _compute_code_hash, compilation_config_hash_factors
584
-
587
vllm_config = self.vllm_config
588
+ # Minimal hashing here with existing utilities, reused below.
589
+
590
+ env_factors = envs.compile_factors()
591
+ env_hash = hash_factors(env_factors)
592
+ # Compute config/compiler/code hashes once and reuse
593
+ config_hash = vllm_config.compute_hash()
594
+ compiler_hash = self.compiler_manager.compute_hash(vllm_config)
595
+ forward_code_files = list(sorted(self.compilation_config.traced_files))
596
+
597
+ logger.debug(
598
+ "Traced files (to be considered for compilation cache):\n%s",
599
+ lazy(lambda: "\n".join(forward_code_files)),
600
+ )
601
+ hash_content = []
602
+ for filepath in forward_code_files:
603
+ hash_content.append(filepath)
604
+ if filepath == "<string>":
605
+ # This means the function was dynamically generated, with
606
+ # e.g. exec(). We can't actually check these.
607
+ continue
608
+ try:
609
+ with open(filepath) as f:
610
+ hash_content.append(f.read())
611
+ except Exception:
612
+ logger.warning("Failed to read file %s", filepath)
613
+ continue
614
+ code_hash = hashlib.sha256("\n".join(hash_content).encode()).hexdigest()
615
+ # Clear after consumption
616
+ self.compilation_config.traced_files.clear()
617
if not self.compilation_config.cache_dir:
618
# no provided cache dir, generate one based on the known factors
619
# that affects the compilation. if none of the factors change,
620
# the cache dir will be the same so that we can reuse the compiled
621
# graph.
591
-
592
- factors = compilation_config_hash_factors(vllm_config)
593
- # 2. factors come from the code files that are traced by Dynamo (
594
- # it mainly summarizes how the model is used in forward pass)
595
- code_hash = _compute_code_hash(self.compilation_config.traced_files)
596
- self.compilation_config.traced_files.clear()
597
- factors.append(code_hash)
598
-
599
- # 3. compiler hash
600
- compiler_hash = self.compiler_manager.compute_hash(vllm_config)
601
- factors.append(compiler_hash)
602
-
603
- # combine all factors to generate the cache dir
604
- hash_key = hashlib.md5(
605
- str(factors).encode(), usedforsecurity=False
606
- ).hexdigest()[:10]
607
-
622
+ factors = [env_hash, config_hash, code_hash, compiler_hash]
623
+ # Use SHA-256 for cache key hashing to be consistent across
624
+ # compute_hash functions. Truncate for a short cache dir name.
625
+ hash_key = hashlib.sha256(str(factors).encode()).hexdigest()[:10]
626
cache_dir = os.path.join(
609
- envs.VLLM_CACHE_ROOT,
610
- "torch_compile_cache",
611
- hash_key,
627
+ envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key
628
)
629
self.compilation_config.cache_dir = cache_dir
630
@@ -621,6 +637,7 @@ class VllmBackend:
637
os.makedirs(local_cache_dir, exist_ok=True)
638
self.compilation_config.local_cache_dir = local_cache_dir
639
640
+ # Honors opt-outs such as CompilationMode.NONE or VLLM_DISABLE_COMPILE_CACHE.
641
disable_cache = not is_compile_cache_enabled(
642
self.compilation_config.inductor_compile_config
643
)
@@ -638,6 +655,50 @@ class VllmBackend:
655
local_cache_dir, disable_cache, self.prefix
656
)
657
658
+ # Reuses existing cache key
659
+
660
+ logger.debug(
661
+ "torch.compile cache factors: env=%s cfg=%s comp=%s code=%s dir=%s",
662
+ env_hash,
663
+ config_hash,
664
+ compiler_hash,
665
+ code_hash,
666
+ local_cache_dir,
667
+ )
668
+
669
+ # Persist and log only hash-relevant factors together.
670
+ try:
671
+ logger.debug(
672
+ "Compile env factors (raw):\n%s\nVllm config hash: %s",
673
+ lazy(partial(pprint.pformat, env_factors, width=120)),
674
+ config_hash,
675
+ )
676
+ meta_path = os.path.join(local_cache_dir, "cache_key_factors.json")
677
+ if not os.path.exists(meta_path):
678
+ with open(meta_path, "w") as f:
679
+ json.dump(
680
+ {
681
+ "env": env_factors, # raw factors used for env_hash
682
+ "config_hash": config_hash,
683
+ "code_hash": code_hash,
684
+ "compiler_hash": compiler_hash,
685
+ },
686
+ f,
687
+ indent=2,
688
+ sort_keys=True,
689
+ )
690
+ except Exception:
691
+ # Best-effort only; metadata write failures are non-fatal.
692
+ logger.warning(
693
+ (
694
+ "Could not write compile cache metadata at %s; continuing without "
695
+ "metadata. Compiled cache remains valid; diagnostics may be "
696
+ "limited."
697
+ ),
698
+ local_cache_dir,
699
+ exc_info=True,
700
+ )
701
+
702
# when dynamo calls the backend, it means the bytecode
703
# transform and analysis are done
704
compilation_counter.num_graphs_seen += 1
vllm/compilation/pass_manager.py
+1
-1
@@ -127,7 +127,7 @@ class PostGradPassManager(CustomGraphPass):
127
affects compilation caching. Its uuid depends on the UUIDs of all
128
dependent passes and the pass config. See InductorPass for more info.
129
"""
130
- state = {"pass_config": self.pass_config.uuid(), "passes": []}
130
+ state = {"pass_config": self.pass_config.compute_hash(), "passes": []}
131
for pass_ in self.passes:
132
state["passes"].append(pass_.uuid())
133
state["passes"].append(self.fix_functionalization.uuid())
vllm/config/cache.py
+23
-8
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import hashlib
4
from dataclasses import field
5
from typing import TYPE_CHECKING, Any, Literal
6
@@ -160,13 +159,29 @@ class CacheConfig:
159
excluding anything before input ids/embeddings and after
160
the final hidden states.
161
"""
163
- factors: list[Any] = []
164
- factors.append(self.cache_dtype)
165
- factors.append(self.mamba_cache_dtype)
166
- factors.append(self.mamba_ssm_cache_dtype)
167
- # `cpu_offload_gb` does not use `torch.compile` yet.
168
- hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
169
- return hash_str
162
+ ignored_factors = {
163
+ # Runtime/derived knobs that don't affect compiled graph shape
164
+ "gpu_memory_utilization",
165
+ "swap_space",
166
+ "is_attention_free",
167
+ "num_gpu_blocks_override",
168
+ "enable_prefix_caching",
169
+ "prefix_caching_hash_algo",
170
+ # `cpu_offload_gb` does not use `torch.compile` yet.
171
+ "cpu_offload_gb",
172
+ "cpu_kvcache_space_bytes",
173
+ "mamba_page_size_padded",
174
+ # Post-init/derived counters
175
+ "num_gpu_blocks",
176
+ "num_cpu_blocks",
177
+ # WIP feature toggle not impacting compiled graph shape
178
+ "kv_sharing_fast_prefill",
179
+ }
180
+
181
+ from vllm.config.utils import get_hash_factors, hash_factors
182
+
183
+ factors = get_hash_factors(self, ignored_factors)
184
+ return hash_factors(factors)
185
186
def metrics_info(self):
187
# convert cache_config to dict(key: str, value: str) for prometheus
vllm/config/compilation.py
+22
-18
@@ -2,7 +2,6 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import enum
5
-import hashlib
5
from collections import Counter
6
from collections.abc import Callable
7
from dataclasses import asdict, field
@@ -160,7 +159,7 @@ class PassConfig:
159
current_platform.get_device_capability().to_int(), {}
160
)
161
163
- def uuid(self):
162
+ def compute_hash(self) -> str:
163
"""
164
Produces a hash unique to the pass configuration.
165
Any new fields that affect compilation should be added to the hash.
@@ -506,28 +505,33 @@ class CompilationConfig:
505
506
def compute_hash(self) -> str:
507
"""
509
- WARNING: Whenever a new field is added to this config,
510
- ensure that it is included in the factors list if
511
- it affects the computation graph.
512
-
508
Provide a hash that uniquely identifies all the configs
509
that affect the structure of the computation
510
graph from input ids/embeddings to the final hidden states,
511
excluding anything before input ids/embeddings and after
512
the final hidden states.
513
"""
519
- factors: list[Any] = []
520
- factors.append(self.mode)
521
- factors.append(self.backend)
522
- factors.append(self.custom_ops)
523
- factors.append(self.splitting_ops)
524
- factors.append(self.use_inductor)
525
- factors.append(self.use_inductor_graph_partition)
526
- factors.append(self.inductor_compile_config)
527
- factors.append(self.inductor_passes)
528
- factors.append(self.pass_config.uuid())
529
- factors.append(self.compile_cache_save_format)
530
- return hashlib.sha256(str(factors).encode()).hexdigest()
514
+ # Opt-out: default-include declared fields; keep a tiny exclude set;
515
+ # normalize types; keep SHA-256. For nested opaque configs, include a
516
+ # stable identifier (e.g., pass_config.compute_hash()) instead of object id.
517
+
518
+ ignored_factors = {
519
+ # Paths/dirs and runtime/metrics that don’t affect compiled graph
520
+ "debug_dump_path",
521
+ "cache_dir",
522
+ "local_cache_dir",
523
+ "bs_to_padded_graph_size",
524
+ "traced_files",
525
+ "compilation_time",
526
+ "static_forward_context",
527
+ "pass_config", # handled separately below
528
+ }
529
+
530
+ from vllm.config.utils import get_hash_factors, hash_factors
531
+
532
+ factors = get_hash_factors(self, ignored_factors)
533
+ factors["pass_config"] = self.pass_config.compute_hash()
534
+ return hash_factors(factors)
535
536
def __repr__(self) -> str:
537
exclude = {
vllm/config/model.py
+45
-47
@@ -1,8 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import hashlib
5
-import json
4
import warnings
5
from collections.abc import Callable
6
from dataclasses import InitVar, field
@@ -18,7 +16,7 @@ import vllm.envs as envs
16
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MultiModalConfig
17
from vllm.config.pooler import PoolerConfig
18
from vllm.config.scheduler import RunnerType
21
-from vllm.config.utils import assert_hashable, config, getattr_iter
19
+from vllm.config.utils import config, getattr_iter
20
from vllm.logger import init_logger
21
from vllm.platforms import current_platform
22
from vllm.transformers_utils.config import (
@@ -324,50 +322,50 @@ class ModelConfig:
322
excluding anything before input ids/embeddings and after
323
the final hidden states.
324
"""
327
- factors: list[Any] = []
328
- factors.append(self.model)
329
- factors.append(self.dtype)
330
- factors.append(self.quantization)
331
- factors.append(self.revision)
332
- factors.append(self.code_revision)
333
- factors.append(self.max_model_len)
334
- factors.append(self.max_logprobs)
335
- factors.append(self.disable_sliding_window)
336
- factors.append(self.trust_remote_code)
337
- factors.append(self.generation_config)
338
- factors.append(self.model_impl)
339
- factors.append(self.override_generation_config)
340
- factors.append(self.video_pruning_rate)
341
- factors.append(self.enable_prompt_embeds)
342
-
343
- # hf_config can control how the model looks!
344
- try:
345
- hf_config_json = self.hf_config.to_json_string(use_diff=False)
346
- except TypeError:
347
- from transformers import PretrainedConfig
348
-
349
- from vllm.utils.jsontree import json_map_leaves
350
-
351
- # Handle nested HF configs with unserializable values gracefully
352
- hf_config_json = (
353
- json.dumps(
354
- json_map_leaves(
355
- lambda v: v.to_dict()
356
- if isinstance(v, PretrainedConfig)
357
- else str(v),
358
- self.hf_config.to_dict(),
359
- ),
360
- indent=2,
361
- sort_keys=True,
362
- )
363
- + "\n"
364
- )
365
-
366
- factors.append(hf_config_json)
367
-
368
- str_factors = str(factors)
369
- assert_hashable(str_factors)
370
- return hashlib.sha256(str(factors).encode()).hexdigest()
325
+ ignored_factors = {
326
+ "runner",
327
+ "convert",
328
+ "task",
329
+ "tokenizer",
330
+ "tokenizer_mode",
331
+ "seed",
332
+ "hf_config_path",
333
+ "allowed_local_media_path",
334
+ "allowed_media_domains",
335
+ "tokenizer_revision",
336
+ "spec_target_max_model_len",
337
+ "enforce_eager",
338
+ "logprobs_mode",
339
+ "disable_cascade_attn",
340
+ "skip_tokenizer_init",
341
+ "enable_prompt_embeds",
342
+ "served_model_name",
343
+ "config_format",
344
+ "hf_token",
345
+ "hf_overrides",
346
+ "logits_processor_pattern",
347
+ "enable_sleep_mode",
348
+ "override_attention_dtype",
349
+ "logits_processors",
350
+ "io_processor_plugin",
351
+ "pooler_config",
352
+ "override_pooler_config",
353
+ "multimodal_config",
354
+ "limit_mm_per_prompt",
355
+ "media_io_kwargs",
356
+ "mm_processor_kwargs",
357
+ "mm_processor_cache_gb",
358
+ "mm_processor_cache_type",
359
+ "mm_shm_cache_max_object_size_mb",
360
+ "mm_encoder_tp_mode",
361
+ "interleave_mm_strings",
362
+ "skip_mm_profiling",
363
+ }
364
+
365
+ from vllm.config.utils import get_hash_factors, hash_factors
366
+
367
+ factors = get_hash_factors(self, ignored_factors)
368
+ return hash_factors(factors)
369
370
def _update_nested(
371
self,
vllm/config/parallel.py
+35
-14
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import hashlib
4
import os
5
from typing import TYPE_CHECKING, Any, Literal
6
@@ -448,19 +447,41 @@ class ParallelConfig:
447
This hash is also used for DP worker configuration validation
448
to prevent hangs from mismatched collective communication patterns.
449
"""
451
- factors: list[Any] = []
452
- factors.append(self.pipeline_parallel_size)
453
- factors.append(self.tensor_parallel_size)
454
- factors.append(self.enable_expert_parallel)
455
- factors.append(self.data_parallel_size)
456
- factors.append(self.all2all_backend)
457
- factors.append(self.enable_eplb)
458
- if self.enable_eplb:
459
- factors.append(self.eplb_config.log_balancedness)
460
- factors.append(self.eplb_config.window_size)
461
- factors.append(self.eplb_config.step_interval)
462
- factors.append(self.eplb_config.num_redundant_experts)
463
- return hashlib.sha256(str(factors).encode()).hexdigest()
450
+ ignored_factors = {
451
+ # Derived/runtime topology, networking, or launch details
452
+ "data_parallel_rank",
453
+ "data_parallel_rank_local",
454
+ "data_parallel_backend",
455
+ "data_parallel_external_lb",
456
+ "data_parallel_hybrid_lb",
457
+ "data_parallel_master_ip",
458
+ "data_parallel_master_port",
459
+ "_data_parallel_master_port_list",
460
+ "data_parallel_rpc_port",
461
+ "rank",
462
+ "master_addr",
463
+ "master_port",
464
+ "node_rank",
465
+ "nnodes",
466
+ "max_parallel_loading_workers",
467
+ "disable_custom_all_reduce",
468
+ "ray_workers_use_nsight",
469
+ "ray_runtime_env",
470
+ "placement_group",
471
+ "distributed_executor_backend",
472
+ "worker_cls",
473
+ "sd_worker_cls",
474
+ "worker_extension_cls",
475
+ "_api_process_count",
476
+ "_api_process_rank",
477
+ }
478
+
479
+ from vllm.config.utils import get_hash_factors, hash_factors
480
+
481
+ factors = get_hash_factors(self, ignored_factors)
482
+ # Explicitly include backend affecting env factor as before
483
+ factors["VLLM_ALL2ALL_BACKEND"] = str(envs.VLLM_ALL2ALL_BACKEND)
484
+ return hash_factors(factors)
485
486
def __post_init__(self) -> None:
487
# Set all2all_backend from env var if not specified, with deprecation warning
vllm/config/utils.py
+118
-1
@@ -3,14 +3,19 @@
3
"""Utility functions for vLLM config dataclasses."""
4
5
import ast
6
+import enum
7
+import hashlib
8
import inspect
9
+import json
10
+import pathlib
11
import textwrap
8
-from collections.abc import Iterable
12
+from collections.abc import Iterable, Mapping, Sequence, Set
13
from dataclasses import MISSING, Field, field, fields, is_dataclass, replace
14
from itertools import pairwise
15
from typing import TYPE_CHECKING, Any, Protocol, TypeVar
16
17
import regex as re
18
+import torch
19
from pydantic.fields import FieldInfo
20
from typing_extensions import runtime_checkable
21
@@ -176,3 +181,115 @@ def update_config(config: ConfigT, overrides: dict[str, Any]) -> ConfigT:
181
)
182
processed_overrides[field_name] = value
183
return replace(config, **processed_overrides)
184
+
185
+
186
+def normalize_value(x):
187
+ """Return a stable, JSON-serializable canonical form for hashing.
188
+ Order: primitives, special types (Enum, callable, torch.dtype, Path), then
189
+ generic containers (Mapping/Set/Sequence) with recursion.
190
+ """
191
+ # Fast path
192
+ if x is None or isinstance(x, (bool, int, float, str)):
193
+ return x
194
+
195
+ # Enums: tag with FQN to avoid primitive collisions.
196
+ # Ex: Enum(1) vs int(1) -> ("module.QualName", value).
197
+ if isinstance(x, enum.Enum):
198
+ enum_type = f"{x.__class__.__module__}.{x.__class__.__qualname__}"
199
+ return (enum_type, normalize_value(x.value))
200
+
201
+ # Classes (types) are accepted and canonicalized by their fully-qualified
202
+ # name (module.qualname) for a stable identifier.
203
+ # Instances are only accepted if they expose uuid(); otherwise they are
204
+ # rejected to avoid under-hashing object state.
205
+
206
+ # Callables: accept classes only; reject funcs/lambdas/methods.
207
+ # Used by LogitsProcessor types and ModelConfig.hf_overrides.
208
+ if isinstance(x, type):
209
+ module = getattr(x, "__module__", "")
210
+ qual = getattr(x, "__qualname__", getattr(x, "__name__", ""))
211
+ return ".".join([p for p in (module, qual) if p]) or repr(x)
212
+
213
+ # Prefer stable uuid identifiers for objects that provide them, even if
214
+ # they are callable instances (e.g., InductorPass wrappers).
215
+ if hasattr(x, "uuid") and callable(getattr(x, "uuid", None)):
216
+ return x.uuid()
217
+
218
+ if callable(x):
219
+ raise TypeError("normalize_value: function or callable instance unsupported")
220
+
221
+ # Torch dtype: stringify (torch.float64 -> "torch.float64").
222
+ # We rely on the string form here; dtype-bearing fields that need additional
223
+ # disambiguation should encode that at the config layer.
224
+ if isinstance(x, torch.dtype):
225
+ return str(x)
226
+
227
+ # Bytes
228
+ if isinstance(x, (bytes, bytearray)):
229
+ return x.hex()
230
+
231
+ # Paths (canonicalize)
232
+ if isinstance(x, pathlib.Path):
233
+ try:
234
+ return str(x.expanduser().resolve())
235
+ except Exception:
236
+ return str(x)
237
+
238
+ # Dataclasses: represent as (FQN, sorted(field,value) tuple) for stability.
239
+ if is_dataclass(x):
240
+ type_fqn = f"{x.__class__.__module__}.{x.__class__.__qualname__}"
241
+ items = tuple(
242
+ (f.name, normalize_value(getattr(x, f.name)))
243
+ for f in sorted(fields(x), key=lambda f: f.name)
244
+ )
245
+ return (type_fqn, items)
246
+
247
+ # Containers (generic)
248
+ if isinstance(x, Mapping):
249
+ return tuple(sorted((str(k), normalize_value(v)) for k, v in x.items()))
250
+ if isinstance(x, Set):
251
+ return tuple(sorted(repr(normalize_value(v)) for v in x))
252
+ if isinstance(x, Sequence) and not isinstance(x, (str, bytes, bytearray)):
253
+ return tuple(normalize_value(v) for v in x)
254
+
255
+ # PretrainedConfig
256
+ if hasattr(x, "to_json_string") and callable(x.to_json_string):
257
+ return x.to_json_string()
258
+
259
+ # Unsupported type: e.g., modules, generators, open files, or objects
260
+ # without a stable JSON/UUID representation. Hard-error to avoid
261
+ # under-hashing.
262
+ # If you hit this, either reshape your config to use supported primitives
263
+ # and containers, or extend normalize_value to provide a stable encoding
264
+ # (e.g., via uuid() or to_json_string()) for this type.
265
+ raise TypeError(
266
+ f"normalize_value: unsupported type '{type(x).__name__}'. "
267
+ "Ensure config values use supported primitives/containers or add a "
268
+ "stable representation for this type."
269
+ )
270
+
271
+
272
+def get_hash_factors(config: ConfigT, ignored_factors: set[str]) -> dict[str, object]:
273
+ """Gets the factors used for hashing a config class.
274
+ - Includes all dataclass fields not in `ignored_factors`.
275
+ - Errors on non-normalizable values.
276
+ """
277
+ factors: dict[str, object] = {}
278
+ for dc_field in fields(config):
279
+ factor = dc_field.name
280
+ if factor in ignored_factors:
281
+ continue
282
+ value = getattr(config, factor, None)
283
+ try:
284
+ factors[factor] = normalize_value(value)
285
+ except TypeError as e:
286
+ raise TypeError(
287
+ f"get_hash_factors: unsupported type for key '{factor}' "
288
+ f"({type(value).__name__})"
289
+ ) from e
290
+ return factors
291
+
292
+
293
+def hash_factors(items: dict[str, object]) -> str:
294
+ """Return a SHA-256 hex digest of the canonical items structure."""
295
+ return hashlib.sha256(json.dumps(items, sort_keys=True).encode()).hexdigest()
vllm/envs.py
+87
-82
@@ -2,8 +2,8 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import functools
5
-import hashlib
5
import json
6
+import logging
7
import os
8
import sys
9
import tempfile
@@ -426,6 +426,8 @@ def get_vllm_port() -> int | None:
426
427
# --8<-- [start:env-vars-definition]
428
429
+logger = logging.getLogger(__name__)
430
+
431
environment_variables: dict[str, Callable[[], Any]] = {
432
# ================== Installation Time Env Vars ==================
433
# Target device of vLLM, supporting [cuda (by default),
@@ -1540,85 +1542,88 @@ def is_set(name: str):
1542
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
1543
1544
1543
-def compute_hash() -> str:
1544
- """
1545
- WARNING: Whenever a new key is added to this environment
1546
- variables, ensure that it is included in the factors list if
1547
- it affects the computation graph. For example, different values
1548
- of VLLM_PP_LAYER_PARTITION will generate different computation
1549
- graphs, so it is included in the factors list. The env vars that
1550
- affect the choice of different kernels or attention backends should
1551
- also be included in the factors list.
1552
- """
1553
-
1554
- # The values of envs may affects the computation graph.
1555
- # TODO(DefTruth): hash all environment variables?
1556
- # for key in environment_variables:
1557
- # factorize(key)
1558
- environment_variables_to_hash = [
1559
- "VLLM_PP_LAYER_PARTITION",
1560
- "VLLM_MLA_DISABLE",
1561
- "VLLM_FLASH_ATTN_MAX_NUM_SPLITS_FOR_CUDA_GRAPH",
1562
- "VLLM_USE_TRITON_AWQ",
1563
- "VLLM_DP_RANK",
1564
- "VLLM_DP_SIZE",
1565
- "VLLM_USE_STANDALONE_COMPILE",
1566
- "VLLM_FUSED_MOE_CHUNK_SIZE",
1567
- "VLLM_FLASHINFER_MOE_BACKEND",
1568
- "VLLM_V1_USE_PREFILL_DECODE_ATTENTION",
1569
- "VLLM_ATTENTION_BACKEND",
1570
- "VLLM_USE_FLASHINFER_SAMPLER",
1571
- "VLLM_DISABLED_KERNELS",
1572
- "VLLM_USE_DEEP_GEMM",
1573
- "VLLM_MOE_USE_DEEP_GEMM",
1574
- "VLLM_USE_DEEP_GEMM_E8M0",
1575
- "VLLM_USE_FUSED_MOE_GROUPED_TOPK",
1576
- "VLLM_USE_FLASHINFER_MOE_FP16",
1577
- "VLLM_USE_FLASHINFER_MOE_FP8",
1578
- "VLLM_USE_FLASHINFER_MOE_FP4",
1579
- "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8",
1580
- "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS",
1581
- "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16",
1582
- "VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE",
1583
- "VLLM_USE_CUDNN_PREFILL",
1584
- "VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL",
1585
- "VLLM_USE_TRTLLM_ATTENTION",
1586
- "VLLM_FLASHINFER_DISABLE_Q_QUANTIZATION",
1587
- "VLLM_ROCM_USE_AITER",
1588
- "VLLM_ROCM_USE_AITER_PAGED_ATTN",
1589
- "VLLM_ROCM_USE_AITER_LINEAR",
1590
- "VLLM_ROCM_USE_AITER_MOE",
1591
- "VLLM_ROCM_USE_AITER_RMSNORM",
1592
- "VLLM_ROCM_USE_AITER_MLA",
1593
- "VLLM_ROCM_USE_AITER_MHA",
1594
- "VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
1595
- "VLLM_ROCM_USE_AITER_TRITON_ROPE",
1596
- "VLLM_ROCM_USE_AITER_FP8BMM",
1597
- "VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION",
1598
- "VLLM_ROCM_USE_AITER_TRITON_GEMM",
1599
- "VLLM_ROCM_USE_SKINNY_GEMM",
1600
- "VLLM_ROCM_FP8_PADDING",
1601
- "VLLM_ROCM_MOE_PADDING",
1602
- "VLLM_ROCM_CUSTOM_PAGED_ATTN",
1603
- "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION",
1604
- "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16",
1605
- "VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB",
1606
- "VLLM_ROCM_FP8_MFMA_PAGE_ATTN",
1607
- "VLLM_ENABLE_INDUCTOR_MAX_AUTOTUNE",
1608
- "VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING",
1609
- "VLLM_NVFP4_GEMM_BACKEND",
1610
- "VLLM_USE_FBGEMM",
1611
- "VLLM_DEEPEP_HIGH_THROUGHPUT_FORCE_INTRA_NODE",
1612
- "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL",
1613
- ]
1614
- for key in environment_variables_to_hash:
1615
- # if this goes out of sync with environment_variables,
1616
- # it's not a user error, it's a bug
1617
- assert key in environment_variables, (
1618
- "Please update environment_variables_to_hash in envs.py"
1619
- )
1545
+def compile_factors() -> dict[str, object]:
1546
+ """Return env vars used for torch.compile cache keys.
1547
+
1548
+ Start with every known vLLM env var; drop entries in `ignored_factors`;
1549
+ hash everything else. This keeps the cache key aligned across workers."""
1550
+
1551
+ ignored_factors: set[str] = {
1552
+ "MAX_JOBS",
1553
+ "VLLM_RPC_BASE_PATH",
1554
+ "VLLM_USE_MODELSCOPE",
1555
+ "VLLM_RINGBUFFER_WARNING_INTERVAL",
1556
+ "VLLM_DEBUG_DUMP_PATH",
1557
+ "VLLM_PORT",
1558
+ "VLLM_CACHE_ROOT",
1559
+ "LD_LIBRARY_PATH",
1560
+ "VLLM_SERVER_DEV_MODE",
1561
+ "VLLM_DP_MASTER_IP",
1562
+ "VLLM_DP_MASTER_PORT",
1563
+ "VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
1564
+ "VLLM_CI_USE_S3",
1565
+ "VLLM_MODEL_REDIRECT_PATH",
1566
+ "VLLM_HOST_IP",
1567
+ "S3_ACCESS_KEY_ID",
1568
+ "S3_SECRET_ACCESS_KEY",
1569
+ "S3_ENDPOINT_URL",
1570
+ "VLLM_USAGE_STATS_SERVER",
1571
+ "VLLM_NO_USAGE_STATS",
1572
+ "VLLM_DO_NOT_TRACK",
1573
+ "VLLM_LOGGING_LEVEL",
1574
+ "VLLM_LOGGING_PREFIX",
1575
+ "VLLM_LOGGING_STREAM",
1576
+ "VLLM_LOGGING_CONFIG_PATH",
1577
+ "VLLM_LOG_STATS_INTERVAL",
1578
+ "VLLM_DEBUG_LOG_API_SERVER_RESPONSE",
1579
+ "VLLM_TUNED_CONFIG_FOLDER",
1580
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S",
1581
+ "VLLM_HTTP_TIMEOUT_KEEP_ALIVE",
1582
+ "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS",
1583
+ "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH",
1584
+ "VLLM_SLEEP_WHEN_IDLE",
1585
+ "VLLM_IMAGE_FETCH_TIMEOUT",
1586
+ "VLLM_VIDEO_FETCH_TIMEOUT",
1587
+ "VLLM_AUDIO_FETCH_TIMEOUT",
1588
+ "VLLM_MEDIA_URL_ALLOW_REDIRECTS",
1589
+ "VLLM_MEDIA_LOADING_THREAD_COUNT",
1590
+ "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB",
1591
+ "VLLM_VIDEO_LOADER_BACKEND",
1592
+ "VLLM_MEDIA_CONNECTOR",
1593
+ "VLLM_ASSETS_CACHE",
1594
+ "VLLM_ASSETS_CACHE_MODEL_CLEAN",
1595
+ "VLLM_MM_INPUT_CACHE_GIB",
1596
+ "VLLM_WORKER_MULTIPROC_METHOD",
1597
+ "VLLM_ENABLE_V1_MULTIPROCESSING",
1598
+ "VLLM_V1_OUTPUT_PROC_CHUNK_SIZE",
1599
+ "VLLM_CPU_KVCACHE_SPACE",
1600
+ "VLLM_CPU_OMP_THREADS_BIND",
1601
+ "VLLM_CPU_NUM_OF_RESERVED_CPU",
1602
+ "VLLM_CPU_MOE_PREPACK",
1603
+ "VLLM_CPU_SGL_KERNEL",
1604
+ "VLLM_TEST_FORCE_LOAD_FORMAT",
1605
+ "LOCAL_RANK",
1606
+ "CUDA_VISIBLE_DEVICES",
1607
+ }
1608
+
1609
+ from vllm.config.utils import normalize_value
1610
+
1611
+ factors: dict[str, object] = {}
1612
+ for factor, getter in environment_variables.items():
1613
+ if factor in ignored_factors:
1614
+ continue
1615
+
1616
+ try:
1617
+ raw = getter()
1618
+ except Exception as exc: # pragma: no cover - defensive logging
1619
+ logger.warning(
1620
+ "Skipping environment variable %s while hashing compile factors: %s",
1621
+ factor,
1622
+ exc,
1623
+ )
1624
+ continue
1625
1621
- factors = [environment_variables[key]() for key in environment_variables_to_hash]
1626
+ factors[factor] = normalize_value(raw)
1627
1628
ray_noset_env_vars = [
1629
# Refer to
@@ -1641,8 +1646,8 @@ def compute_hash() -> str:
1646
"RAY_EXPERIMENTAL_NOSET_ONEAPI_DEVICE_SELECTOR",
1647
"RAY_EXPERIMENTAL_NOSET_RBLN_RT_VISIBLE_DEVICES",
1648
]
1644
- factors.extend([os.getenv(var) for var in ray_noset_env_vars])
1649
1646
- hash_str = hashlib.md5(str(factors).encode(), usedforsecurity=False).hexdigest()
1650
+ for var in ray_noset_env_vars:
1651
+ factors[var] = normalize_value(os.getenv(var))
1652
1648
- return hash_str
1653
+ return factors
vllm/logging_utils/__init__.py
+2
@@ -2,9 +2,11 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from vllm.logging_utils.formatter import NewLineFormatter
5
+from vllm.logging_utils.lazy import lazy
6
from vllm.logging_utils.log_time import logtime
7
8
__all__ = [
9
"NewLineFormatter",
10
+ "lazy",
11
"logtime",
12
]
vllm/logging_utils/lazy.py
new
+20
@@ -0,0 +1,20 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Callable
5
+from typing import Any
6
+
7
+
8
+class lazy:
9
+ """Wrap a zero-argument callable evaluated only during log formatting."""
10
+
11
+ __slots__ = ("_factory",)
12
+
13
+ def __init__(self, factory: Callable[[], Any]) -> None:
14
+ self._factory = factory
15
+
16
+ def __str__(self) -> str:
17
+ return str(self._factory())
18
+
19
+ def __repr__(self) -> str:
20
+ return str(self)