A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[torch.compile] fast inductor (#11108)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
youkaichao committed
Dec 16, 2024 at 16:15 UTC
88a412ed3d964de3443c42a6a35108115ee0ad25
3 files changed
+624
-7
vllm/compilation/backends.py
+210
-3
@@ -1,6 +1,10 @@
1
+import ast
2
import copy
3
import dataclasses
4
+import os
5
+import pprint
6
import time
7
+from collections import defaultdict
8
from contextlib import ExitStack
9
from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple
10
from unittest.mock import patch
@@ -21,6 +25,122 @@ from .pass_manager import PostGradPassManager
25
logger = init_logger(__name__)
26
27
28
+class InductorHashCache:
29
+ """
30
+ Disk format: a Python list of tuples, each tuple is
31
+ (runtime_shape, graph_index, hash_str)
32
+ We use list of tuple for readability.
33
+
34
+ In-memory format: a defaultdict of dict, where the key is
35
+ runtime_shape, and the value is a dict of graph_index to hash_str.
36
+
37
+ The data is essentially `Dict[Optional[int], Dict[int, str]]`,
38
+ we don't use json here because json doesn't support int as key.
39
+
40
+ TODO: better off-the-shelf solution to serialize the data?
41
+ """
42
+
43
+ def __init__(self, cache_dir: str, disabled: bool = False):
44
+ self.cache: defaultdict = defaultdict(dict)
45
+ self.disabled = disabled
46
+ self.cache_dir = cache_dir
47
+ self.cache_file_path = os.path.join(cache_dir,
48
+ "inductor_hash_cache.py")
49
+ if disabled:
50
+ return
51
+ # set flags so that Inductor and Triton store their cache
52
+ # in the cache_dir, then users only need to copy the cache_dir
53
+ # to another machine to reuse the cache.
54
+ inductor_cache = os.path.join(cache_dir, "inductor_cache")
55
+ os.makedirs(inductor_cache, exist_ok=True)
56
+ os.environ["TORCHINDUCTOR_CACHE_DIR"] = inductor_cache
57
+ triton_cache = os.path.join(cache_dir, "triton_cache")
58
+ os.makedirs(triton_cache, exist_ok=True)
59
+ os.environ["TRITON_CACHE_DIR"] = triton_cache
60
+ if os.path.exists(self.cache_file_path):
61
+ with open(self.cache_file_path) as f:
62
+ self.deserialize(f.read())
63
+
64
+ def deserialize(self, data: str):
65
+ # we use ast.literal_eval to parse the data
66
+ # because it is a safe way to parse Python literals.
67
+ # do not use eval(), it is unsafe.
68
+ list_data = ast.literal_eval(data)
69
+ for runtime_shape, graph_index, hash_str in list_data:
70
+ self.cache[runtime_shape][graph_index] = hash_str
71
+
72
+ def serialize(self) -> str:
73
+ data = []
74
+ for runtime_shape, graph_index_to_hash_str in self.cache.items():
75
+ for graph_index, hash_str in graph_index_to_hash_str.items():
76
+ data.append((runtime_shape, graph_index, hash_str))
77
+ printer = pprint.PrettyPrinter(indent=4)
78
+ return printer.pformat(data)
79
+
80
+ def save_to_file(self):
81
+ if self.disabled:
82
+ return
83
+ with open(self.cache_file_path, "w") as f:
84
+ f.write(self.serialize())
85
+
86
+ def __contains__(self, key: Tuple[Optional[int], int]) -> bool:
87
+ if self.disabled:
88
+ return False
89
+ runtime_shape, graph_index = key
90
+ return runtime_shape in self.cache and graph_index in self.cache[
91
+ runtime_shape]
92
+
93
+ def __getitem__(self, key: Tuple[Optional[int], int]) -> str:
94
+ if self.disabled:
95
+ raise KeyError("cannot read from disabled cache")
96
+ runtime_shape, graph_index = key
97
+ return self.cache[runtime_shape][graph_index]
98
+
99
+ def __setitem__(self, key: Tuple[Optional[int], int], value: str):
100
+ # setitem for disabled cache is fine, because we
101
+ # don't actually write to the disk
102
+ runtime_shape, graph_index = key
103
+ self.cache[runtime_shape][graph_index] = value
104
+
105
+
106
+class AlwaysHitShapeEnv:
107
+ """
108
+ Why do we need this class:
109
+
110
+ For normal `torch.compile` usage, every compilation will have
111
+ one Dynamo bytecode compilation and one Inductor compilation.
112
+ The Inductor compilation happens under the context of the
113
+ Dynamo bytecode compilation, and that context is used to
114
+ determine the dynamic shape information, etc.
115
+
116
+ For our use case, we only run Dynamo bytecode compilation once,
117
+ and run Inductor compilation multiple times with different shapes
118
+ plus a general shape. The compilation for specific shapes happens
119
+ outside of the context of the Dynamo bytecode compilation. At that
120
+ time, we don't have shape environment to provide to Inductor, and
121
+ it will fail the Inductor code cache lookup.
122
+
123
+ By providing a dummy shape environment that always hits, we can
124
+ make the Inductor code cache lookup always hit, and we can
125
+ compile the graph for different shapes as needed.
126
+
127
+ The following dummy methods are obtained by trial-and-error
128
+ until it works.
129
+ """
130
+
131
+ def __init__(self) -> None:
132
+ self.guards: List[Any] = []
133
+
134
+ def evaluate_guards_expression(self, *args, **kwargs):
135
+ return True
136
+
137
+ def get_pruned_guards(self, *args, **kwargs):
138
+ return []
139
+
140
+ def produce_guards_expression(self, *args, **kwargs):
141
+ return ""
142
+
143
+
144
def wrap_inductor(graph,
145
example_inputs,
146
additional_inductor_config,
@@ -55,9 +175,93 @@ def wrap_inductor(graph,
175
# inductor can inplace modify the graph, so we need to copy it
176
# see https://github.com/pytorch/pytorch/issues/138980
177
graph = copy.deepcopy(graph)
58
- compiled_graph = compile_fx(graph,
59
- example_inputs,
60
- config_patches=current_config)
178
+
179
+ cache_data = compilation_config.inductor_hash_cache
180
+ if (runtime_shape, graph_index) in cache_data:
181
+ # we compiled this graph before
182
+ # so we can directly lookup the compiled graph via hash
183
+ hash_str = cache_data[(runtime_shape, graph_index)]
184
+ if graph_index == 0:
185
+ # adds some info logging for the first graph
186
+ logger.info(
187
+ "Directly lookup the graph for shape %s from the cache",
188
+ str(runtime_shape)) # noqa
189
+ logger.debug(
190
+ "directly lookup the %s-th graph for shape %s via hash %s",
191
+ graph_index, str(runtime_shape), hash_str)
192
+ from torch._inductor.codecache import FxGraphCache
193
+ with patch("torch._inductor.codecache.FxGraphCache._get_shape_env",
194
+ lambda *args, **kwargs: AlwaysHitShapeEnv()):
195
+ inductor_compiled_graph = FxGraphCache._lookup_graph(
196
+ hash_str, example_inputs, True, False)
197
+ assert inductor_compiled_graph is not None, (
198
+ "Inductor cache lookup failed. Please remove"
199
+ f"the cache file {compilation_config.inductor_hash_cache.cache_file_path} and try again." # noqa
200
+ )
201
+
202
+ # Inductor calling convention (function signature):
203
+ # f(list) -> tuple
204
+ # Dynamo calling convention (function signature):
205
+ # f(*args) -> Any
206
+
207
+ # need to know if the graph returns a tuple
208
+ from torch._inductor.compile_fx import graph_returns_tuple
209
+ returns_tuple = graph_returns_tuple(graph)
210
+
211
+ # this is the graph we return to Dynamo to run
212
+ def compiled_graph(*args):
213
+ # convert args to list
214
+ list_args = list(args)
215
+ graph_output = inductor_compiled_graph(list_args)
216
+ # unpack the tuple if needed
217
+ if returns_tuple:
218
+ return graph_output
219
+ else:
220
+ return graph_output[0]
221
+ else:
222
+ # it's the first time we compile this graph
223
+ # the assumption is that we don't have nested Inductor compilation.
224
+ # compiled_fx_graph_hash will only be called once, and we can hook
225
+ # it to get the hash of the compiled graph directly.
226
+ from torch._inductor.codecache import compiled_fx_graph_hash
227
+
228
+ def hijack_compiled_fx_graph_hash(*args, **kwargs):
229
+ out = compiled_fx_graph_hash(*args, **kwargs)
230
+ # store the hash in the cache
231
+ nonlocal cache_data
232
+ cache_data[(runtime_shape, graph_index)] = out[0]
233
+ if graph_index == 0:
234
+ # adds some info logging for the first graph
235
+ logger.info("Cache the graph of shape %s for later use",
236
+ str(runtime_shape))
237
+ logger.debug("store the %s-th graph for shape %s via hash %s",
238
+ graph_index, str(runtime_shape), out[0])
239
+ return out
240
+
241
+ def _check_can_cache(*args, **kwargs):
242
+ # no error means it can be cached.
243
+ # Inductor refuses to cache the graph outside of Dynamo
244
+ # tracing context, and also disables caching for graphs
245
+ # with high-order ops.
246
+ # For vLLM, in either case, we want to cache the graph.
247
+ # see https://github.com/pytorch/pytorch/blob/9f5ebf3fc609105a74eab4ccc24932d6353ff566/torch/_inductor/codecache.py#L1221 # noqa
248
+ return
249
+
250
+ def _get_shape_env():
251
+ return AlwaysHitShapeEnv()
252
+
253
+ with patch(# for hijacking the hash of the compiled graph
254
+ "torch._inductor.codecache.compiled_fx_graph_hash",
255
+ hijack_compiled_fx_graph_hash), \
256
+ patch(# for providing a dummy shape environment
257
+ "torch._inductor.codecache.FxGraphCache._get_shape_env",
258
+ _get_shape_env), \
259
+ patch(# for forcing the graph to be cached
260
+ "torch._inductor.codecache.FxGraphCache._check_can_cache",
261
+ _check_can_cache):
262
+ compiled_graph = compile_fx(graph,
263
+ example_inputs,
264
+ config_patches=current_config)
265
266
# after compiling the last graph, record the end time
267
if graph_index == num_graphs - 1:
@@ -457,6 +661,9 @@ class PiecewiseBackend:
661
662
# finished compilations for all required shapes
663
if self.is_last_graph and not self.to_be_compiled_sizes:
664
+
665
+ # save the hash of the inductor graph for the next run
666
+ self.compilation_config.inductor_hash_cache.save_to_file()
667
end_monitoring_torch_compile(self.vllm_config)
668
669
if not entry.use_cudagraph:
vllm/config.py
+411
-4
@@ -3,6 +3,7 @@ import copy
3
import enum
4
import hashlib
5
import json
6
+import os
7
import warnings
8
from contextlib import contextmanager
9
from dataclasses import dataclass, field, replace
@@ -162,6 +163,30 @@ class ModelConfig:
163
which allows no processors.
164
"""
165
166
+ def compute_hash(self) -> str:
167
+ """
168
+ WARNING: Whenever a new field is added to this config,
169
+ ensure that it is included in the factors list if
170
+ it affects the computation graph.
171
+
172
+ Provide a hash that uniquely identifies all the configs
173
+ that affect the structure of the computation
174
+ graph from input ids/embeddings to the final hidden states,
175
+ excluding anything before input ids/embeddings and after
176
+ the final hidden states.
177
+ """
178
+ factors: List[Any] = []
179
+ factors.append(self.model)
180
+ factors.append(self.dtype)
181
+ factors.append(self.quantization)
182
+ factors.append(self.quantization_param_path)
183
+ factors.append(self.revision)
184
+ factors.append(self.code_revision)
185
+ factors.append(self.trust_remote_code)
186
+ factors.append(self.rope_scaling)
187
+ factors.append(self.rope_theta)
188
+ return hashlib.sha256(str(factors).encode()).hexdigest()
189
+
190
def __init__(self,
191
model: str,
192
task: Union[TaskOption, Literal["draft"]],
@@ -203,6 +228,8 @@ class ModelConfig:
228
self.seed = seed
229
self.revision = revision
230
self.code_revision = code_revision
231
+ self.rope_scaling = rope_scaling
232
+ self.rope_theta = rope_theta
233
234
if hf_overrides is None:
235
hf_overrides = {}
@@ -832,6 +859,24 @@ class CacheConfig:
859
cpu_offload_gb: Size of the CPU offload buffer in GiB.
860
"""
861
862
+ def compute_hash(self) -> str:
863
+ """
864
+ WARNING: Whenever a new field is added to this config,
865
+ ensure that it is included in the factors list if
866
+ it affects the computation graph.
867
+
868
+ Provide a hash that uniquely identifies all the configs
869
+ that affect the structure of the computation
870
+ graph from input ids/embeddings to the final hidden states,
871
+ excluding anything before input ids/embeddings and after
872
+ the final hidden states.
873
+ """
874
+ factors: List[Any] = []
875
+ factors.append(self.cache_dtype)
876
+ # `cpu_offload_gb` does not use `torch.compile` yet.
877
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
878
+ return hash_str
879
+
880
def __init__(
881
self,
882
block_size: int,
@@ -928,6 +973,24 @@ class TokenizerPoolConfig:
973
pool_type: Union[str, Type["BaseTokenizerGroup"]]
974
extra_config: dict
975
976
+ def compute_hash(self) -> str:
977
+ """
978
+ WARNING: Whenever a new field is added to this config,
979
+ ensure that it is included in the factors list if
980
+ it affects the computation graph.
981
+
982
+ Provide a hash that uniquely identifies all the configs
983
+ that affect the structure of the computation
984
+ graph from input ids/embeddings to the final hidden states,
985
+ excluding anything before input ids/embeddings and after
986
+ the final hidden states.
987
+ """
988
+ # no factors to consider.
989
+ # this config will not affect the computation graph.
990
+ factors: List[Any] = []
991
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
992
+ return hash_str
993
+
994
def __post_init__(self):
995
if self.pool_type not in ("ray", ) and not isinstance(
996
self.pool_type, type):
@@ -1010,6 +1073,24 @@ class LoadConfig:
1073
default_factory=dict)
1074
ignore_patterns: Optional[Union[List[str], str]] = None
1075
1076
+ def compute_hash(self) -> str:
1077
+ """
1078
+ WARNING: Whenever a new field is added to this config,
1079
+ ensure that it is included in the factors list if
1080
+ it affects the computation graph.
1081
+
1082
+ Provide a hash that uniquely identifies all the configs
1083
+ that affect the structure of the computation
1084
+ graph from input ids/embeddings to the final hidden states,
1085
+ excluding anything before input ids/embeddings and after
1086
+ the final hidden states.
1087
+ """
1088
+ # no factors to consider.
1089
+ # this config will not affect the computation graph.
1090
+ factors: List[Any] = []
1091
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1092
+ return hash_str
1093
+
1094
def __post_init__(self):
1095
model_loader_extra_config = self.model_loader_extra_config or {}
1096
if isinstance(model_loader_extra_config, str):
@@ -1073,6 +1154,19 @@ class ParallelConfig:
1154
1155
rank: int = 0
1156
1157
+ def compute_hash(self):
1158
+ """
1159
+ Provide a hash that uniquely identifies all the configs
1160
+ that affect the structure of the computation
1161
+ graph from input ids/embeddings to the final hidden states,
1162
+ excluding anything before input ids/embeddings and after
1163
+ the final hidden states.
1164
+ """
1165
+ factors: List[Any] = []
1166
+ factors.append(self.pipeline_parallel_size)
1167
+ factors.append(self.tensor_parallel_size)
1168
+ return hashlib.sha256(str(factors).encode()).hexdigest()
1169
+
1170
def __post_init__(self) -> None:
1171
self.world_size = self.pipeline_parallel_size * \
1172
self.tensor_parallel_size
@@ -1209,6 +1303,24 @@ class SchedulerConfig:
1303
1304
chunked_prefill_enabled: bool = field(init=False)
1305
1306
+ def compute_hash(self) -> str:
1307
+ """
1308
+ WARNING: Whenever a new field is added to this config,
1309
+ ensure that it is included in the factors list if
1310
+ it affects the computation graph.
1311
+
1312
+ Provide a hash that uniquely identifies all the configs
1313
+ that affect the structure of the computation
1314
+ graph from input ids/embeddings to the final hidden states,
1315
+ excluding anything before input ids/embeddings and after
1316
+ the final hidden states.
1317
+ """
1318
+ # no factors to consider.
1319
+ # this config will not affect the computation graph.
1320
+ factors: List[Any] = []
1321
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1322
+ return hash_str
1323
+
1324
def __post_init__(self) -> None:
1325
if self.max_num_batched_tokens is None:
1326
if self.enable_chunked_prefill:
@@ -1286,6 +1398,25 @@ class DeviceConfig:
1398
device: Optional[torch.device]
1399
device_type: str
1400
1401
+ def compute_hash(self) -> str:
1402
+ """
1403
+ WARNING: Whenever a new field is added to this config,
1404
+ ensure that it is included in the factors list if
1405
+ it affects the computation graph.
1406
+
1407
+ Provide a hash that uniquely identifies all the configs
1408
+ that affect the structure of the computation
1409
+ graph from input ids/embeddings to the final hidden states,
1410
+ excluding anything before input ids/embeddings and after
1411
+ the final hidden states.
1412
+ """
1413
+ # no factors to consider.
1414
+ # the device/platform information will be summarized
1415
+ # by torch/vllm automatically.
1416
+ factors: List[Any] = []
1417
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1418
+ return hash_str
1419
+
1420
def __init__(self, device: str = "auto") -> None:
1421
if device == "auto":
1422
# Automated device type detection
@@ -1313,6 +1444,24 @@ class SpeculativeConfig:
1444
decoding with top-1 proposals.
1445
"""
1446
1447
+ def compute_hash(self) -> str:
1448
+ """
1449
+ WARNING: Whenever a new field is added to this config,
1450
+ ensure that it is included in the factors list if
1451
+ it affects the computation graph.
1452
+
1453
+ Provide a hash that uniquely identifies all the configs
1454
+ that affect the structure of the computation
1455
+ graph from input ids/embeddings to the final hidden states,
1456
+ excluding anything before input ids/embeddings and after
1457
+ the final hidden states.
1458
+ """
1459
+ # no factors to consider.
1460
+ # spec decode does not use `torch.compile` yet.
1461
+ factors: List[Any] = []
1462
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1463
+ return hash_str
1464
+
1465
@staticmethod
1466
def maybe_create_spec_config(
1467
target_model_config: ModelConfig,
@@ -1753,6 +1902,24 @@ class LoRAConfig:
1902
long_lora_scaling_factors: Optional[Tuple[float]] = None
1903
bias_enabled: bool = False
1904
1905
+ def compute_hash(self) -> str:
1906
+ """
1907
+ WARNING: Whenever a new field is added to this config,
1908
+ ensure that it is included in the factors list if
1909
+ it affects the computation graph.
1910
+
1911
+ Provide a hash that uniquely identifies all the configs
1912
+ that affect the structure of the computation
1913
+ graph from input ids/embeddings to the final hidden states,
1914
+ excluding anything before input ids/embeddings and after
1915
+ the final hidden states.
1916
+ """
1917
+ # no factors to consider.
1918
+ # LoRA is not compatible with `torch.compile` .
1919
+ factors: List[Any] = []
1920
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1921
+ return hash_str
1922
+
1923
def __post_init__(self):
1924
# Setting the maximum rank to 256 should be able to satisfy the vast
1925
# majority of applications.
@@ -1802,6 +1969,24 @@ class PromptAdapterConfig:
1969
max_cpu_prompt_adapters: Optional[int] = None
1970
prompt_adapter_dtype: Optional[torch.dtype] = None
1971
1972
+ def compute_hash(self) -> str:
1973
+ """
1974
+ WARNING: Whenever a new field is added to this config,
1975
+ ensure that it is included in the factors list if
1976
+ it affects the computation graph.
1977
+
1978
+ Provide a hash that uniquely identifies all the configs
1979
+ that affect the structure of the computation
1980
+ graph from input ids/embeddings to the final hidden states,
1981
+ excluding anything before input ids/embeddings and after
1982
+ the final hidden states.
1983
+ """
1984
+ # no factors to consider.
1985
+ # this config will not affect the computation graph.
1986
+ factors: List[Any] = []
1987
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
1988
+ return hash_str
1989
+
1990
def __post_init__(self):
1991
1992
if self.max_prompt_adapters < 1:
@@ -1830,6 +2015,24 @@ class MultiModalConfig:
2015
for each :class:`~vllm.multimodal.MultiModalPlugin`.
2016
"""
2017
2018
+ def compute_hash(self) -> str:
2019
+ """
2020
+ WARNING: Whenever a new field is added to this config,
2021
+ ensure that it is included in the factors list if
2022
+ it affects the computation graph.
2023
+
2024
+ Provide a hash that uniquely identifies all the configs
2025
+ that affect the structure of the computation
2026
+ graph from input ids/embeddings to the final hidden states,
2027
+ excluding anything before input ids/embeddings and after
2028
+ the final hidden states.
2029
+ """
2030
+ # no factors to consider.
2031
+ # this config will not affect the computation graph.
2032
+ factors: List[Any] = []
2033
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2034
+ return hash_str
2035
+
2036
# TODO: Add configs to init vision tower or not.
2037
2038
@@ -1869,6 +2072,24 @@ class PoolerConfig:
2072
``math-shepherd-mistral-7b-prm`` model.
2073
"""
2074
2075
+ def compute_hash(self) -> str:
2076
+ """
2077
+ WARNING: Whenever a new field is added to this config,
2078
+ ensure that it is included in the factors list if
2079
+ it affects the computation graph.
2080
+
2081
+ Provide a hash that uniquely identifies all the configs
2082
+ that affect the structure of the computation
2083
+ graph from input ids/embeddings to the final hidden states,
2084
+ excluding anything before input ids/embeddings and after
2085
+ the final hidden states.
2086
+ """
2087
+ # no factors to consider.
2088
+ # this config will not affect the computation graph.
2089
+ factors: List[Any] = []
2090
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2091
+ return hash_str
2092
+
2093
@staticmethod
2094
def from_json(json_str: str) -> "PoolerConfig":
2095
return PoolerConfig(**json.loads(json_str))
@@ -2103,6 +2324,24 @@ class DecodingConfig:
2324
# 'outlines' / 'lm-format-enforcer' / 'xgrammar'
2325
guided_decoding_backend: str = 'xgrammar'
2326
2327
+ def compute_hash(self) -> str:
2328
+ """
2329
+ WARNING: Whenever a new field is added to this config,
2330
+ ensure that it is included in the factors list if
2331
+ it affects the computation graph.
2332
+
2333
+ Provide a hash that uniquely identifies all the configs
2334
+ that affect the structure of the computation
2335
+ graph from input ids/embeddings to the final hidden states,
2336
+ excluding anything before input ids/embeddings and after
2337
+ the final hidden states.
2338
+ """
2339
+ # no factors to consider.
2340
+ # this config will not affect the computation graph.
2341
+ factors: List[Any] = []
2342
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2343
+ return hash_str
2344
+
2345
def __post_init__(self):
2346
valid_guided_backends = ['outlines', 'lm-format-enforcer', 'xgrammar']
2347
backend = self.guided_decoding_backend
@@ -2124,6 +2363,24 @@ class ObservabilityConfig:
2363
# If set, collects the model execute time for the request.
2364
collect_model_execute_time: bool = False
2365
2366
+ def compute_hash(self) -> str:
2367
+ """
2368
+ WARNING: Whenever a new field is added to this config,
2369
+ ensure that it is included in the factors list if
2370
+ it affects the computation graph.
2371
+
2372
+ Provide a hash that uniquely identifies all the configs
2373
+ that affect the structure of the computation
2374
+ graph from input ids/embeddings to the final hidden states,
2375
+ excluding anything before input ids/embeddings and after
2376
+ the final hidden states.
2377
+ """
2378
+ # no factors to consider.
2379
+ # this config will not affect the computation graph.
2380
+ factors: List[Any] = []
2381
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2382
+ return hash_str
2383
+
2384
def __post_init__(self):
2385
if not is_otel_available() and self.otlp_traces_endpoint is not None:
2386
raise ValueError(
@@ -2165,6 +2422,24 @@ class KVTransferConfig(BaseModel):
2422
# The KV connector port, used to build distributed connection
2423
kv_port: int = 14579
2424
2425
+ def compute_hash(self) -> str:
2426
+ """
2427
+ WARNING: Whenever a new field is added to this config,
2428
+ ensure that it is included in the factors list if
2429
+ it affects the computation graph.
2430
+
2431
+ Provide a hash that uniquely identifies all the configs
2432
+ that affect the structure of the computation
2433
+ graph from input ids/embeddings to the final hidden states,
2434
+ excluding anything before input ids/embeddings and after
2435
+ the final hidden states.
2436
+ """
2437
+ # no factors to consider.
2438
+ # this config will not affect the computation graph.
2439
+ factors: List[Any] = []
2440
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()
2441
+ return hash_str
2442
+
2443
@classmethod
2444
def from_cli(cls, cli_value: str) -> "KVTransferConfig":
2445
"""Parse the CLI value for the kv cache transfer config."""
@@ -2234,6 +2509,9 @@ class CompilationConfig(BaseModel):
2509
- 2: dynamo once.
2510
- 3: piecewise compilation.
2511
- debug_dump_path: the path to dump the debug information.
2512
+ - cache_dir: the directory to store the compiled graph, to
2513
+ accelerate Inductor compilation. By default, it will use
2514
+ model-related information to generate a cache directory.
2515
- backend: the backend for compilation. It needs to be a string.
2516
- "" (empty string): use the default backend.
2517
- "eager"/"openxla"/...: use the specified backend registered in PyTorch.
@@ -2302,12 +2580,10 @@ class CompilationConfig(BaseModel):
2580
""" # noqa
2581
level: int = 0
2582
debug_dump_path: str = ""
2583
+ cache_dir: str = ""
2584
backend: str = ""
2585
custom_ops: List[str] = Field(default_factory=list)
2307
- splitting_ops: List[str] = Field(default_factory=lambda: [
2308
- "vllm.unified_attention",
2309
- "vllm.unified_attention_with_output",
2310
- ])
2586
+ splitting_ops: List[str] = Field(default=None) # type: ignore
2587
2588
use_inductor: bool = True
2589
candidate_compile_sizes: Optional[List[int]] = Field(default=None)
@@ -2371,12 +2647,37 @@ class CompilationConfig(BaseModel):
2647
enabled_custom_ops: Counter[str] = PrivateAttr
2648
disabled_custom_ops: Counter[str] = PrivateAttr
2649
compilation_time: float = PrivateAttr
2650
+ # should be InductorHashCache, but Pydantic does not support it
2651
+ inductor_hash_cache: Any = PrivateAttr
2652
2653
# Per-model forward context
2654
# Mainly used to store attention cls
2655
# Map from layer name to the attention cls
2656
static_forward_context: Dict[str, Any] = PrivateAttr
2657
2658
+ def compute_hash(self) -> str:
2659
+ """
2660
+ WARNING: Whenever a new field is added to this config,
2661
+ ensure that it is included in the factors list if
2662
+ it affects the computation graph.
2663
+
2664
+ Provide a hash that uniquely identifies all the configs
2665
+ that affect the structure of the computation
2666
+ graph from input ids/embeddings to the final hidden states,
2667
+ excluding anything before input ids/embeddings and after
2668
+ the final hidden states.
2669
+ """
2670
+ factors: List[Any] = []
2671
+ factors.append(self.level)
2672
+ factors.append(self.backend)
2673
+ factors.append(self.custom_ops)
2674
+ factors.append(self.splitting_ops)
2675
+ factors.append(self.use_inductor)
2676
+ factors.append(self.inductor_compile_config)
2677
+ factors.append(self.inductor_passes)
2678
+ factors.append(self.pass_config.uuid())
2679
+ return hashlib.sha256(str(factors).encode()).hexdigest()
2680
+
2681
def __repr__(self) -> str:
2682
exclude = {
2683
"static_forward_context",
@@ -2405,6 +2706,27 @@ class CompilationConfig(BaseModel):
2706
count_all = self.custom_ops.count("all")
2707
assert count_none + count_all <= 1, "Can only specify 'none' or 'all'"
2708
2709
+ if self.splitting_ops is None:
2710
+ if envs.VLLM_USE_V1:
2711
+ # v1 must split the graph on attention ops
2712
+ # for piecewise cudagraph
2713
+ self.splitting_ops = [
2714
+ "vllm.unified_attention",
2715
+ "vllm.unified_attention_with_output",
2716
+ ]
2717
+ else:
2718
+ # v0 can use full graph compilation without splitting,
2719
+ # splitting is optional.
2720
+ # right now we still need it. kv cache shape
2721
+ # will be included in the graph if we don't split
2722
+ # the graph.
2723
+ # TODO: hide kv cache in static forward context
2724
+ # so that inductor does not see it.
2725
+ self.splitting_ops = [
2726
+ "vllm.unified_attention",
2727
+ "vllm.unified_attention_with_output",
2728
+ ]
2729
+
2730
for k, v in self.inductor_passes.items():
2731
if not isinstance(v, str):
2732
assert callable(v), (
@@ -2444,6 +2766,30 @@ class CompilationConfig(BaseModel):
2766
# TODO: pass user-specified backend to piecewise compilation
2767
# merge with the config use_inductor
2768
assert self.level == CompilationLevel.PIECEWISE
2769
+
2770
+ if not self.cache_dir:
2771
+ # no provided cache dir, generate one based on the known factors
2772
+ # that affects the compilation. if none of the factors change,
2773
+ # the cache dir will be the same so that we can reuse the compiled
2774
+ # graph.
2775
+ hash_key = vllm_config.compute_hash()
2776
+ cache_dir = os.path.join(
2777
+ envs.VLLM_CACHE_ROOT, "torch_compile_cache", hash_key,
2778
+ f"rank_{vllm_config.parallel_config.rank}")
2779
+ os.makedirs(cache_dir, exist_ok=True)
2780
+ self.cache_dir = cache_dir
2781
+
2782
+ disabled = envs.VLLM_DISABLE_COMPILE_CACHE
2783
+ from vllm.compilation.backends import InductorHashCache
2784
+ self.inductor_hash_cache: InductorHashCache = InductorHashCache(
2785
+ self.cache_dir, disabled=disabled)
2786
+ if disabled:
2787
+ logger.info("vLLM's torch.compile cache is disabled.")
2788
+ else:
2789
+ logger.info(
2790
+ "Using cache directory: %s for vLLM's torch.compile",
2791
+ self.cache_dir)
2792
+
2793
from vllm.compilation.backends import VllmBackend
2794
return VllmBackend(vllm_config)
2795
@@ -2520,6 +2866,67 @@ class VllmConfig:
2866
init=True) # type: ignore
2867
instance_id: str = ""
2868
2869
+ def compute_hash(self) -> str:
2870
+ """
2871
+ WARNING: Whenever a new field is added to this config,
2872
+ ensure that it is included in the factors list if
2873
+ it affects the computation graph.
2874
+
2875
+ Provide a hash that uniquely identifies all the configs
2876
+ that affect the structure of the computation
2877
+ graph from input ids/embeddings to the final hidden states,
2878
+ excluding anything before input ids/embeddings and after
2879
+ the final hidden states.
2880
+ """
2881
+ factors: List[Any] = []
2882
+ # summarize system state
2883
+ from torch._inductor.codecache import CacheBase
2884
+ system_factors = CacheBase.get_system()
2885
+ factors.append(system_factors)
2886
+
2887
+ # summarize pytorch state
2888
+ from torch._inductor.codecache import torch_key
2889
+ torch_factors = torch_key()
2890
+ factors.append(torch_factors)
2891
+
2892
+ # summarize vllm config
2893
+ vllm_factors: List[Any] = []
2894
+ from vllm import __version__
2895
+ vllm_factors.append(__version__)
2896
+ if self.model_config:
2897
+ vllm_factors.append(self.model_config.compute_hash())
2898
+ if self.cache_config:
2899
+ vllm_factors.append(self.cache_config.compute_hash())
2900
+ if self.parallel_config:
2901
+ vllm_factors.append(self.parallel_config.compute_hash())
2902
+ if self.scheduler_config:
2903
+ vllm_factors.append(self.scheduler_config.compute_hash())
2904
+ if self.device_config:
2905
+ vllm_factors.append(self.device_config.compute_hash())
2906
+ if self.load_config:
2907
+ vllm_factors.append(self.load_config.compute_hash())
2908
+ if self.lora_config:
2909
+ vllm_factors.append(self.lora_config.compute_hash())
2910
+ if self.speculative_config:
2911
+ vllm_factors.append(self.speculative_config.compute_hash())
2912
+ if self.decoding_config:
2913
+ vllm_factors.append(self.decoding_config.compute_hash())
2914
+ if self.observability_config:
2915
+ vllm_factors.append(self.observability_config.compute_hash())
2916
+ if self.prompt_adapter_config:
2917
+ vllm_factors.append(self.prompt_adapter_config.compute_hash())
2918
+ if self.quant_config:
2919
+ pass # should be captured by model_config.quantization
2920
+ if self.compilation_config:
2921
+ vllm_factors.append(self.compilation_config.compute_hash())
2922
+ if self.kv_transfer_config:
2923
+ vllm_factors.append(self.kv_transfer_config.compute_hash())
2924
+
2925
+ factors.append(vllm_factors)
2926
+
2927
+ hash_str = hashlib.md5(str(factors).encode()).hexdigest()[:10]
2928
+ return hash_str
2929
+
2930
def pad_for_cudagraph(self, batch_size: int) -> int:
2931
# if batch_size > self.compilation_config.max_capture_size,
2932
# it should raise an IndexError.
vllm/envs.py
+3
@@ -71,6 +71,7 @@ if TYPE_CHECKING:
71
VLLM_USE_V1: bool = False
72
VLLM_ENABLE_V1_MULTIPROCESSING: bool = True
73
VLLM_LOG_BATCHSIZE_INTERVAL: float = -1
74
+ VLLM_DISABLE_COMPILE_CACHE: bool = False
75
76
77
def get_default_cache_root():
@@ -463,6 +464,8 @@ environment_variables: Dict[str, Callable[[], Any]] = {
464
lambda: bool(int(os.getenv("VLLM_ENABLE_V1_MULTIPROCESSING", "1"))),
465
"VLLM_LOG_BATCHSIZE_INTERVAL":
466
lambda: float(os.getenv("VLLM_LOG_BATCHSIZE_INTERVAL", "-1")),
467
+ "VLLM_DISABLE_COMPILE_CACHE":
468
+ lambda: bool(int(os.getenv("VLLM_DISABLE_COMPILE_CACHE", "0"))),
469
}
470
471
# end-env-vars-definition