A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Avoid overwriting vllm_compile_cache.py (#17418)
Signed-off-by: Keyun Tong <tongkeyun@gmail.com>
Keyun Tong committed
May 1, 2025 at 00:30 UTC
26bc4bbcd8ad081de53436be657a00ea0cc9efd7
1 file changed
+5
-3
vllm/compilation/backends.py
+5
-3
@@ -45,6 +45,7 @@ class CompilerManager:
45
self.cache: Dict[Tuple[Optional[int], int, str], Any] = dict()
46
cls = InductorAdaptor if use_inductor else EagerAdaptor
47
self.compiler = cls()
48
+ self.is_cache_updated = False
49
50
def compute_hash(self, vllm_config: VllmConfig) -> str:
51
return self.compiler.compute_hash(vllm_config)
@@ -66,11 +67,11 @@ class CompilerManager:
67
disable_cache=disable_cache)
68
69
def save_to_file(self):
69
- if self.disable_cache:
70
+ if self.disable_cache or not self.is_cache_updated:
71
return
72
+ printer = pprint.PrettyPrinter(indent=4)
73
+ data = printer.pformat(self.cache)
74
with open(self.cache_file_path, "w") as f:
72
- printer = pprint.PrettyPrinter(indent=4)
73
- data = printer.pformat(self.cache)
75
f.write(data)
76
77
def load(self,
@@ -131,6 +132,7 @@ class CompilerManager:
132
if handle is not None:
133
self.cache[(runtime_shape, graph_index,
134
self.compiler.name)] = handle
135
+ self.is_cache_updated = True
136
if graph_index == 0:
137
# adds some info logging for the first graph
138
logger.info("Cache the graph of shape %s for later use",