A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 15:00 UTC
[compile] Parse compile range cache keys as Range during cache loading. (#30516)
Signed-off-by: zhxchen17 <zhxchen17@fb.com>
Zhengxu Chen committed
Dec 11, 2025 at 23:30 UTC
fe1787107e5214ffb1f0943bf9dd0215cf85ebf2
1 file changed
+19
-1
vllm/compilation/backends.py
+19
-1
@@ -141,7 +141,25 @@ class CompilerManager:
141
# we use ast.literal_eval to parse the data
142
# because it is a safe way to parse Python literals.
143
# do not use eval(), it is unsafe.
144
- self.cache = ast.literal_eval(f.read())
144
+ cache = ast.literal_eval(f.read())
145
+
146
+ def check_type(value, ty):
147
+ if not isinstance(value, ty):
148
+ raise TypeError(f"Expected {ty} but got {type(value)} for {value}")
149
+
150
+ def parse_key(key: Any) -> tuple[Range, int, str]:
151
+ range_tuple, graph_index, compiler_name = key
152
+ check_type(graph_index, int)
153
+ check_type(compiler_name, str)
154
+ if isinstance(range_tuple, tuple):
155
+ start, end = range_tuple
156
+ check_type(start, int)
157
+ check_type(end, int)
158
+ range_tuple = Range(start=start, end=end)
159
+ check_type(range_tuple, Range)
160
+ return range_tuple, graph_index, compiler_name
161
+
162
+ self.cache = {parse_key(key): value for key, value in cache.items()}
163
164
self.compiler.initialize_cache(
165
cache_dir=cache_dir, disable_cache=disable_cache, prefix=prefix