A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[BugFix] VLLM_DISABLE_COMPILE_CACHE=1 should disable all reads and writes from the cache (#20942)
Signed-off-by: Richard Zou <zou3519@gmail.com>
Richard Zou committed
Jul 14, 2025 at 21:26 UTC
ba8c300018e18dd4cdd2b7d904086feec5a79287
4 files changed
+33
-2
tests/compile/test_config.py
+24
@@ -26,6 +26,30 @@ def test_use_cudagraphs_dynamic(monkeypatch):
26
assert not vllm_config.compilation_config.use_cudagraph
27
28
29
+# NB: We don't test VLLM_DISABLE_COMPILE_CACHE=0 because that depends
30
+# on the state of the cache directory on the current machine, which
31
+# may be influenced by other tests.
32
+@pytest.mark.parametrize("val", ["1"])
33
+def test_VLLM_DISABLE_COMPILE_CACHE(vllm_runner, monkeypatch, val):
34
+ assert vllm.envs.VLLM_USE_V1
35
+
36
+ # spawn means that the counters are in the same process.
37
+ monkeypatch.setenv('VLLM_WORKER_MULTIPROC_METHOD', "spawn")
38
+ monkeypatch.setenv('VLLM_DISABLE_COMPILE_CACHE', val)
39
+
40
+ compilation_config = {
41
+ "use_cudagraph": False, # speed things up a bit
42
+ }
43
+ with (
44
+ compilation_counter.expect(num_cache_entries_updated=0,
45
+ num_compiled_artifacts_saved=0),
46
+ # loading the model causes compilation (if enabled) to happen
47
+ vllm_runner('facebook/opt-125m',
48
+ compilation_config=compilation_config,
49
+ gpu_memory_utilization=0.4) as _):
50
+ pass
51
+
52
+
53
@pytest.mark.parametrize("enabled", [True, False])
54
def test_use_cudagraphs(vllm_runner, monkeypatch, enabled):
55
assert vllm.envs.VLLM_USE_V1
vllm/compilation/backends.py
+2
-1
@@ -183,9 +183,10 @@ class CompilerManager:
183
assert compiled_graph is not None, "Failed to compile the graph"
184
185
# store the artifact in the cache
186
- if handle is not None:
186
+ if not envs.VLLM_DISABLE_COMPILE_CACHE and handle is not None:
187
self.cache[(runtime_shape, graph_index,
188
self.compiler.name)] = handle
189
+ compilation_counter.num_cache_entries_updated += 1
190
self.is_cache_updated = True
191
if graph_index == 0:
192
# adds some info logging for the first graph
vllm/compilation/compiler_interface.py
+3
-1
@@ -213,7 +213,9 @@ class InductorStandaloneAdaptor(CompilerInterface):
213
# Save the compiled artifact to disk in the specified path
214
assert key is not None
215
path = os.path.join(self.cache_dir, key)
216
- compiled_graph.save(path=path, format="unpacked")
216
+ if not envs.VLLM_DISABLE_COMPILE_CACHE:
217
+ compiled_graph.save(path=path, format="unpacked")
218
+ compilation_counter.num_compiled_artifacts_saved += 1
219
return compiled_graph, (key, path)
220
221
def load(self,
vllm/compilation/counter.py
+4
@@ -23,6 +23,10 @@ class CompilationCounter:
23
num_inductor_compiles: int = 0
24
# EagerAdapter.compile calls
25
num_eager_compiles: int = 0
26
+ # The number of time vLLM's compiler cache entry was updated
27
+ num_cache_entries_updated: int = 0
28
+ # The number of standalone_compile compiled artifacts saved
29
+ num_compiled_artifacts_saved: int = 0
30
31
def clone(self) -> "CompilationCounter":
32
return copy.deepcopy(self)