A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[torch.compile] remove reset (#7975)
youkaichao committed
Aug 28, 2024 at 17:32 UTC
a7f65c2be93f491771aca31106f790bf381c0bad
3 files changed
+28
-15
tests/tpu/test_compilation.py
+28
-7
@@ -5,6 +5,10 @@ import tempfile
5
6
import depyf
7
8
+# disable custom dispatcher, let Dynamo takes over
9
+# all the control
10
+os.environ['VLLM_DYNAMO_USE_CUSTOM_DISPATCHER'] = "0"
11
+
12
temp_dir = tempfile.mkdtemp()
13
with depyf.prepare_debug(temp_dir):
14
cur_dir = os.path.dirname(__file__)
@@ -16,19 +20,36 @@ with depyf.prepare_debug(temp_dir):
20
21
compiled_code = sorted(
22
glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
19
-full_code = glob.glob(os.path.join(temp_dir, "full_code*.py"))[0]
23
+
24
# we should only trigger Dynamo compilation three times:
21
-# one for the profiling phase (and the compiled artifact will be discarded)
25
+# one for the profiling phase without kv cache
26
# one for the prefill phase with symbolic shapes
27
# one for the decode phase with symbolic shapes
28
# and later calls should not trigger Dynamo compilation again.
29
# NOTE: it might still trigger XLA compilation.
30
31
# check we have three compiled code
32
+# this is the assumption when we use the custom dispatcher
33
assert len(compiled_code) == 3
34
30
-# check the first compilation is discarded
31
-with open(full_code) as f:
32
- full_code_content = f.read()
33
- profile_function = compiled_code[0].split(".")[0]
34
- assert profile_function not in full_code_content
35
+# check all the compilations are as expected
36
+compiled_fn = sorted(
37
+ glob.glob(os.path.join(temp_dir, "__compiled_fn*Captured*.py")))
38
+
39
+# the first compilation is the profiling phase,
40
+# it should not have any kv cache
41
+with open(compiled_fn[0]) as f:
42
+ content = f.read()
43
+ assert "kv_caches" not in content
44
+
45
+# the second compilation is the prefill phase,
46
+# it should have kv cache and the flash_attention op
47
+with open(compiled_fn[1]) as f:
48
+ content = f.read()
49
+ assert "kv_caches" in content and "torch.ops.xla.flash_attention" in content
50
+
51
+# the third compilation is the decode phase,
52
+# it should have kv cache and the paged_attention op
53
+with open(compiled_fn[2]) as f:
54
+ content = f.read()
55
+ assert "kv_caches" in content and "torch.ops.xla.paged_attention" in content
vllm/worker/model_runner.py
-4
@@ -1123,10 +1123,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1123
device=self.device)
1124
self.execute_model(model_input, kv_caches, intermediate_tensors)
1125
torch.cuda.synchronize()
1126
-
1127
- # reset and discard the guard and compiled bytecode for profiling runs
1128
- torch._dynamo.reset()
1129
-
1126
return
1127
1128
def remove_all_loras(self):
vllm/worker/tpu_worker.py
-4
@@ -143,10 +143,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
143
num_cpu_blocks = int(self.cache_config.swap_space_bytes //
144
block_size_bytes)
145
num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
146
-
147
- # reset and discard the guard and compiled bytecode for profiling runs
148
- torch._dynamo.reset()
149
-
146
return num_tpu_blocks, num_cpu_blocks
147
148
def initialize_cache(