@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a7f65c2be9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[torch.compile] remove reset (#7975)

youkaichao committed Aug 28, 2024 at 17:32 UTC a7f65c2be93f491771aca31106f790bf381c0bad
3 files changed +28 -15
tests/tpu/test_compilation.py
+28 -7
@@ -5,6 +5,10 @@ import tempfile
5
6 import depyf
7
8 +# disable custom dispatcher, let Dynamo takes over
9 +# all the control
10 +os.environ['VLLM_DYNAMO_USE_CUSTOM_DISPATCHER'] = "0"
11 +
12 temp_dir = tempfile.mkdtemp()
13 with depyf.prepare_debug(temp_dir):
14 cur_dir = os.path.dirname(__file__)
@@ -16,19 +20,36 @@ with depyf.prepare_debug(temp_dir):
20
21 compiled_code = sorted(
22 glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
19 -full_code = glob.glob(os.path.join(temp_dir, "full_code*.py"))[0]
23 +
24 # we should only trigger Dynamo compilation three times:
21 -# one for the profiling phase (and the compiled artifact will be discarded)
25 +# one for the profiling phase without kv cache
26 # one for the prefill phase with symbolic shapes
27 # one for the decode phase with symbolic shapes
28 # and later calls should not trigger Dynamo compilation again.
29 # NOTE: it might still trigger XLA compilation.
30
31 # check we have three compiled code
32 +# this is the assumption when we use the custom dispatcher
33 assert len(compiled_code) == 3
34
30 -# check the first compilation is discarded
31 -with open(full_code) as f:
32 - full_code_content = f.read()
33 - profile_function = compiled_code[0].split(".")[0]
34 - assert profile_function not in full_code_content
35 +# check all the compilations are as expected
36 +compiled_fn = sorted(
37 + glob.glob(os.path.join(temp_dir, "__compiled_fn*Captured*.py")))
38 +
39 +# the first compilation is the profiling phase,
40 +# it should not have any kv cache
41 +with open(compiled_fn[0]) as f:
42 + content = f.read()
43 + assert "kv_caches" not in content
44 +
45 +# the second compilation is the prefill phase,
46 +# it should have kv cache and the flash_attention op
47 +with open(compiled_fn[1]) as f:
48 + content = f.read()
49 + assert "kv_caches" in content and "torch.ops.xla.flash_attention" in content
50 +
51 +# the third compilation is the decode phase,
52 +# it should have kv cache and the paged_attention op
53 +with open(compiled_fn[2]) as f:
54 + content = f.read()
55 + assert "kv_caches" in content and "torch.ops.xla.paged_attention" in content
vllm/worker/model_runner.py
-4
@@ -1123,10 +1123,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1123 device=self.device)
1124 self.execute_model(model_input, kv_caches, intermediate_tensors)
1125 torch.cuda.synchronize()
1126 -
1127 - # reset and discard the guard and compiled bytecode for profiling runs
1128 - torch._dynamo.reset()
1129 -
1126 return
1127
1128 def remove_all_loras(self):
vllm/worker/tpu_worker.py
-4
@@ -143,10 +143,6 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
143 num_cpu_blocks = int(self.cache_config.swap_space_bytes //
144 block_size_bytes)
145 num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
146 -
147 - # reset and discard the guard and compiled bytecode for profiling runs
148 - torch._dynamo.reset()
149 -
146 return num_tpu_blocks, num_cpu_blocks
147
148 def initialize_cache(