A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[core][torch.compile] discard the compile for profiling (#7796)
youkaichao committed
Aug 26, 2024 at 21:33 UTC
64cc64442546c829a28e6779e315b457edf76455
4 files changed
+43
-2
.buildkite/run-tpu-test.sh
+1
-2
@@ -12,5 +12,4 @@ remove_docker_container
12
# For HF_TOKEN.
13
source /etc/environment
14
# Run a simple end-to-end example.
15
-docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu \
16
- python3 /workspace/vllm/examples/offline_inference_tpu.py
15
+docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
tests/tpu/test_compilation.py
new
+34
@@ -0,0 +1,34 @@
1
+import glob
2
+import os
3
+import runpy
4
+import tempfile
5
+
6
+import depyf
7
+
8
+temp_dir = tempfile.mkdtemp()
9
+with depyf.prepare_debug(temp_dir):
10
+ cur_dir = os.path.dirname(__file__)
11
+ parent_dir = os.path.dirname(cur_dir)
12
+ root_dir = os.path.dirname(parent_dir)
13
+ example_file = os.path.join(root_dir, "examples",
14
+ "offline_inference_tpu.py")
15
+ runpy.run_path(example_file)
16
+
17
+compiled_code = sorted(
18
+ glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
19
+full_code = glob.glob(os.path.join(temp_dir, "full_code*.py"))[0]
20
+# we should only trigger Dynamo compilation three times:
21
+# one for the profiling phase (and the compiled artifact will be discarded)
22
+# one for the prefill phase with symbolic shapes
23
+# one for the decode phase with symbolic shapes
24
+# and later calls should not trigger Dynamo compilation again.
25
+# NOTE: it might still trigger XLA compilation.
26
+
27
+# check we have three compiled code
28
+assert len(compiled_code) == 3
29
+
30
+# check the first compilation is discarded
31
+with open(full_code) as f:
32
+ full_code_content = f.read()
33
+ profile_function = compiled_code[0].split(".")[0]
34
+ assert profile_function not in full_code_content
vllm/worker/model_runner.py
+4
@@ -1097,6 +1097,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1097
device=self.device)
1098
self.execute_model(model_input, kv_caches, intermediate_tensors)
1099
torch.cuda.synchronize()
1100
+
1101
+ # reset and discard the guard and compiled bytecode for profiling runs
1102
+ torch._dynamo.reset()
1103
+
1104
return
1105
1106
def remove_all_loras(self):
vllm/worker/tpu_worker.py
+4
@@ -143,6 +143,10 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
143
num_cpu_blocks = int(self.cache_config.swap_space_bytes //
144
block_size_bytes)
145
num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
146
+
147
+ # reset and discard the guard and compiled bytecode for profiling runs
148
+ torch._dynamo.reset()
149
+
150
return num_tpu_blocks, num_cpu_blocks
151
152
def initialize_cache(