@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 64cc644425
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[core][torch.compile] discard the compile for profiling (#7796)

youkaichao committed Aug 26, 2024 at 21:33 UTC 64cc64442546c829a28e6779e315b457edf76455
4 files changed +43 -2
.buildkite/run-tpu-test.sh
+1 -2
@@ -12,5 +12,4 @@ remove_docker_container
12 # For HF_TOKEN.
13 source /etc/environment
14 # Run a simple end-to-end example.
15 -docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu \
16 - python3 /workspace/vllm/examples/offline_inference_tpu.py
15 +docker run --privileged --net host --shm-size=16G -it -e HF_TOKEN=$HF_TOKEN --name tpu-test vllm-tpu /bin/bash -c "python3 -m pip install git+https://github.com/thuml/depyf.git && python3 /workspace/vllm/tests/tpu/test_compilation.py && python3 /workspace/vllm/examples/offline_inference_tpu.py"
tests/tpu/test_compilation.py new
+34
@@ -0,0 +1,34 @@
1 +import glob
2 +import os
3 +import runpy
4 +import tempfile
5 +
6 +import depyf
7 +
8 +temp_dir = tempfile.mkdtemp()
9 +with depyf.prepare_debug(temp_dir):
10 + cur_dir = os.path.dirname(__file__)
11 + parent_dir = os.path.dirname(cur_dir)
12 + root_dir = os.path.dirname(parent_dir)
13 + example_file = os.path.join(root_dir, "examples",
14 + "offline_inference_tpu.py")
15 + runpy.run_path(example_file)
16 +
17 +compiled_code = sorted(
18 + glob.glob(os.path.join(temp_dir, "__transformed_code*.py")))
19 +full_code = glob.glob(os.path.join(temp_dir, "full_code*.py"))[0]
20 +# we should only trigger Dynamo compilation three times:
21 +# one for the profiling phase (and the compiled artifact will be discarded)
22 +# one for the prefill phase with symbolic shapes
23 +# one for the decode phase with symbolic shapes
24 +# and later calls should not trigger Dynamo compilation again.
25 +# NOTE: it might still trigger XLA compilation.
26 +
27 +# check we have three compiled code
28 +assert len(compiled_code) == 3
29 +
30 +# check the first compilation is discarded
31 +with open(full_code) as f:
32 + full_code_content = f.read()
33 + profile_function = compiled_code[0].split(".")[0]
34 + assert profile_function not in full_code_content
vllm/worker/model_runner.py
+4
@@ -1097,6 +1097,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1097 device=self.device)
1098 self.execute_model(model_input, kv_caches, intermediate_tensors)
1099 torch.cuda.synchronize()
1100 +
1101 + # reset and discard the guard and compiled bytecode for profiling runs
1102 + torch._dynamo.reset()
1103 +
1104 return
1105
1106 def remove_all_loras(self):
vllm/worker/tpu_worker.py
+4
@@ -143,6 +143,10 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
143 num_cpu_blocks = int(self.cache_config.swap_space_bytes //
144 block_size_bytes)
145 num_cpu_blocks = (num_cpu_blocks // 8) * 8 # Round down to 8.
146 +
147 + # reset and discard the guard and compiled bytecode for profiling runs
148 + torch._dynamo.reset()
149 +
150 return num_tpu_blocks, num_cpu_blocks
151
152 def initialize_cache(