A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[TPU] Add example for profiling TPU inference (#12531)
Signed-off-by: mgoin <mgoin@redhat.com>
Michael Goin committed
Jan 28, 2025 at 22:16 UTC
fbb5bd4cefd62e3e389e2b873d5859eb8e07cbfa
2 files changed
+168
examples/offline_inference/profiling_tpu/README.md
new
+67
@@ -0,0 +1,67 @@
1
+# vLLM TPU Profiling
2
+
3
+This script is used to profile the TPU performance of vLLM for specific prefill or decode token shapes.
4
+
5
+Note: an actual running server is a mix of both prefill of many shapes and decode of many shapes.
6
+
7
+We assume you are on a TPU already (this was tested on TPU v6e) and have installed vLLM according to the [installation guide](https://docs.vllm.ai/en/latest/getting_started/installation/ai_accelerator/index.html).
8
+
9
+> In all examples below, we run several warmups before (so `--enforce-eager` is okay)
10
+
11
+## Profile Examples
12
+
13
+### Generate Prefill Trace
14
+
15
+This example runs Qwen/Qwen2.5-7B-Instruct with a single request of 1024 input tokens. This is set up in attempt to profile just the prefill time and operations.
16
+
17
+```bash
18
+export XLA_HLO_DEBUG=1
19
+export MODEL=Qwen/Qwen2.5-7B-Instruct
20
+export VLLM_TPU_PROFILE_DURATION_MS=3000
21
+export VLLM_TPU_PROFILE_DELAY_MS=0
22
+
23
+python3 profiling.py \
24
+ --model $MODEL \
25
+ --input-len 1024 --output-len 1 \
26
+ --batch-size 1 --enforce-eager \
27
+ --max-model-len 2048 \
28
+ --tensor-parallel-size 1 \
29
+ --profile-result-dir profiles
30
+```
31
+
32
+
33
+### Generate Decode Trace
34
+
35
+This example runs Llama 3.1 70B with a batch of 32 requests where each has 1 input token and 128 output tokens. This is set up in attempt to profile just the 32 decodes running in parallel by having an extremely small prefill of 1 token and setting `VLLM_TPU_PROFILE_DELAY_MS=1000` to skip the first second of inference (hopefully prefill).
36
+
37
+```bash
38
+export XLA_HLO_DEBUG=1
39
+export MODEL=meta-llama/Llama-3.1-70B-Instruct
40
+export VLLM_TPU_PROFILE_DURATION_MS=2000
41
+export VLLM_TPU_PROFILE_DELAY_MS=1000
42
+
43
+rm -rf ~/.cache/vllm/xla_cache
44
+python3 profiling.py \
45
+ --model $MODEL \
46
+ --input-len 1 \
47
+ --output-len 128 \
48
+ --batch-size 32 \
49
+ --enforce-eager \
50
+ --profile-result-dir profiles \
51
+ --max-model-len 2048 --tensor-parallel-size 8
52
+```
53
+
54
+
55
+## Visualizing the profiles
56
+
57
+Once you have collected your profiles with this script, you can visualize them using [TensorBoard](https://cloud.google.com/tpu/docs/pytorch-xla-performance-profiling-tpu-vm).
58
+
59
+Here are most likely the dependencies you need to install:
60
+```bash
61
+pip install tensorflow-cpu tensorboard-plugin-profile etils importlib_resources
62
+```
63
+
64
+Then you just need to point TensorBoard to the directory where you saved the profiles and visit `http://localhost:6006/` in your browser:
65
+```bash
66
+tensorboard --logdir profiles/ --port 6006
67
+```
\ No newline at end of file
examples/offline_inference/profiling_tpu/profiling.py
new
+101
@@ -0,0 +1,101 @@
1
+import argparse
2
+import dataclasses
3
+import os
4
+import time
5
+from typing import List
6
+
7
+import numpy as np
8
+import torch_xla.debug.profiler as xp
9
+from tqdm import tqdm
10
+
11
+from vllm import LLM, SamplingParams
12
+from vllm.engine.arg_utils import EngineArgs
13
+from vllm.inputs import PromptType
14
+from vllm.utils import FlexibleArgumentParser
15
+
16
+DURATION_MS = int(os.getenv("VLLM_TPU_PROFILE_DURATION_MS", 3000))
17
+DELAY_MS = int(os.getenv("VLLM_TPU_PROFILE_DELAY_MS", 0))
18
+
19
+
20
+def main(args: argparse.Namespace):
21
+ print(args)
22
+
23
+ engine_args = EngineArgs.from_cli_args(args)
24
+ llm = LLM(**dataclasses.asdict(engine_args))
25
+ _ = xp.start_server(9012)
26
+
27
+ sampling_params = SamplingParams(
28
+ temperature=0.0,
29
+ ignore_eos=True,
30
+ max_tokens=args.output_len,
31
+ )
32
+ print(sampling_params)
33
+ dummy_prompt_token_ids = np.random.randint(10000,
34
+ size=(args.batch_size,
35
+ args.input_len))
36
+ dummy_prompts: List[PromptType] = [{
37
+ "prompt_token_ids": batch
38
+ } for batch in dummy_prompt_token_ids.tolist()]
39
+
40
+ def run_to_completion():
41
+ start_time = time.perf_counter()
42
+ llm.generate(dummy_prompts,
43
+ sampling_params=sampling_params,
44
+ use_tqdm=False)
45
+ end_time = time.perf_counter()
46
+ latency = end_time - start_time
47
+ return latency
48
+
49
+ # Warmup
50
+ print("Warming up...")
51
+ warmup_latencies = []
52
+ for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
53
+ warmup_latencies.append(run_to_completion())
54
+ print(f"Average warmup latency: {np.mean(warmup_latencies):.4f}s")
55
+
56
+ # Profile
57
+ profile_dir = args.profile_result_dir
58
+ print(f"Profiling (results will be saved to '{profile_dir}')...")
59
+ # Enable tracing on server
60
+ xp.trace_detached("localhost:9012",
61
+ profile_dir,
62
+ delay_ms=DELAY_MS,
63
+ duration_ms=DURATION_MS)
64
+ if DELAY_MS == 0:
65
+ time.sleep(1.0)
66
+ profile_latencies = []
67
+ for _ in tqdm(range(args.num_iters), desc="Profile iterations"):
68
+ profile_latencies.append(run_to_completion())
69
+ print(f"Average profile latency: {np.mean(profile_latencies):.4f}s")
70
+
71
+ return
72
+
73
+
74
+if __name__ == '__main__':
75
+ parser = FlexibleArgumentParser(
76
+ description='Benchmark the latency of processing a single batch of '
77
+ 'requests till completion.')
78
+ parser.add_argument('--input-len', type=int, default=32)
79
+ parser.add_argument('--output-len', type=int, default=128)
80
+ parser.add_argument('--batch-size', type=int, default=8)
81
+ parser.add_argument('--num-iters-warmup',
82
+ type=int,
83
+ default=5,
84
+ help='Number of iterations to run for warmup.')
85
+ parser.add_argument('--num-iters',
86
+ type=int,
87
+ default=1,
88
+ help='Number of iterations to run for profiling.')
89
+ parser.add_argument(
90
+ '--profile-result-dir',
91
+ type=str,
92
+ default="profiles",
93
+ help=
94
+ ('path to save the pytorch profiler output. Can be visualized '
95
+ 'with ui.perfetto.dev or Tensorboard '
96
+ '(https://cloud.google.com/tpu/docs/pytorch-xla-performance-profiling-tpu-vm).'
97
+ ))
98
+
99
+ parser = EngineArgs.add_cli_args(parser)
100
+ args = parser.parse_args()
101
+ main(args)