@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fbb5bd4cef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[TPU] Add example for profiling TPU inference (#12531)

Signed-off-by: mgoin <mgoin@redhat.com>

Michael Goin committed Jan 28, 2025 at 22:16 UTC fbb5bd4cefd62e3e389e2b873d5859eb8e07cbfa
2 files changed +168
examples/offline_inference/profiling_tpu/README.md new
+67
@@ -0,0 +1,67 @@
1 +# vLLM TPU Profiling
2 +
3 +This script is used to profile the TPU performance of vLLM for specific prefill or decode token shapes.
4 +
5 +Note: an actual running server is a mix of both prefill of many shapes and decode of many shapes.
6 +
7 +We assume you are on a TPU already (this was tested on TPU v6e) and have installed vLLM according to the [installation guide](https://docs.vllm.ai/en/latest/getting_started/installation/ai_accelerator/index.html).
8 +
9 +> In all examples below, we run several warmups before (so `--enforce-eager` is okay)
10 +
11 +## Profile Examples
12 +
13 +### Generate Prefill Trace
14 +
15 +This example runs Qwen/Qwen2.5-7B-Instruct with a single request of 1024 input tokens. This is set up in attempt to profile just the prefill time and operations.
16 +
17 +```bash
18 +export XLA_HLO_DEBUG=1
19 +export MODEL=Qwen/Qwen2.5-7B-Instruct
20 +export VLLM_TPU_PROFILE_DURATION_MS=3000
21 +export VLLM_TPU_PROFILE_DELAY_MS=0
22 +
23 +python3 profiling.py \
24 + --model $MODEL \
25 + --input-len 1024 --output-len 1 \
26 + --batch-size 1 --enforce-eager \
27 + --max-model-len 2048 \
28 + --tensor-parallel-size 1 \
29 + --profile-result-dir profiles
30 +```
31 +
32 +
33 +### Generate Decode Trace
34 +
35 +This example runs Llama 3.1 70B with a batch of 32 requests where each has 1 input token and 128 output tokens. This is set up in attempt to profile just the 32 decodes running in parallel by having an extremely small prefill of 1 token and setting `VLLM_TPU_PROFILE_DELAY_MS=1000` to skip the first second of inference (hopefully prefill).
36 +
37 +```bash
38 +export XLA_HLO_DEBUG=1
39 +export MODEL=meta-llama/Llama-3.1-70B-Instruct
40 +export VLLM_TPU_PROFILE_DURATION_MS=2000
41 +export VLLM_TPU_PROFILE_DELAY_MS=1000
42 +
43 +rm -rf ~/.cache/vllm/xla_cache
44 +python3 profiling.py \
45 + --model $MODEL \
46 + --input-len 1 \
47 + --output-len 128 \
48 + --batch-size 32 \
49 + --enforce-eager \
50 + --profile-result-dir profiles \
51 + --max-model-len 2048 --tensor-parallel-size 8
52 +```
53 +
54 +
55 +## Visualizing the profiles
56 +
57 +Once you have collected your profiles with this script, you can visualize them using [TensorBoard](https://cloud.google.com/tpu/docs/pytorch-xla-performance-profiling-tpu-vm).
58 +
59 +Here are most likely the dependencies you need to install:
60 +```bash
61 +pip install tensorflow-cpu tensorboard-plugin-profile etils importlib_resources
62 +```
63 +
64 +Then you just need to point TensorBoard to the directory where you saved the profiles and visit `http://localhost:6006/` in your browser:
65 +```bash
66 +tensorboard --logdir profiles/ --port 6006
67 +```
\ No newline at end of file
examples/offline_inference/profiling_tpu/profiling.py new
+101
@@ -0,0 +1,101 @@
1 +import argparse
2 +import dataclasses
3 +import os
4 +import time
5 +from typing import List
6 +
7 +import numpy as np
8 +import torch_xla.debug.profiler as xp
9 +from tqdm import tqdm
10 +
11 +from vllm import LLM, SamplingParams
12 +from vllm.engine.arg_utils import EngineArgs
13 +from vllm.inputs import PromptType
14 +from vllm.utils import FlexibleArgumentParser
15 +
16 +DURATION_MS = int(os.getenv("VLLM_TPU_PROFILE_DURATION_MS", 3000))
17 +DELAY_MS = int(os.getenv("VLLM_TPU_PROFILE_DELAY_MS", 0))
18 +
19 +
20 +def main(args: argparse.Namespace):
21 + print(args)
22 +
23 + engine_args = EngineArgs.from_cli_args(args)
24 + llm = LLM(**dataclasses.asdict(engine_args))
25 + _ = xp.start_server(9012)
26 +
27 + sampling_params = SamplingParams(
28 + temperature=0.0,
29 + ignore_eos=True,
30 + max_tokens=args.output_len,
31 + )
32 + print(sampling_params)
33 + dummy_prompt_token_ids = np.random.randint(10000,
34 + size=(args.batch_size,
35 + args.input_len))
36 + dummy_prompts: List[PromptType] = [{
37 + "prompt_token_ids": batch
38 + } for batch in dummy_prompt_token_ids.tolist()]
39 +
40 + def run_to_completion():
41 + start_time = time.perf_counter()
42 + llm.generate(dummy_prompts,
43 + sampling_params=sampling_params,
44 + use_tqdm=False)
45 + end_time = time.perf_counter()
46 + latency = end_time - start_time
47 + return latency
48 +
49 + # Warmup
50 + print("Warming up...")
51 + warmup_latencies = []
52 + for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
53 + warmup_latencies.append(run_to_completion())
54 + print(f"Average warmup latency: {np.mean(warmup_latencies):.4f}s")
55 +
56 + # Profile
57 + profile_dir = args.profile_result_dir
58 + print(f"Profiling (results will be saved to '{profile_dir}')...")
59 + # Enable tracing on server
60 + xp.trace_detached("localhost:9012",
61 + profile_dir,
62 + delay_ms=DELAY_MS,
63 + duration_ms=DURATION_MS)
64 + if DELAY_MS == 0:
65 + time.sleep(1.0)
66 + profile_latencies = []
67 + for _ in tqdm(range(args.num_iters), desc="Profile iterations"):
68 + profile_latencies.append(run_to_completion())
69 + print(f"Average profile latency: {np.mean(profile_latencies):.4f}s")
70 +
71 + return
72 +
73 +
74 +if __name__ == '__main__':
75 + parser = FlexibleArgumentParser(
76 + description='Benchmark the latency of processing a single batch of '
77 + 'requests till completion.')
78 + parser.add_argument('--input-len', type=int, default=32)
79 + parser.add_argument('--output-len', type=int, default=128)
80 + parser.add_argument('--batch-size', type=int, default=8)
81 + parser.add_argument('--num-iters-warmup',
82 + type=int,
83 + default=5,
84 + help='Number of iterations to run for warmup.')
85 + parser.add_argument('--num-iters',
86 + type=int,
87 + default=1,
88 + help='Number of iterations to run for profiling.')
89 + parser.add_argument(
90 + '--profile-result-dir',
91 + type=str,
92 + default="profiles",
93 + help=
94 + ('path to save the pytorch profiler output. Can be visualized '
95 + 'with ui.perfetto.dev or Tensorboard '
96 + '(https://cloud.google.com/tpu/docs/pytorch-xla-performance-profiling-tpu-vm).'
97 + ))
98 +
99 + parser = EngineArgs.add_cli_args(parser)
100 + args = parser.parse_args()
101 + main(args)