A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[misc] CUDA Time Layerwise Profiler (#8337)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Lucas Wilkinson committed
Oct 17, 2024 at 10:36 UTC
9d30a056e7a1c81382a53ac63dc476c5fbe0091d
8 files changed
+1390
-4
.buildkite/test-pipeline.yaml
+1
@@ -184,6 +184,7 @@ steps:
184
- python3 offline_inference_vision_language_multi_image.py
185
- python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
186
- python3 offline_inference_encoder_decoder.py
187
+ - python3 offline_profile.py --model facebook/opt-125m
188
189
- label: Prefix Caching Test # 9min
190
#mirror_hardwares: [amd]
examples/offline_profile.py
new
+282
@@ -0,0 +1,282 @@
1
+import inspect
2
+import json
3
+import os
4
+import sys
5
+from argparse import RawTextHelpFormatter
6
+from dataclasses import asdict, dataclass
7
+from typing import Optional
8
+
9
+import torch
10
+
11
+from vllm import LLM, SamplingParams
12
+from vllm.engine.arg_utils import EngineArgs
13
+from vllm.profiler import layerwise_profile
14
+from vllm.utils import FlexibleArgumentParser
15
+
16
+BATCH_SIZE_DEFAULT = 1
17
+PROMPT_LEN_DEFAULT = 256
18
+OUTPUT_LEN_DEFAULT = 2
19
+
20
+
21
+@dataclass
22
+class ProfileContext:
23
+ engine_args: EngineArgs
24
+ prompt_len: int
25
+ output_len: int
26
+ batch_size: int
27
+ save_chrome_traces_folder: Optional[str]
28
+
29
+
30
+def get_dtype(dtype: str):
31
+ if dtype == "torch.float":
32
+ return torch.float
33
+ else:
34
+ return dtype
35
+
36
+
37
+def run_profile(context: ProfileContext, csv_output: Optional[str],
38
+ json_output: Optional[str]):
39
+ print("Run profile with:")
40
+ for key, value in asdict(context).items():
41
+ print(f" {key} = {value}")
42
+
43
+ # Create sampling params
44
+ sampling_params = SamplingParams(temperature=0.8,
45
+ top_p=0.95,
46
+ max_tokens=args.output_len,
47
+ ignore_eos=True)
48
+
49
+ # Create LLM
50
+ llm = LLM(**asdict(context.engine_args))
51
+ batch_size = context.batch_size
52
+ prompt_len = context.prompt_len
53
+ output_len = context.output_len
54
+
55
+ scheduler_config = llm.llm_engine.scheduler_config
56
+ max_model_len = llm.llm_engine.model_config.max_model_len
57
+ max_num_batched_tokens = scheduler_config.max_num_batched_tokens
58
+ max_num_seqs = scheduler_config.max_num_seqs
59
+
60
+ if batch_size * prompt_len > max_num_batched_tokens:
61
+ print(f"ERROR: chosen batch_size * prompt_len "
62
+ f"({batch_size} * {prompt_len} = {batch_size * prompt_len}) is "
63
+ f"larger than max_num_batched_tokens ({max_num_batched_tokens}) "
64
+ f"and therefore cannot be run in a single profile step, please "
65
+ f"choose a smaller batch size or prompt length, or increase "
66
+ f"--max-num-batched-tokens")
67
+ sys.exit(-1)
68
+ if batch_size >= max_num_seqs:
69
+ print(
70
+ f"ERROR: chosen batch_size ({batch_size}) is larger than "
71
+ f"max_num_seqs ({max_num_seqs}) and therefore cannot be run in a "
72
+ f"single profile step, please choose a smaller batch size")
73
+ sys.exit(-1)
74
+ print("llm.llm_engine.model_config.max_model_len: ",
75
+ llm.llm_engine.model_config.max_model_len)
76
+ if prompt_len + output_len > llm.llm_engine.model_config.max_model_len:
77
+ print(
78
+ f"ERROR: chosen prompt_len + output_len ({prompt_len} + "
79
+ f"{output_len} = {prompt_len + output_len}) is larger than the "
80
+ f"model's max_model_len ({max_model_len}), please choose a smaller "
81
+ f"prompt_len or output_len, or increase --max-model-len")
82
+ sys.exit(-1)
83
+
84
+ def add_requests():
85
+ for i in range(batch_size):
86
+ prompt_token_ids = torch.randint(
87
+ llm.llm_engine.model_config.get_vocab_size(),
88
+ size=(prompt_len, )).tolist()
89
+
90
+ llm.llm_engine.add_request(
91
+ request_id=f"seq{i}",
92
+ prompt={'prompt_token_ids': prompt_token_ids},
93
+ params=sampling_params)
94
+
95
+ def abort_requests():
96
+ for i in range(batch_size):
97
+ llm.llm_engine.abort_request(f"seq{i}")
98
+
99
+ # Warm up run
100
+ print("Warm up run ...")
101
+ add_requests()
102
+ llm.llm_engine.step() # Prefill
103
+ llm.llm_engine.step() # Decode
104
+ abort_requests()
105
+
106
+ print("Profile run ...")
107
+ add_requests()
108
+
109
+ with layerwise_profile() as prefill_prof:
110
+ llm.llm_engine.step() # First step is prefill
111
+
112
+ decode_profs = []
113
+ for x in range(args.output_len - 1):
114
+ with layerwise_profile() as decode_prof:
115
+ llm.llm_engine.step()
116
+ decode_profs.append(decode_prof)
117
+
118
+ decode_results_list = [prof.results for prof in decode_profs]
119
+ prefill_results = prefill_prof.results
120
+ has_decode = len(decode_results_list) > 0
121
+
122
+ LINE_WIDTH = 80
123
+ print("=" * LINE_WIDTH)
124
+ print(f"= Prefill Model Table "
125
+ f"(prompt_len={prompt_len}, batch_size={batch_size})")
126
+ print("=" * LINE_WIDTH)
127
+ print()
128
+ prefill_results.print_model_table()
129
+
130
+ if has_decode:
131
+ print()
132
+ print("=" * LINE_WIDTH)
133
+ print(f"= First Decode Step Model Table "
134
+ f"(prompt_len={prompt_len}, batch_size={batch_size})")
135
+ print("=" * LINE_WIDTH)
136
+ print()
137
+ decode_results_list[0].print_model_table()
138
+
139
+ print()
140
+ print("=" * LINE_WIDTH)
141
+ print(f"= Prefill Summary Table "
142
+ f"(prompt_len={prompt_len}, batch_size={batch_size})")
143
+ print("=" * LINE_WIDTH)
144
+ print()
145
+ prefill_results.print_summary_table()
146
+
147
+ if has_decode:
148
+ print()
149
+ print("=" * LINE_WIDTH)
150
+ print(f"= First Decode Step Summary Table "
151
+ f"(prompt_len={prompt_len}, batch_size={batch_size})")
152
+ print("=" * LINE_WIDTH)
153
+ print()
154
+ decode_results_list[0].print_summary_table()
155
+
156
+ if csv_output:
157
+ csv_filename_base = csv_output.rstrip(".csv")
158
+ prefill_results.export_model_stats_table_csv(
159
+ csv_filename_base + "_prefill_model_table.csv")
160
+ prefill_results.export_summary_stats_table_csv(
161
+ csv_filename_base + "_prefill_summary_table.csv")
162
+
163
+ if has_decode:
164
+ decode_results_list[0].export_model_stats_table_csv(\
165
+ csv_filename_base + "_decode_model_table.csv")
166
+ decode_results_list[0].export_summary_stats_table_csv(
167
+ csv_filename_base + "_decode_summary_table.csv")
168
+
169
+ if json_output:
170
+ cuda_devices = [
171
+ torch.cuda.get_device_properties(dev_idx)
172
+ for dev_idx in range(torch.cuda.device_count())
173
+ ]
174
+
175
+ json_dict = {
176
+ "context": {
177
+ "python_version": f"{sys.version}",
178
+ "torch_version": f"{torch.__version__}",
179
+ "torch_cuda_version": f"{torch.version.cuda}",
180
+ "cuda_devices": f"{cuda_devices}",
181
+ **asdict(context)
182
+ },
183
+ "prefill": prefill_results.convert_stats_to_dict(),
184
+ }
185
+
186
+ if has_decode:
187
+ for idx, dr in enumerate(decode_results_list):
188
+ json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
189
+
190
+ for idx, dr in enumerate(decode_results_list[1:]):
191
+ json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
192
+
193
+ with open(json_output.rstrip(".json") + ".json", "w+") as f:
194
+ json.dump(json_dict, f, indent=2)
195
+ pass
196
+
197
+ if context.save_chrome_traces_folder is not None:
198
+ os.makedirs(context.save_chrome_traces_folder, exist_ok=True)
199
+ prefill_prof.profiler.export_chrome_trace(
200
+ context.save_chrome_traces_folder + "/prefill.json")
201
+ for idx, decode_prof in enumerate(decode_profs):
202
+ decode_prof.profiler.export_chrome_trace(
203
+ context.save_chrome_traces_folder + f"/decode_{idx + 1}.json")
204
+ print("Traces saved as prefill.json and decode_1.json, etc."
205
+ f" in folder {context.save_chrome_traces_folder}")
206
+
207
+
208
+if __name__ == "__main__":
209
+ parser = FlexibleArgumentParser(description="""
210
+Profile a model
211
+
212
+ example:
213
+ ```
214
+ python examples/offline_profile.py \\
215
+ --model neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 --batch-size 4 \\
216
+ --prompt-len 512 --max-num-batched-tokens 8196 --json Llama31-8b-FP8 \\
217
+ --enforce-eager
218
+ ```
219
+
220
+ then you can use various tools to analyze the json output
221
+ terminal ascii tables:
222
+ ```
223
+ python tools/profiler/print_layerwise_table.py \\
224
+ --json-trace Llama31-8b-FP8.json --phase prefill --table summary
225
+ ```
226
+ or create matplotlib stacked bar charts:
227
+ ```
228
+ python tools/profiler/visualize_layerwise_profile.py \\
229
+ --json-trace Llama31-8b-FP8.json \\
230
+ --output-directory profile_breakdown --plot-metric pct_cuda_time
231
+ ```
232
+""",
233
+ formatter_class=RawTextHelpFormatter)
234
+ parser.add_argument(
235
+ "--csv",
236
+ type=str,
237
+ default=None,
238
+ help="Export the results as multiple csv file. This should be the root "
239
+ "filename, will create <filename>_prefill_model_table.csv, "
240
+ "<filename>_prefill_summary_table.csv, "
241
+ "<filename>_decode_model_table.csv, and "
242
+ "<filename>_decode_summary_table.csv")
243
+ parser.add_argument(
244
+ "--json",
245
+ type=str,
246
+ default=None,
247
+ help="Export the results as a json file. This should be the filename")
248
+ parser.add_argument("--save-chrome-traces-folder",
249
+ type=str,
250
+ help="Save chrome traces for the prefill and decode "
251
+ "will save traces as prefill.json and decode_1.json, "
252
+ "etc. inside this folder")
253
+ parser.add_argument(
254
+ "--prompt-len",
255
+ type=int,
256
+ default=PROMPT_LEN_DEFAULT,
257
+ help=f"Length of the random prompt to use when profiling, all batched "
258
+ f"requests use the same prompt_len, default={PROMPT_LEN_DEFAULT}")
259
+ parser.add_argument("--batch-size",
260
+ type=int,
261
+ default=BATCH_SIZE_DEFAULT,
262
+ help=f"Number of requests to run as a single batch, "
263
+ f"default={BATCH_SIZE_DEFAULT}")
264
+ parser.add_argument(
265
+ "--output-len",
266
+ type=int,
267
+ default=OUTPUT_LEN_DEFAULT,
268
+ help="Number of llm steps to run (includes prefill and decode) "
269
+ "- default={OUTPUT_LEN_DEFAULT}")
270
+
271
+ EngineArgs.add_cli_args(parser)
272
+
273
+ args = parser.parse_args()
274
+
275
+ context = ProfileContext(
276
+ engine_args=EngineArgs.from_cli_args(args),
277
+ **{
278
+ k: v
279
+ for k, v in vars(args).items()
280
+ if k in inspect.signature(ProfileContext).parameters
281
+ })
282
+ run_profile(context, csv_output=args.csv, json_output=args.json)
tools/profiler/print_layerwise_table.py
new
+77
@@ -0,0 +1,77 @@
1
+import argparse
2
+import json
3
+from typing import Dict
4
+
5
+from vllm.profiler.layerwise_profile import ModelStatsEntry, SummaryStatsEntry
6
+from vllm.profiler.utils import TablePrinter, indent_string
7
+
8
+
9
+def flatten_entries(entry_cls, profile_dict: Dict):
10
+ entries_and_depth = []
11
+
12
+ def get_entries(node, curr_depth=0):
13
+ entries_and_depth.append((entry_cls(**node["entry"]), curr_depth))
14
+
15
+ for child in node["children"]:
16
+ get_entries(
17
+ child,
18
+ curr_depth=curr_depth + 1,
19
+ )
20
+
21
+ for root in profile_dict:
22
+ get_entries(root)
23
+
24
+ return entries_and_depth
25
+
26
+
27
+if __name__ == "__main__":
28
+ parser = argparse.ArgumentParser()
29
+
30
+ parser.add_argument("--json-trace",
31
+ type=str,
32
+ required=True,
33
+ help="json trace file output by "
34
+ "examples/offline_profile.py")
35
+ parser.add_argument("--phase",
36
+ type=str,
37
+ choices=["prefill", "decode_1"],
38
+ required=True,
39
+ help="The phase to print the table for.")
40
+ parser.add_argument("--table",
41
+ type=str,
42
+ choices=["summary", "model"],
43
+ default="summary",
44
+ help="Which table to print, the summary table or the "
45
+ "layerwise model table")
46
+
47
+ args = parser.parse_args()
48
+
49
+ with open(args.json_trace, "r") as f:
50
+ profile_data = json.load(f)
51
+
52
+ if args.table == "summary":
53
+ entries_and_depths = flatten_entries(
54
+ SummaryStatsEntry, profile_data[args.phase]["summary_stats"])
55
+ column_widths = dict(name=80,
56
+ cuda_time_us=12,
57
+ pct_cuda_time=12,
58
+ invocations=15)
59
+ elif args.table == "model":
60
+ entries_and_depths = flatten_entries(
61
+ ModelStatsEntry, profile_data[args.phase]["model_stats"])
62
+ column_widths = dict(name=60,
63
+ cpu_time_us=12,
64
+ cuda_time_us=12,
65
+ pct_cuda_time=12,
66
+ trace=60)
67
+
68
+ # indent entry names based on the depth
69
+ entries = []
70
+ for entry, depth in entries_and_depths:
71
+ entry.name = indent_string(
72
+ entry.name,
73
+ indent=depth,
74
+ indent_style=lambda indent: "|" + "-" * indent + " ")
75
+ entries.append(entry)
76
+
77
+ TablePrinter(type(entries[0]), column_widths).print_table(entries)
tools/profiler/visualize_layerwise_profile.py
new
+522
@@ -0,0 +1,522 @@
1
+import argparse
2
+import copy
3
+import json
4
+import math
5
+import os
6
+from pathlib import Path
7
+from typing import Any, List, Optional, Tuple
8
+
9
+import matplotlib.pyplot as plt
10
+import pandas as pd
11
+
12
+## JSON parsing utils ####
13
+
14
+
15
+def largest_dist_from_leaf(node: dict, depth: int = 0):
16
+ if len(node["children"]) == 0:
17
+ return depth
18
+ return max([
19
+ largest_dist_from_leaf(child, depth=depth + 1)
20
+ for child in node["children"]
21
+ ])
22
+
23
+
24
+def get_entries_at_depth(depth: int,
25
+ entries_and_traces: List[Tuple[Any, Any]],
26
+ node: dict,
27
+ curr_depth: int = 0,
28
+ trace=()):
29
+ # assert that the query is at kernel or module level
30
+ assert depth == -1 or depth == -2
31
+
32
+ if curr_depth == 0 and largest_dist_from_leaf(node) <= (abs(depth) - 1):
33
+ # The tree is not tall enough!
34
+ entries_and_traces.append((node["entry"], trace))
35
+ return
36
+
37
+ if largest_dist_from_leaf(node) == (abs(depth) - 1):
38
+ entries_and_traces.append((node["entry"], trace))
39
+
40
+ trace = (node["entry"]["name"], ) + trace
41
+ for child in node["children"]:
42
+ get_entries_at_depth(depth,
43
+ entries_and_traces,
44
+ child,
45
+ curr_depth=curr_depth + 1,
46
+ trace=trace)
47
+
48
+
49
+def fold_nodes(root: dict, nodes_to_fold: List[str]):
50
+
51
+ stack: List[dict] = [root]
52
+ while len(stack) != 0:
53
+ node = stack.pop()
54
+ if node['entry']['name'] in nodes_to_fold:
55
+ node["children"] = []
56
+ continue
57
+ for child in node["children"]:
58
+ stack.append(child)
59
+ return root
60
+
61
+
62
+## Operation name cleanup utils ####
63
+
64
+
65
+def trim_string_back(string: str, width: int) -> str:
66
+ if len(string) > width:
67
+ offset = len(string) - width + 3
68
+ string = string[:-offset]
69
+ if len(string) > 3:
70
+ string = string + "..."
71
+ return string
72
+
73
+
74
+def shorten_plot_legend_strings(legend, max_char_len: int):
75
+ for t in legend.get_texts():
76
+ t.set_text(
77
+ trim_string_back(abbreviate_known_names(t.get_text()),
78
+ max_char_len))
79
+
80
+
81
+def abbreviate_known_names(name: str) -> str:
82
+ abbreviations = {
83
+ "MergedColumnParallelLinear": "MCPLinear",
84
+ "QKVParallelLinear": "QKVPLinear",
85
+ "RowParallelLinear": "RPLinear",
86
+ "weight=": "w=",
87
+ "bfloat16": "bf16",
88
+ "float16": "f16",
89
+ }
90
+ for key, value in abbreviations.items():
91
+ name = name.replace(key, value)
92
+ return name
93
+
94
+
95
+def attempt_to_make_names_unique(entries_and_traces):
96
+ names, non_unique_names = (set(), set())
97
+
98
+ def all_the_same(items) -> bool:
99
+ return all(i == items[0] for i in items)
100
+
101
+ for entry, _ in entries_and_traces:
102
+ if entry["name"] in names:
103
+ non_unique_names.add(entry["name"])
104
+ else:
105
+ names.add(entry["name"])
106
+
107
+ for name in non_unique_names:
108
+ entries_and_traces_with_name = [(entry, trace)
109
+ for entry, trace in entries_and_traces
110
+ if entry["name"] == name]
111
+
112
+ zipped_traces = list(
113
+ zip(*[trace for _, trace in entries_and_traces_with_name]))
114
+ first_trace_difference = next(
115
+ (i for i, trace_eles in enumerate(zipped_traces)
116
+ if not all_the_same(trace_eles)), None)
117
+
118
+ if first_trace_difference is None:
119
+ # can't create a unique name, leave them names as the
120
+ # are they will get aggregated by the pivot_table call
121
+ continue
122
+
123
+ for entry, trace in entries_and_traces_with_name:
124
+ entry["name"] = " <- ".join((entry["name"], ) +
125
+ trace[:first_trace_difference + 1])
126
+
127
+
128
+## Operation grouping utils ####
129
+'''
130
+ Group operations in the given dataframe by some high-level ops like,
131
+ - gemms
132
+ - attention
133
+ - rms_norm
134
+ etc.
135
+'''
136
+
137
+
138
+def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
139
+
140
+ def is_rms_norm(op_name: str):
141
+ if "rms_norm_kernel" in op_name:
142
+ return True
143
+
144
+ def is_attention_block(op_name: str):
145
+ if "flash_fwd" in op_name or \
146
+ "reshape_and_cache_flash_kernel" in op_name:
147
+ return True
148
+
149
+ def is_quant(op_name: str):
150
+ if "scaled_fp8_quant" in op_name or \
151
+ "scaled_int8_quant" in op_name:
152
+ return True
153
+
154
+ def is_gemm_op(op_name: str):
155
+ if is_quant(op_name):
156
+ return False
157
+ if "xmma_gemm" in op_name or \
158
+ "gemv2T_kernel" in op_name or \
159
+ "splitKreduce" in op_name or \
160
+ "void cutlass::Kernel" in op_name or \
161
+ "void cutlass::device_kernel" in op_name or \
162
+ "s16816gemm" in op_name:
163
+ return True
164
+
165
+ def is_elementwise_op(op_name: str):
166
+ return "elementwise_kernel" in op_name
167
+
168
+ def is_mem_op(op_name: str):
169
+ return "memcpy" in op_name.lower() or \
170
+ "memset" in op_name.lower()
171
+
172
+ def is_vocab_embedding_op(op_name: str):
173
+ return "vocabparallelembed" in op_name.lower()
174
+
175
+ # nccl ops
176
+ def is_nccl_op(op_name: str):
177
+ return "nccl" in op_name.lower()
178
+
179
+ def is_nccl_all_reduce(op_name: str):
180
+ return is_nccl_op(op_name) and \
181
+ ("all_reduce" in op_name.lower() or \
182
+ "allreduce" in op_name.lower())
183
+
184
+ def is_nccl_gather(op_name: str):
185
+ return is_nccl_op(op_name) and \
186
+ "gather" in op_name.lower()
187
+
188
+ def is_nccl_broadcast(op_name: str):
189
+ return is_nccl_op(op_name) and \
190
+ "broadcast" in op_name.lower()
191
+
192
+ # Reduce ops types
193
+ def is_cross_device_reduce_1stage(op_name: str):
194
+ return "cross_device_reduce_1stage" in op_name
195
+
196
+ def is_cross_device_reduce_2stage(op_name: str):
197
+ return "cross_device_reduce_2stage" in op_name
198
+
199
+ def is_custom_ar_all_reduce_unreg(op_name: str):
200
+ return "_C_custom_ar::all_reduce_unreg" in op_name
201
+
202
+ def is_reduce_kernel(op_name: str):
203
+ return "reduce_kernel" in op_name
204
+
205
+ headers = list(trace_df)
206
+ ops = copy.deepcopy(headers)
207
+
208
+ attention_ops = list(filter(lambda x: is_attention_block(x), ops))
209
+ ops = list(filter(lambda x: x not in attention_ops, ops))
210
+
211
+ quant_ops = list(filter(lambda x: is_quant(x), ops))
212
+ ops = list(filter(lambda x: x not in quant_ops, ops))
213
+
214
+ gemm_ops = list(filter(lambda x: is_gemm_op(x), ops))
215
+ ops = list(filter(lambda x: x not in gemm_ops, ops))
216
+
217
+ rms_norm_ops = list(filter(lambda x: is_rms_norm(x), ops))
218
+ ops = list(filter(lambda x: x not in rms_norm_ops, ops))
219
+
220
+ vocab_embed_ops = list(filter(lambda x: is_vocab_embedding_op(x), ops))
221
+ ops = list(filter(lambda x: x not in vocab_embed_ops, ops))
222
+
223
+ mem_ops = list(filter(lambda x: is_mem_op(x), ops))
224
+ ops = list(filter(lambda x: x not in mem_ops, ops))
225
+
226
+ elementwise_ops = list(filter(lambda x: is_elementwise_op(x), ops))
227
+ ops = list(filter(lambda x: x not in elementwise_ops, ops))
228
+
229
+ nccl_all_reduce_ops = list(filter(lambda x: is_nccl_all_reduce(x), ops))
230
+ ops = list(filter(lambda x: x not in nccl_all_reduce_ops, ops))
231
+
232
+ nccl_gather_ops = list(filter(lambda x: is_nccl_gather(x), ops))
233
+ ops = list(filter(lambda x: x not in nccl_gather_ops, ops))
234
+
235
+ nccl_broadcast_ops = list(filter(lambda x: is_nccl_broadcast(x), ops))
236
+ ops = list(filter(lambda x: x not in nccl_broadcast_ops, ops))
237
+
238
+ nccl_other_ops = list(filter(lambda x: is_nccl_op(x), ops))
239
+ ops = list(filter(lambda x: x not in nccl_other_ops, ops))
240
+
241
+ cross_device_reduce_1stage_ops = list(
242
+ filter(lambda x: is_cross_device_reduce_1stage(x), ops))
243
+ ops = list(filter(lambda x: x not in cross_device_reduce_1stage_ops, ops))
244
+
245
+ cross_device_reduce_2stage_ops = list(
246
+ filter(lambda x: is_cross_device_reduce_2stage(x), ops))
247
+ ops = list(filter(lambda x: x not in cross_device_reduce_2stage_ops, ops))
248
+
249
+ custom_ar_all_reduce_unreg_ops = list(
250
+ filter(lambda x: is_custom_ar_all_reduce_unreg(x), ops))
251
+ ops = list(filter(lambda x: x not in custom_ar_all_reduce_unreg_ops, ops))
252
+
253
+ reduce_kernel_ops = list(filter(lambda x: is_reduce_kernel(x), ops))
254
+ ops = list(filter(lambda x: x not in reduce_kernel_ops, ops))
255
+
256
+ if len(attention_ops):
257
+ trace_df['attention'] = trace_df[attention_ops].agg("sum", axis=1)
258
+ if len(quant_ops):
259
+ trace_df['quant_ops'] = trace_df[quant_ops].agg("sum", axis=1)
260
+ if len(gemm_ops):
261
+ trace_df['gemm_ops'] = trace_df[gemm_ops].agg("sum", axis=1)
262
+ if len(rms_norm_ops):
263
+ trace_df['rms_norm_ops'] = trace_df[rms_norm_ops].agg("sum", axis=1)
264
+ if len(vocab_embed_ops):
265
+ trace_df['vocab_embed_ops'] = trace_df[vocab_embed_ops].agg("sum",
266
+ axis=1)
267
+ if len(mem_ops):
268
+ trace_df['mem_ops'] = trace_df[mem_ops].agg("sum", axis=1)
269
+ if len(elementwise_ops):
270
+ trace_df['elementwise_ops'] = trace_df[elementwise_ops].agg("sum",
271
+ axis=1)
272
+
273
+ if len(nccl_all_reduce_ops):
274
+ trace_df['nccl_all_reduce_ops'] = trace_df[nccl_all_reduce_ops].agg(
275
+ "sum", axis=1)
276
+ if len(nccl_gather_ops):
277
+ trace_df['nccl_gather_ops'] = trace_df[nccl_gather_ops].agg("sum",
278
+ axis=1)
279
+ if len(nccl_broadcast_ops):
280
+ trace_df['nccl_broadcast_ops'] = trace_df[nccl_broadcast_ops].agg(
281
+ "sum", axis=1)
282
+ if len(nccl_other_ops):
283
+ trace_df['nccl_other_ops'] = trace_df[nccl_other_ops].agg("sum",
284
+ axis=1)
285
+
286
+ if len(cross_device_reduce_1stage_ops):
287
+ trace_df['cross_device_reduce_1stage_ops'] = trace_df[
288
+ cross_device_reduce_1stage_ops].agg("sum", axis=1)
289
+ if len(cross_device_reduce_2stage_ops):
290
+ trace_df['cross_device_reduce_2stage_ops'] = trace_df[
291
+ cross_device_reduce_2stage_ops].agg("sum", axis=1)
292
+ if len(custom_ar_all_reduce_unreg_ops):
293
+ trace_df['custom_ar_all_reduce_unreg_ops'] = trace_df[
294
+ custom_ar_all_reduce_unreg_ops].agg("sum", axis=1)
295
+ if len(reduce_kernel_ops):
296
+ trace_df['reduce_kernel_ops'] = trace_df[reduce_kernel_ops].agg("sum",
297
+ axis=1)
298
+
299
+ trace_df.drop(
300
+ attention_ops + quant_ops + gemm_ops + rms_norm_ops + vocab_embed_ops +
301
+ mem_ops + elementwise_ops + nccl_all_reduce_ops + nccl_gather_ops +
302
+ nccl_broadcast_ops + nccl_other_ops + cross_device_reduce_1stage_ops +
303
+ cross_device_reduce_2stage_ops + custom_ar_all_reduce_unreg_ops +
304
+ reduce_kernel_ops,
305
+ axis=1,
306
+ inplace=True)
307
+ return trace_df
308
+
309
+
310
+## Data plotting utils ####
311
+
312
+
313
+def plot_trace_df(traces_df: pd.DataFrame,
314
+ plot_metric: str,
315
+ plot_title: str,
316
+ output: Optional[Path] = None):
317
+
318
+ phases = traces_df['phase'].unique()
319
+ traces_df = traces_df.pivot_table(index="phase",
320
+ columns="name",
321
+ values=plot_metric,
322
+ aggfunc="sum")
323
+
324
+ traces_df = group_trace_by_operations(traces_df)
325
+
326
+ # Make the figure
327
+ fig, ax = plt.subplots(1, figsize=(5, 8), sharex=True)
328
+
329
+ # Draw the stacked bars
330
+ ops = list(traces_df)
331
+ bottom = [0] * len(phases)
332
+ for op in ops:
333
+ values = [traces_df[op][phase] for phase in phases]
334
+ values = list(map(lambda x: 0.0 if math.isnan(x) else x, values))
335
+ ax.bar(phases, values, label=op, bottom=bottom)
336
+ bottom = [bottom[j] + values[j] for j in range(len(phases))]
337
+
338
+ # Write the values as text on the bars
339
+ for bar in ax.patches:
340
+ if bar.get_height() != 0:
341
+ ax.text(bar.get_x() + bar.get_width() / 2,
342
+ bar.get_height() / 2 + bar.get_y(),
343
+ f"{round(bar.get_height(), 2)}",
344
+ ha='center',
345
+ color='w',
346
+ weight='bold',
347
+ size=5)
348
+
349
+ # Setup legend
350
+ handles, labels = plt.gca().get_legend_handles_labels()
351
+ legend = fig.legend(handles,
352
+ labels,
353
+ loc='center left',
354
+ bbox_to_anchor=(1, 1))
355
+ shorten_plot_legend_strings(legend, 50)
356
+
357
+ # Setup labels and title
358
+ plt.setp(ax.get_xticklabels(), rotation=90)
359
+ ax.set_ylabel(plot_metric)
360
+ plt.suptitle(plot_title)
361
+
362
+ plt.savefig(output, bbox_inches='tight')
363
+ print("Created: ", output)
364
+
365
+
366
+def main(
367
+ json_trace: Path,
368
+ output_directory: Path,
369
+ depth: int, # Fetch/Plot operations at this depth of the Json tree
370
+ plot_metric: str,
371
+ make_names_unique: bool,
372
+ top_k: int,
373
+ json_nodes_to_fold: List[str]):
374
+
375
+ def prepare_data(profile_json: dict, step_keys: List[str]) -> pd.DataFrame:
376
+
377
+ def get_entries_and_traces(key: str):
378
+ entries_and_traces: List[Tuple[Any, Any]] = []
379
+ for root in profile_json[key]["summary_stats"]:
380
+ # Fold nodes in the traces as per user request. i.e. simply
381
+ # make the requested nodes leaf-nodes.
382
+ root = fold_nodes(root, json_nodes_to_fold)
383
+ get_entries_at_depth(depth, entries_and_traces, root)
384
+ return entries_and_traces
385
+
386
+ def keep_only_top_entries(df: pd.DataFrame,
387
+ metric: str,
388
+ top_k: int = 9) -> pd.DataFrame:
389
+ df.loc[df.nsmallest(len(df) - top_k + 1, metric).index,
390
+ ["name"]] = "others"
391
+ return df
392
+
393
+ # Get data for each key
394
+ traces = list(map(lambda x: get_entries_and_traces(x), step_keys))
395
+
396
+ # Attempt some cleanup
397
+ if make_names_unique:
398
+ for trace in traces:
399
+ attempt_to_make_names_unique(trace)
400
+
401
+ # To pandas dataframe
402
+ trace_dfs = list(
403
+ map(lambda t: pd.DataFrame([entry for entry, _ in t]).fillna(0),
404
+ traces))
405
+
406
+ # Respect top_k
407
+ if top_k:
408
+ trace_dfs = list(
409
+ map(
410
+ lambda trace_df: keep_only_top_entries(
411
+ trace_df, "cuda_time_us", top_k), trace_dfs))
412
+
413
+ # Fill in information about the step-keys
414
+ for trace_df, step_key in zip(trace_dfs, step_keys):
415
+ trace_df['phase'] = step_key
416
+
417
+ # Combine all data frames so they can be put in a single plot
418
+ traces_df = pd.concat(trace_dfs)
419
+
420
+ # Add a derived metric `cuda_time_ms`
421
+ traces_df["cuda_time_ms"] = traces_df["cuda_time_us"] / 1000
422
+ traces_df = traces_df.fillna(0)
423
+
424
+ return traces_df
425
+
426
+ def make_plot_title_suffix(profile_json: dict) -> str:
427
+ context = profile_json["context"]
428
+ sparsity = context.get('sparsity', None)
429
+ return (f"{context['model']}\n"
430
+ f"Batch={context['batch_size']}, "
431
+ f"PromptLen={context['prompt_len']}, "
432
+ f"OutputLen={context['output_len']},"
433
+ f"NumGpus={context['tensor_parallel_size']}"
434
+ f"{', Sparsity ' + sparsity if sparsity else ''}")
435
+
436
+ profile_json = None
437
+ with open(json_trace, "r") as f:
438
+ profile_json = json.load(f)
439
+ assert profile_json is not None
440
+
441
+ # Get all `llm.generate.step()` profile
442
+ step_traces = list(profile_json.keys())
443
+ assert (step_traces[0] == 'context')
444
+ step_traces = step_traces[1:] # have only prefill and decodes
445
+ prefills = list(filter(lambda x: "prefill" in x, step_traces))
446
+ all_decodes = list(filter(lambda x: "decode" in x, step_traces))
447
+ assert len(prefills) + len(all_decodes) == len(step_traces)
448
+ assert len(prefills) == 1
449
+
450
+ decodes = all_decodes[::args.step_plot_interval]
451
+ if decodes[-1] != all_decodes[-1]:
452
+ # Always have the last decode
453
+ decodes.append(all_decodes[-1])
454
+
455
+ prefill_traces = prepare_data(profile_json, prefills)
456
+ decode_traces = prepare_data(profile_json, decodes)
457
+
458
+ plot_title_suffix = make_plot_title_suffix(profile_json)
459
+
460
+ plot_trace_df(prefill_traces, plot_metric, "prefill " + plot_title_suffix,
461
+ output_directory / Path("prefill.png"))
462
+ plot_trace_df(decode_traces, plot_metric, "decodes " + plot_title_suffix,
463
+ output_directory / Path("decode_steps.png"))
464
+
465
+
466
+if __name__ == "__main__":
467
+ parser = argparse.ArgumentParser()
468
+
469
+ parser.add_argument(
470
+ "--json-trace",
471
+ type=str,
472
+ required=True,
473
+ help="json trace file output by examples/offline_profile.py")
474
+ parser.add_argument("--output-directory",
475
+ type=str,
476
+ required=False,
477
+ help="Directory to output plots")
478
+ parser.add_argument("--level",
479
+ type=str,
480
+ default="module",
481
+ choices=["module", "kernel"])
482
+ parser.add_argument("--top-k",
483
+ type=int,
484
+ default=12,
485
+ help="Only graph the top `top_k` entries by time.")
486
+ parser.add_argument("--fold-json-node",
487
+ nargs='+',
488
+ default=['Sampler', 'LogitsProcessor'],
489
+ help='Do not plot the children of these nodes. Let, \
490
+ the node represent the aggregate of all its \
491
+ children')
492
+ parser.add_argument("--plot-metric",
493
+ type=str,
494
+ default="cuda_time_ms",
495
+ help='Metric to plot. some options are cuda_time_ms, \
496
+ pct_cuda_time')
497
+ parser.add_argument(
498
+ "--step-plot-interval",
499
+ type=int,
500
+ default=4,
501
+ help="For every `step_plot_interval` steps, plot 1 step")
502
+
503
+ args = parser.parse_args()
504
+
505
+ # Prepare/Extract relevant args
506
+ make_names_unique = False
507
+ if args.level == "module":
508
+ depth = -2
509
+ make_names_unique = True
510
+ elif args.level == "kernel":
511
+ depth = -1
512
+ else:
513
+ raise Exception(f"Unexpected level value ({args.level})")
514
+
515
+ output_directory = args.output_directory if args.output_directory else Path(
516
+ args.json_trace).parent
517
+
518
+ if not os.path.exists(output_directory):
519
+ os.makedirs(output_directory)
520
+
521
+ main(Path(args.json_trace), output_directory, depth, args.plot_metric,
522
+ make_names_unique, args.top_k, args.fold_json_node)
vllm/profiler/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from .layerwise_profile import layerwise_profile
2
+
3
+__all__ = [
4
+ "layerwise_profile",
5
+]
vllm/profiler/layerwise_profile.py
new
+354
@@ -0,0 +1,354 @@
1
+import copy
2
+from collections import defaultdict
3
+from dataclasses import asdict, dataclass, field
4
+from typing import Callable, Dict, List, Optional, Tuple, TypeAlias, Union
5
+
6
+import pandas as pd
7
+from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
8
+from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent
9
+from torch.autograd.profiler import FunctionEvent
10
+from torch.profiler import ProfilerActivity, profile
11
+
12
+from vllm.profiler.utils import (TablePrinter, event_has_module,
13
+ event_is_torch_op, event_module_repr,
14
+ event_torch_op_stack_trace, indent_string)
15
+
16
+
17
+@dataclass
18
+class _ModuleTreeNode:
19
+ event: _ProfilerEvent
20
+ parent: Optional['_ModuleTreeNode'] = None
21
+ children: List['_ModuleTreeNode'] = field(default_factory=list)
22
+ trace: str = ""
23
+
24
+ @property
25
+ def is_leaf(self):
26
+ return (self.event.children is None or len(self.event.children) == 0)
27
+
28
+ @property
29
+ def is_torch_op(self):
30
+ return event_is_torch_op(self.event)
31
+
32
+ @property
33
+ def is_cuda(self):
34
+ return (self.event.tag == _EventType.Kineto
35
+ and self.event.typed[1].device_type == DeviceType.CUDA)
36
+
37
+
38
+@dataclass
39
+class SummaryStatsEntry:
40
+ name: str
41
+ cuda_time_us: float
42
+ pct_cuda_time: float
43
+ invocations: int
44
+
45
+
46
+@dataclass
47
+class ModelStatsEntry:
48
+ name: str
49
+ cpu_time_us: float
50
+ cuda_time_us: float
51
+ pct_cuda_time: float
52
+ trace: str
53
+
54
+
55
+StatsEntry: TypeAlias = Union[ModelStatsEntry, SummaryStatsEntry]
56
+
57
+
58
+@dataclass
59
+class _StatsTreeNode:
60
+ entry: StatsEntry
61
+ children: List[StatsEntry]
62
+ parent: Optional[StatsEntry]
63
+
64
+
65
+@dataclass
66
+class LayerwiseProfileResults(profile):
67
+ _kineto_results: _ProfilerResult
68
+ _kineto_event_correlation_map: Dict[int,
69
+ List[_KinetoEvent]] = field(init=False)
70
+ _event_correlation_map: Dict[int, List[FunctionEvent]] = field(init=False)
71
+ _module_tree: List[_ModuleTreeNode] = field(init=False)
72
+ _model_stats_tree: List[_StatsTreeNode] = field(init=False)
73
+ _summary_stats_tree: List[_StatsTreeNode] = field(init=False)
74
+
75
+ def __post_init__(self):
76
+ self._build_correlation_map()
77
+ self._build_module_tree()
78
+ self._build_stats_trees()
79
+
80
+ def print_model_table(self, column_widths: Dict[str, int] = None):
81
+ _column_widths = dict(name=60,
82
+ cpu_time_us=12,
83
+ cuda_time_us=12,
84
+ pct_cuda_time=12,
85
+ trace=60)
86
+ if column_widths:
87
+ _column_widths.update(**column_widths)
88
+ filtered_model_table = [
89
+ (depth, row)
90
+ for depth, row in self._flatten_stats_tree(self._model_stats_tree)
91
+ if row.cuda_time_us > 0 or row.cpu_time_us > 0
92
+ ]
93
+ TablePrinter(ModelStatsEntry, _column_widths).print_table(
94
+ self._indent_row_names_based_on_depth(
95
+ filtered_model_table,
96
+ indent_style=lambda indent: "|" + "-" * indent + " "))
97
+
98
+ def print_summary_table(self, column_widths: Dict[str, int] = None):
99
+ _column_widths = dict(name=80,
100
+ cuda_time_us=12,
101
+ pct_cuda_time=12,
102
+ invocations=15)
103
+ if column_widths:
104
+ _column_widths.update(**column_widths)
105
+ filtered_summary_table = [(depth, row)
106
+ for depth, row in self._flatten_stats_tree(
107
+ self._summary_stats_tree)
108
+ if row.cuda_time_us > 0]
109
+ TablePrinter(SummaryStatsEntry, _column_widths).print_table(
110
+ self._indent_row_names_based_on_depth(
111
+ filtered_summary_table,
112
+ indent_style=lambda indent: "|" + "-" * indent + " "))
113
+
114
+ def export_model_stats_table_csv(self, filename: str):
115
+ df = pd.DataFrame([
116
+ asdict(row)
117
+ for _, row in self._flatten_stats_tree(self._model_stats_tree)
118
+ ])
119
+ df.to_csv(filename)
120
+
121
+ def export_summary_stats_table_csv(self, filename: str):
122
+ df = pd.DataFrame([
123
+ asdict(row)
124
+ for _, row in self._flatten_stats_tree(self._summary_stats_tree)
125
+ ])
126
+ df.to_csv(filename)
127
+
128
+ def convert_stats_to_dict(self) -> str:
129
+ return {
130
+ "summary_stats":
131
+ self._convert_stats_tree_to_dict(self._summary_stats_tree),
132
+ "model_stats":
133
+ self._convert_stats_tree_to_dict(self._model_stats_tree)
134
+ }
135
+
136
+ @staticmethod
137
+ def _indent_row_names_based_on_depth(depths_rows: List[Tuple[int,
138
+ StatsEntry]],
139
+ indent_style: Union[Callable[[int],
140
+ str],
141
+ str] = " "):
142
+ indented_rows = []
143
+ for depth, row in depths_rows:
144
+ if row.cuda_time_us == 0:
145
+ continue
146
+ indented_row = copy.deepcopy(row)
147
+ indented_row.name = indent_string(indented_row.name, depth,
148
+ indent_style)
149
+ indented_rows.append(indented_row)
150
+ return indented_rows
151
+
152
+ def _build_correlation_map(self):
153
+ self._kineto_event_correlation_map = defaultdict(list)
154
+ for event in self._kineto_results.events():
155
+ self._kineto_event_correlation_map[event.correlation_id()].append(
156
+ event)
157
+
158
+ def _build_module_tree(self):
159
+ self._module_tree = []
160
+ event_tree = self._kineto_results.experimental_event_tree()
161
+
162
+ def _df_traversal(event: _ProfilerEvent,
163
+ curr_node: Optional[_ModuleTreeNode] = None):
164
+
165
+ # For the tensor parallel case for now only look at task 1
166
+ if event.start_tid != 1:
167
+ return
168
+
169
+ if event_has_module(event):
170
+ node = _ModuleTreeNode(event=event, parent=curr_node)
171
+ if curr_node:
172
+ curr_node.children.append(node)
173
+ else:
174
+ self._module_tree.append(node)
175
+ curr_node = node
176
+
177
+ is_leaf = (event.children is None or len(event.children) == 0)
178
+ if is_leaf and curr_node:
179
+ node = _ModuleTreeNode(
180
+ event=event,
181
+ parent=curr_node,
182
+ trace=event_torch_op_stack_trace(
183
+ event, until=lambda x: event_has_module(x)))
184
+ curr_node.children.append(node)
185
+ curr_node = node
186
+
187
+ for child in event.children:
188
+ _df_traversal(child, curr_node)
189
+
190
+ for root in event_tree:
191
+ _df_traversal(root)
192
+
193
+ def _get_kineto_gpu_event(self, node: _ModuleTreeNode):
194
+ if node.event.tag != _EventType.Kineto:
195
+ return None
196
+ correlated_kineto_events = self._kineto_event_correlation_map.get(
197
+ node.event.correlation_id, [])
198
+ iterator = (x for x in correlated_kineto_events
199
+ if x.device_type() == DeviceType.CUDA
200
+ and x.name() == node.event.name)
201
+ return next(iterator, None)
202
+
203
+ def _cumulative_cuda_time(self, node: _ModuleTreeNode):
204
+ 'Return cuda time in microseconds'
205
+
206
+ def _cumulative_cuda_time_recursive(node: _ModuleTreeNode):
207
+ if node.is_leaf and (gpu_kineto_event :=
208
+ self._get_kineto_gpu_event(node)):
209
+ return gpu_kineto_event.duration_ns() / 1000.0
210
+ else:
211
+ cumulative_cuda_time = 0
212
+ for child in node.children:
213
+ cumulative_cuda_time += _cumulative_cuda_time_recursive(
214
+ child)
215
+ return cumulative_cuda_time
216
+
217
+ return _cumulative_cuda_time_recursive(node)
218
+
219
+ def _total_cuda_time(self):
220
+ return sum(
221
+ [self._cumulative_cuda_time(root) for root in self._module_tree])
222
+
223
+ def _build_stats_trees(self):
224
+ summary_dict: Dict[str, self.StatsTreeNode] = {}
225
+ total_cuda_time = self._total_cuda_time()
226
+
227
+ def pct_cuda_time(cuda_time_us):
228
+ return (cuda_time_us / total_cuda_time) * 100
229
+
230
+ def build_summary_stats_tree_df(
231
+ node: _ModuleTreeNode,
232
+ parent: Optional[_StatsTreeNode] = None,
233
+ summary_trace: Tuple[str] = ()):
234
+
235
+ if event_has_module(node.event):
236
+ name = event_module_repr(node.event)
237
+ cuda_time_us = self._cumulative_cuda_time(node)
238
+ elif (gpu_kineto_event := self._get_kineto_gpu_event(node)):
239
+ name = gpu_kineto_event.name()
240
+ cuda_time_us = gpu_kineto_event.duration_ns() / 1000.0
241
+ else:
242
+ return None
243
+
244
+ summary_trace = summary_trace + (name, )
245
+ if summary_trace in summary_dict:
246
+ entry = summary_dict[summary_trace].entry
247
+ entry.cuda_time_us += cuda_time_us
248
+ entry.invocations += 1
249
+ entry.pct_cuda_time = pct_cuda_time(entry.cuda_time_us)
250
+ else:
251
+ new_node = _StatsTreeNode(entry=SummaryStatsEntry(
252
+ name=name,
253
+ cuda_time_us=cuda_time_us,
254
+ pct_cuda_time=pct_cuda_time(cuda_time_us),
255
+ invocations=1),
256
+ children=[],
257
+ parent=parent)
258
+ if parent:
259
+ parent.children.append(new_node)
260
+ summary_dict[summary_trace] = new_node
261
+
262
+ for child in node.children:
263
+ build_summary_stats_tree_df(child, summary_dict[summary_trace],
264
+ summary_trace)
265
+
266
+ return summary_dict[summary_trace]
267
+
268
+ self._summary_stats_tree = []
269
+ for root in self._module_tree:
270
+ self._summary_stats_tree.append(build_summary_stats_tree_df(root))
271
+
272
+ def build_model_stats_tree_df(node: _ModuleTreeNode,
273
+ parent: Optional[_StatsTreeNode] = None):
274
+ if event_has_module(node.event, ):
275
+ name = event_module_repr(node.event)
276
+ cuda_time_us = self._cumulative_cuda_time(node)
277
+ cpu_time_us = node.event.duration_time_ns / 1000
278
+ trace = ""
279
+ elif (gpu_kineto_event := self._get_kineto_gpu_event(node)):
280
+ name = gpu_kineto_event.name()
281
+ cuda_time_us = gpu_kineto_event.duration_ns() / 1000.0
282
+ cpu_time_us = 0
283
+ trace = node.trace
284
+ else:
285
+ return None
286
+
287
+ new_node = _StatsTreeNode(entry=ModelStatsEntry(
288
+ name=name,
289
+ cpu_time_us=cpu_time_us,
290
+ cuda_time_us=cuda_time_us,
291
+ pct_cuda_time=pct_cuda_time(cuda_time_us),
292
+ trace=trace),
293
+ parent=parent,
294
+ children=[])
295
+ if parent:
296
+ parent.children.append(new_node)
297
+
298
+ for child in node.children:
299
+ build_model_stats_tree_df(child, new_node)
300
+
301
+ return new_node
302
+
303
+ self._model_stats_tree = []
304
+ for root in self._module_tree:
305
+ self._model_stats_tree.append(build_model_stats_tree_df(root))
306
+
307
+ def _flatten_stats_tree(
308
+ self, tree: List[_StatsTreeNode]) -> List[Tuple[int, StatsEntry]]:
309
+ entries: List[Tuple[int, StatsEntry]] = []
310
+
311
+ def df_traversal(node: _StatsTreeNode, depth=0):
312
+ entries.append((depth, node.entry))
313
+ for child in node.children:
314
+ df_traversal(child, depth=depth + 1)
315
+
316
+ for root in tree:
317
+ df_traversal(root)
318
+
319
+ return entries
320
+
321
+ def _convert_stats_tree_to_dict(self,
322
+ tree: List[_StatsTreeNode]) -> List[Dict]:
323
+ root_dicts: List[Dict] = []
324
+
325
+ def df_traversal(node: _StatsTreeNode, curr_json_list: List[Dict]):
326
+ curr_json_list.append({
327
+ "entry": asdict(node.entry),
328
+ "children": []
329
+ })
330
+ for child in node.children:
331
+ df_traversal(child, curr_json_list[-1]["children"])
332
+
333
+ for root in tree:
334
+ df_traversal(root, root_dicts)
335
+
336
+ return root_dicts
337
+
338
+
339
+class layerwise_profile(profile):
340
+
341
+ def __init__(self):
342
+ super().__init__(
343
+ activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
344
+ record_shapes=True,
345
+ with_stack=True,
346
+ with_modules=True,
347
+ experimental_config=_ExperimentalConfig(verbose=True))
348
+
349
+ def __enter__(self):
350
+ return super().__enter__()
351
+
352
+ def __exit__(self, exc_type, exc_val, exc_tb):
353
+ super().__exit__(exc_type, exc_val, exc_tb)
354
+ self.results = LayerwiseProfileResults(self.profiler.kineto_results)
vllm/profiler/utils.py
new
+145
@@ -0,0 +1,145 @@
1
+import dataclasses
2
+from typing import Callable, Dict, List, Type, Union
3
+
4
+from torch._C._profiler import _EventType, _ProfilerEvent, _TensorMetadata
5
+
6
+#
7
+# String / Print Manipulation
8
+#
9
+
10
+
11
+def trim_string_front(string, width):
12
+ if len(string) > width:
13
+ offset = len(string) - width + 3
14
+ string = string[offset:]
15
+ if len(string) > 3:
16
+ string = "..." + string[3:]
17
+ return string
18
+
19
+
20
+def trim_string_back(string, width):
21
+ if len(string) > width:
22
+ offset = len(string) - width + 3
23
+ string = string[:-offset]
24
+ if len(string) > 3:
25
+ string = string + "..."
26
+ return string
27
+
28
+
29
+class TablePrinter:
30
+
31
+ def __init__(self, row_cls: Type[dataclasses.dataclass],
32
+ column_widths: Dict[str, int]):
33
+ self.row_cls = row_cls
34
+ self.fieldnames = [x.name for x in dataclasses.fields(row_cls)]
35
+ self.column_widths = column_widths
36
+ assert set(self.column_widths.keys()) == set(self.fieldnames)
37
+
38
+ def print_table(self, rows: List[dataclasses.dataclass]):
39
+ self._print_header()
40
+ self._print_line()
41
+ for row in rows:
42
+ self._print_row(row)
43
+
44
+ def _print_header(self):
45
+ for i, f in enumerate(self.fieldnames):
46
+ last = (i == len(self.fieldnames) - 1)
47
+ col_width = self.column_widths[f]
48
+ print(trim_string_back(f, col_width).ljust(col_width),
49
+ end=" | " if not last else "\n")
50
+
51
+ def _print_row(self, row):
52
+ assert isinstance(row, self.row_cls)
53
+
54
+ for i, f in enumerate(self.fieldnames):
55
+ last = (i == len(self.fieldnames) - 1)
56
+ col_width = self.column_widths[f]
57
+ val = getattr(row, f)
58
+
59
+ val_str = ""
60
+ if isinstance(val, str):
61
+ val_str = trim_string_back(val, col_width).ljust(col_width)
62
+ elif type(val) in [float, int]:
63
+ val_str = f"{float(val):>.2f}".rjust(col_width)
64
+ else:
65
+ val_str = f"{val}".rjust(col_width)
66
+ print(val_str, end=" | " if not last else "\n")
67
+
68
+ def _print_line(self):
69
+ total_col_width = 0
70
+ for column_width in self.column_widths.values():
71
+ total_col_width += column_width
72
+ print("=" * (total_col_width + 3 * (len(self.column_widths) - 1)))
73
+
74
+
75
+def indent_string(string: str,
76
+ indent: int,
77
+ indent_style: Union[Callable[[int], str], str] = " ") -> str:
78
+ if indent:
79
+ if isinstance(indent_style, str):
80
+ return indent_style * indent + string
81
+ else:
82
+ return indent_style(indent) + string
83
+ else:
84
+ return string
85
+
86
+
87
+#
88
+# _ProfilerEvent utils
89
+#
90
+
91
+
92
+def event_has_module(event: _ProfilerEvent) -> bool:
93
+ event_type, typed_event = event.typed
94
+ if event_type == _EventType.PyCall:
95
+ return typed_event.module is not None
96
+ return False
97
+
98
+
99
+def event_is_torch_op(event: _ProfilerEvent) -> bool:
100
+ return event.tag == _EventType.TorchOp
101
+
102
+
103
+def event_arg_repr(arg) -> str:
104
+ if arg is None or type(arg) in [float, int, bool, str]:
105
+ return f"{arg}"
106
+ elif isinstance(arg, list):
107
+ return f"[{', '.join([event_arg_repr(x) for x in arg])}]"
108
+ elif isinstance(arg, tuple):
109
+ return f"({', '.join([event_arg_repr(x) for x in arg])})"
110
+ else:
111
+ assert isinstance(arg,
112
+ _TensorMetadata), f"Unsupported type: {type(arg)}"
113
+ sizes_str = ', '.join([str(x) for x in arg.sizes])
114
+ return f"{str(arg.dtype).replace('torch.', '')}[{sizes_str}]"
115
+
116
+
117
+def event_torch_op_repr(event: _ProfilerEvent) -> str:
118
+ assert event.tag == _EventType.TorchOp
119
+ args_str = ', '.join([event_arg_repr(x) for x in event.typed[1].inputs])
120
+ return f"{event.name}({args_str})".replace("aten::", "")
121
+
122
+
123
+def event_module_repr(event: _ProfilerEvent) -> str:
124
+ assert event_has_module(event)
125
+ module = event.typed[1].module
126
+ if module.parameters and len(module.parameters) > 0:
127
+ args_str = ', '.join(
128
+ [f'{x[0]}={event_arg_repr(x[1])}' for x in module.parameters])
129
+ return f"{module.cls_name}({args_str})"
130
+ else:
131
+ return module.cls_name
132
+
133
+
134
+def event_torch_op_stack_trace(curr_event: _ProfilerEvent,
135
+ until: Callable[[_ProfilerEvent], bool]) -> str:
136
+ trace = ""
137
+ curr_event = curr_event.parent
138
+ while curr_event and not until(curr_event):
139
+ if event_is_torch_op(curr_event):
140
+ if len(trace) > 0:
141
+ trace += " <- "
142
+ trace += event_torch_op_repr(curr_event)
143
+ curr_event = curr_event.parent
144
+
145
+ return trace
vllm/worker/model_runner.py
+4
-4
@@ -1742,10 +1742,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1742
return [output]
1743
1744
1745
-class CUDAGraphRunner:
1745
+# NOTE: this is nn.Module so the profiler can properly capture/group
1746
+# kernels calls made within the graph
1747
+class CUDAGraphRunner(nn.Module):
1748
1749
def __init__(self, model: nn.Module, backend_name: str,
1750
attn_state: AttentionState, is_encoder_decoder_model: bool):
1751
+ super().__init__()
1752
self.model = model
1753
self.backend_name = backend_name
1754
self.attn_state = attn_state
@@ -1892,9 +1895,6 @@ class CUDAGraphRunner:
1895
1896
return self.output_buffers
1897
1895
- def __call__(self, *args, **kwargs):
1896
- return self.forward(*args, **kwargs)
1897
-
1898
1899
def _get_graph_batch_size(batch_size: int) -> int:
1900
"""Returns the padded batch size given actual batch size.