@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d30a056e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[misc] CUDA Time Layerwise Profiler (#8337)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Lucas Wilkinson committed Oct 17, 2024 at 10:36 UTC 9d30a056e7a1c81382a53ac63dc476c5fbe0091d
8 files changed +1390 -4
.buildkite/test-pipeline.yaml
+1
@@ -184,6 +184,7 @@ steps:
184 - python3 offline_inference_vision_language_multi_image.py
185 - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
186 - python3 offline_inference_encoder_decoder.py
187 + - python3 offline_profile.py --model facebook/opt-125m
188
189 - label: Prefix Caching Test # 9min
190 #mirror_hardwares: [amd]
examples/offline_profile.py new
+282
@@ -0,0 +1,282 @@
1 +import inspect
2 +import json
3 +import os
4 +import sys
5 +from argparse import RawTextHelpFormatter
6 +from dataclasses import asdict, dataclass
7 +from typing import Optional
8 +
9 +import torch
10 +
11 +from vllm import LLM, SamplingParams
12 +from vllm.engine.arg_utils import EngineArgs
13 +from vllm.profiler import layerwise_profile
14 +from vllm.utils import FlexibleArgumentParser
15 +
16 +BATCH_SIZE_DEFAULT = 1
17 +PROMPT_LEN_DEFAULT = 256
18 +OUTPUT_LEN_DEFAULT = 2
19 +
20 +
21 +@dataclass
22 +class ProfileContext:
23 + engine_args: EngineArgs
24 + prompt_len: int
25 + output_len: int
26 + batch_size: int
27 + save_chrome_traces_folder: Optional[str]
28 +
29 +
30 +def get_dtype(dtype: str):
31 + if dtype == "torch.float":
32 + return torch.float
33 + else:
34 + return dtype
35 +
36 +
37 +def run_profile(context: ProfileContext, csv_output: Optional[str],
38 + json_output: Optional[str]):
39 + print("Run profile with:")
40 + for key, value in asdict(context).items():
41 + print(f" {key} = {value}")
42 +
43 + # Create sampling params
44 + sampling_params = SamplingParams(temperature=0.8,
45 + top_p=0.95,
46 + max_tokens=args.output_len,
47 + ignore_eos=True)
48 +
49 + # Create LLM
50 + llm = LLM(**asdict(context.engine_args))
51 + batch_size = context.batch_size
52 + prompt_len = context.prompt_len
53 + output_len = context.output_len
54 +
55 + scheduler_config = llm.llm_engine.scheduler_config
56 + max_model_len = llm.llm_engine.model_config.max_model_len
57 + max_num_batched_tokens = scheduler_config.max_num_batched_tokens
58 + max_num_seqs = scheduler_config.max_num_seqs
59 +
60 + if batch_size * prompt_len > max_num_batched_tokens:
61 + print(f"ERROR: chosen batch_size * prompt_len "
62 + f"({batch_size} * {prompt_len} = {batch_size * prompt_len}) is "
63 + f"larger than max_num_batched_tokens ({max_num_batched_tokens}) "
64 + f"and therefore cannot be run in a single profile step, please "
65 + f"choose a smaller batch size or prompt length, or increase "
66 + f"--max-num-batched-tokens")
67 + sys.exit(-1)
68 + if batch_size >= max_num_seqs:
69 + print(
70 + f"ERROR: chosen batch_size ({batch_size}) is larger than "
71 + f"max_num_seqs ({max_num_seqs}) and therefore cannot be run in a "
72 + f"single profile step, please choose a smaller batch size")
73 + sys.exit(-1)
74 + print("llm.llm_engine.model_config.max_model_len: ",
75 + llm.llm_engine.model_config.max_model_len)
76 + if prompt_len + output_len > llm.llm_engine.model_config.max_model_len:
77 + print(
78 + f"ERROR: chosen prompt_len + output_len ({prompt_len} + "
79 + f"{output_len} = {prompt_len + output_len}) is larger than the "
80 + f"model's max_model_len ({max_model_len}), please choose a smaller "
81 + f"prompt_len or output_len, or increase --max-model-len")
82 + sys.exit(-1)
83 +
84 + def add_requests():
85 + for i in range(batch_size):
86 + prompt_token_ids = torch.randint(
87 + llm.llm_engine.model_config.get_vocab_size(),
88 + size=(prompt_len, )).tolist()
89 +
90 + llm.llm_engine.add_request(
91 + request_id=f"seq{i}",
92 + prompt={'prompt_token_ids': prompt_token_ids},
93 + params=sampling_params)
94 +
95 + def abort_requests():
96 + for i in range(batch_size):
97 + llm.llm_engine.abort_request(f"seq{i}")
98 +
99 + # Warm up run
100 + print("Warm up run ...")
101 + add_requests()
102 + llm.llm_engine.step() # Prefill
103 + llm.llm_engine.step() # Decode
104 + abort_requests()
105 +
106 + print("Profile run ...")
107 + add_requests()
108 +
109 + with layerwise_profile() as prefill_prof:
110 + llm.llm_engine.step() # First step is prefill
111 +
112 + decode_profs = []
113 + for x in range(args.output_len - 1):
114 + with layerwise_profile() as decode_prof:
115 + llm.llm_engine.step()
116 + decode_profs.append(decode_prof)
117 +
118 + decode_results_list = [prof.results for prof in decode_profs]
119 + prefill_results = prefill_prof.results
120 + has_decode = len(decode_results_list) > 0
121 +
122 + LINE_WIDTH = 80
123 + print("=" * LINE_WIDTH)
124 + print(f"= Prefill Model Table "
125 + f"(prompt_len={prompt_len}, batch_size={batch_size})")
126 + print("=" * LINE_WIDTH)
127 + print()
128 + prefill_results.print_model_table()
129 +
130 + if has_decode:
131 + print()
132 + print("=" * LINE_WIDTH)
133 + print(f"= First Decode Step Model Table "
134 + f"(prompt_len={prompt_len}, batch_size={batch_size})")
135 + print("=" * LINE_WIDTH)
136 + print()
137 + decode_results_list[0].print_model_table()
138 +
139 + print()
140 + print("=" * LINE_WIDTH)
141 + print(f"= Prefill Summary Table "
142 + f"(prompt_len={prompt_len}, batch_size={batch_size})")
143 + print("=" * LINE_WIDTH)
144 + print()
145 + prefill_results.print_summary_table()
146 +
147 + if has_decode:
148 + print()
149 + print("=" * LINE_WIDTH)
150 + print(f"= First Decode Step Summary Table "
151 + f"(prompt_len={prompt_len}, batch_size={batch_size})")
152 + print("=" * LINE_WIDTH)
153 + print()
154 + decode_results_list[0].print_summary_table()
155 +
156 + if csv_output:
157 + csv_filename_base = csv_output.rstrip(".csv")
158 + prefill_results.export_model_stats_table_csv(
159 + csv_filename_base + "_prefill_model_table.csv")
160 + prefill_results.export_summary_stats_table_csv(
161 + csv_filename_base + "_prefill_summary_table.csv")
162 +
163 + if has_decode:
164 + decode_results_list[0].export_model_stats_table_csv(\
165 + csv_filename_base + "_decode_model_table.csv")
166 + decode_results_list[0].export_summary_stats_table_csv(
167 + csv_filename_base + "_decode_summary_table.csv")
168 +
169 + if json_output:
170 + cuda_devices = [
171 + torch.cuda.get_device_properties(dev_idx)
172 + for dev_idx in range(torch.cuda.device_count())
173 + ]
174 +
175 + json_dict = {
176 + "context": {
177 + "python_version": f"{sys.version}",
178 + "torch_version": f"{torch.__version__}",
179 + "torch_cuda_version": f"{torch.version.cuda}",
180 + "cuda_devices": f"{cuda_devices}",
181 + **asdict(context)
182 + },
183 + "prefill": prefill_results.convert_stats_to_dict(),
184 + }
185 +
186 + if has_decode:
187 + for idx, dr in enumerate(decode_results_list):
188 + json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
189 +
190 + for idx, dr in enumerate(decode_results_list[1:]):
191 + json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
192 +
193 + with open(json_output.rstrip(".json") + ".json", "w+") as f:
194 + json.dump(json_dict, f, indent=2)
195 + pass
196 +
197 + if context.save_chrome_traces_folder is not None:
198 + os.makedirs(context.save_chrome_traces_folder, exist_ok=True)
199 + prefill_prof.profiler.export_chrome_trace(
200 + context.save_chrome_traces_folder + "/prefill.json")
201 + for idx, decode_prof in enumerate(decode_profs):
202 + decode_prof.profiler.export_chrome_trace(
203 + context.save_chrome_traces_folder + f"/decode_{idx + 1}.json")
204 + print("Traces saved as prefill.json and decode_1.json, etc."
205 + f" in folder {context.save_chrome_traces_folder}")
206 +
207 +
208 +if __name__ == "__main__":
209 + parser = FlexibleArgumentParser(description="""
210 +Profile a model
211 +
212 + example:
213 + ```
214 + python examples/offline_profile.py \\
215 + --model neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 --batch-size 4 \\
216 + --prompt-len 512 --max-num-batched-tokens 8196 --json Llama31-8b-FP8 \\
217 + --enforce-eager
218 + ```
219 +
220 + then you can use various tools to analyze the json output
221 + terminal ascii tables:
222 + ```
223 + python tools/profiler/print_layerwise_table.py \\
224 + --json-trace Llama31-8b-FP8.json --phase prefill --table summary
225 + ```
226 + or create matplotlib stacked bar charts:
227 + ```
228 + python tools/profiler/visualize_layerwise_profile.py \\
229 + --json-trace Llama31-8b-FP8.json \\
230 + --output-directory profile_breakdown --plot-metric pct_cuda_time
231 + ```
232 +""",
233 + formatter_class=RawTextHelpFormatter)
234 + parser.add_argument(
235 + "--csv",
236 + type=str,
237 + default=None,
238 + help="Export the results as multiple csv file. This should be the root "
239 + "filename, will create <filename>_prefill_model_table.csv, "
240 + "<filename>_prefill_summary_table.csv, "
241 + "<filename>_decode_model_table.csv, and "
242 + "<filename>_decode_summary_table.csv")
243 + parser.add_argument(
244 + "--json",
245 + type=str,
246 + default=None,
247 + help="Export the results as a json file. This should be the filename")
248 + parser.add_argument("--save-chrome-traces-folder",
249 + type=str,
250 + help="Save chrome traces for the prefill and decode "
251 + "will save traces as prefill.json and decode_1.json, "
252 + "etc. inside this folder")
253 + parser.add_argument(
254 + "--prompt-len",
255 + type=int,
256 + default=PROMPT_LEN_DEFAULT,
257 + help=f"Length of the random prompt to use when profiling, all batched "
258 + f"requests use the same prompt_len, default={PROMPT_LEN_DEFAULT}")
259 + parser.add_argument("--batch-size",
260 + type=int,
261 + default=BATCH_SIZE_DEFAULT,
262 + help=f"Number of requests to run as a single batch, "
263 + f"default={BATCH_SIZE_DEFAULT}")
264 + parser.add_argument(
265 + "--output-len",
266 + type=int,
267 + default=OUTPUT_LEN_DEFAULT,
268 + help="Number of llm steps to run (includes prefill and decode) "
269 + "- default={OUTPUT_LEN_DEFAULT}")
270 +
271 + EngineArgs.add_cli_args(parser)
272 +
273 + args = parser.parse_args()
274 +
275 + context = ProfileContext(
276 + engine_args=EngineArgs.from_cli_args(args),
277 + **{
278 + k: v
279 + for k, v in vars(args).items()
280 + if k in inspect.signature(ProfileContext).parameters
281 + })
282 + run_profile(context, csv_output=args.csv, json_output=args.json)
tools/profiler/print_layerwise_table.py new
+77
@@ -0,0 +1,77 @@
1 +import argparse
2 +import json
3 +from typing import Dict
4 +
5 +from vllm.profiler.layerwise_profile import ModelStatsEntry, SummaryStatsEntry
6 +from vllm.profiler.utils import TablePrinter, indent_string
7 +
8 +
9 +def flatten_entries(entry_cls, profile_dict: Dict):
10 + entries_and_depth = []
11 +
12 + def get_entries(node, curr_depth=0):
13 + entries_and_depth.append((entry_cls(**node["entry"]), curr_depth))
14 +
15 + for child in node["children"]:
16 + get_entries(
17 + child,
18 + curr_depth=curr_depth + 1,
19 + )
20 +
21 + for root in profile_dict:
22 + get_entries(root)
23 +
24 + return entries_and_depth
25 +
26 +
27 +if __name__ == "__main__":
28 + parser = argparse.ArgumentParser()
29 +
30 + parser.add_argument("--json-trace",
31 + type=str,
32 + required=True,
33 + help="json trace file output by "
34 + "examples/offline_profile.py")
35 + parser.add_argument("--phase",
36 + type=str,
37 + choices=["prefill", "decode_1"],
38 + required=True,
39 + help="The phase to print the table for.")
40 + parser.add_argument("--table",
41 + type=str,
42 + choices=["summary", "model"],
43 + default="summary",
44 + help="Which table to print, the summary table or the "
45 + "layerwise model table")
46 +
47 + args = parser.parse_args()
48 +
49 + with open(args.json_trace, "r") as f:
50 + profile_data = json.load(f)
51 +
52 + if args.table == "summary":
53 + entries_and_depths = flatten_entries(
54 + SummaryStatsEntry, profile_data[args.phase]["summary_stats"])
55 + column_widths = dict(name=80,
56 + cuda_time_us=12,
57 + pct_cuda_time=12,
58 + invocations=15)
59 + elif args.table == "model":
60 + entries_and_depths = flatten_entries(
61 + ModelStatsEntry, profile_data[args.phase]["model_stats"])
62 + column_widths = dict(name=60,
63 + cpu_time_us=12,
64 + cuda_time_us=12,
65 + pct_cuda_time=12,
66 + trace=60)
67 +
68 + # indent entry names based on the depth
69 + entries = []
70 + for entry, depth in entries_and_depths:
71 + entry.name = indent_string(
72 + entry.name,
73 + indent=depth,
74 + indent_style=lambda indent: "|" + "-" * indent + " ")
75 + entries.append(entry)
76 +
77 + TablePrinter(type(entries[0]), column_widths).print_table(entries)
tools/profiler/visualize_layerwise_profile.py new
+522
@@ -0,0 +1,522 @@
1 +import argparse
2 +import copy
3 +import json
4 +import math
5 +import os
6 +from pathlib import Path
7 +from typing import Any, List, Optional, Tuple
8 +
9 +import matplotlib.pyplot as plt
10 +import pandas as pd
11 +
12 +## JSON parsing utils ####
13 +
14 +
15 +def largest_dist_from_leaf(node: dict, depth: int = 0):
16 + if len(node["children"]) == 0:
17 + return depth
18 + return max([
19 + largest_dist_from_leaf(child, depth=depth + 1)
20 + for child in node["children"]
21 + ])
22 +
23 +
24 +def get_entries_at_depth(depth: int,
25 + entries_and_traces: List[Tuple[Any, Any]],
26 + node: dict,
27 + curr_depth: int = 0,
28 + trace=()):
29 + # assert that the query is at kernel or module level
30 + assert depth == -1 or depth == -2
31 +
32 + if curr_depth == 0 and largest_dist_from_leaf(node) <= (abs(depth) - 1):
33 + # The tree is not tall enough!
34 + entries_and_traces.append((node["entry"], trace))
35 + return
36 +
37 + if largest_dist_from_leaf(node) == (abs(depth) - 1):
38 + entries_and_traces.append((node["entry"], trace))
39 +
40 + trace = (node["entry"]["name"], ) + trace
41 + for child in node["children"]:
42 + get_entries_at_depth(depth,
43 + entries_and_traces,
44 + child,
45 + curr_depth=curr_depth + 1,
46 + trace=trace)
47 +
48 +
49 +def fold_nodes(root: dict, nodes_to_fold: List[str]):
50 +
51 + stack: List[dict] = [root]
52 + while len(stack) != 0:
53 + node = stack.pop()
54 + if node['entry']['name'] in nodes_to_fold:
55 + node["children"] = []
56 + continue
57 + for child in node["children"]:
58 + stack.append(child)
59 + return root
60 +
61 +
62 +## Operation name cleanup utils ####
63 +
64 +
65 +def trim_string_back(string: str, width: int) -> str:
66 + if len(string) > width:
67 + offset = len(string) - width + 3
68 + string = string[:-offset]
69 + if len(string) > 3:
70 + string = string + "..."
71 + return string
72 +
73 +
74 +def shorten_plot_legend_strings(legend, max_char_len: int):
75 + for t in legend.get_texts():
76 + t.set_text(
77 + trim_string_back(abbreviate_known_names(t.get_text()),
78 + max_char_len))
79 +
80 +
81 +def abbreviate_known_names(name: str) -> str:
82 + abbreviations = {
83 + "MergedColumnParallelLinear": "MCPLinear",
84 + "QKVParallelLinear": "QKVPLinear",
85 + "RowParallelLinear": "RPLinear",
86 + "weight=": "w=",
87 + "bfloat16": "bf16",
88 + "float16": "f16",
89 + }
90 + for key, value in abbreviations.items():
91 + name = name.replace(key, value)
92 + return name
93 +
94 +
95 +def attempt_to_make_names_unique(entries_and_traces):
96 + names, non_unique_names = (set(), set())
97 +
98 + def all_the_same(items) -> bool:
99 + return all(i == items[0] for i in items)
100 +
101 + for entry, _ in entries_and_traces:
102 + if entry["name"] in names:
103 + non_unique_names.add(entry["name"])
104 + else:
105 + names.add(entry["name"])
106 +
107 + for name in non_unique_names:
108 + entries_and_traces_with_name = [(entry, trace)
109 + for entry, trace in entries_and_traces
110 + if entry["name"] == name]
111 +
112 + zipped_traces = list(
113 + zip(*[trace for _, trace in entries_and_traces_with_name]))
114 + first_trace_difference = next(
115 + (i for i, trace_eles in enumerate(zipped_traces)
116 + if not all_the_same(trace_eles)), None)
117 +
118 + if first_trace_difference is None:
119 + # can't create a unique name, leave them names as the
120 + # are they will get aggregated by the pivot_table call
121 + continue
122 +
123 + for entry, trace in entries_and_traces_with_name:
124 + entry["name"] = " <- ".join((entry["name"], ) +
125 + trace[:first_trace_difference + 1])
126 +
127 +
128 +## Operation grouping utils ####
129 +'''
130 + Group operations in the given dataframe by some high-level ops like,
131 + - gemms
132 + - attention
133 + - rms_norm
134 + etc.
135 +'''
136 +
137 +
138 +def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
139 +
140 + def is_rms_norm(op_name: str):
141 + if "rms_norm_kernel" in op_name:
142 + return True
143 +
144 + def is_attention_block(op_name: str):
145 + if "flash_fwd" in op_name or \
146 + "reshape_and_cache_flash_kernel" in op_name:
147 + return True
148 +
149 + def is_quant(op_name: str):
150 + if "scaled_fp8_quant" in op_name or \
151 + "scaled_int8_quant" in op_name:
152 + return True
153 +
154 + def is_gemm_op(op_name: str):
155 + if is_quant(op_name):
156 + return False
157 + if "xmma_gemm" in op_name or \
158 + "gemv2T_kernel" in op_name or \
159 + "splitKreduce" in op_name or \
160 + "void cutlass::Kernel" in op_name or \
161 + "void cutlass::device_kernel" in op_name or \
162 + "s16816gemm" in op_name:
163 + return True
164 +
165 + def is_elementwise_op(op_name: str):
166 + return "elementwise_kernel" in op_name
167 +
168 + def is_mem_op(op_name: str):
169 + return "memcpy" in op_name.lower() or \
170 + "memset" in op_name.lower()
171 +
172 + def is_vocab_embedding_op(op_name: str):
173 + return "vocabparallelembed" in op_name.lower()
174 +
175 + # nccl ops
176 + def is_nccl_op(op_name: str):
177 + return "nccl" in op_name.lower()
178 +
179 + def is_nccl_all_reduce(op_name: str):
180 + return is_nccl_op(op_name) and \
181 + ("all_reduce" in op_name.lower() or \
182 + "allreduce" in op_name.lower())
183 +
184 + def is_nccl_gather(op_name: str):
185 + return is_nccl_op(op_name) and \
186 + "gather" in op_name.lower()
187 +
188 + def is_nccl_broadcast(op_name: str):
189 + return is_nccl_op(op_name) and \
190 + "broadcast" in op_name.lower()
191 +
192 + # Reduce ops types
193 + def is_cross_device_reduce_1stage(op_name: str):
194 + return "cross_device_reduce_1stage" in op_name
195 +
196 + def is_cross_device_reduce_2stage(op_name: str):
197 + return "cross_device_reduce_2stage" in op_name
198 +
199 + def is_custom_ar_all_reduce_unreg(op_name: str):
200 + return "_C_custom_ar::all_reduce_unreg" in op_name
201 +
202 + def is_reduce_kernel(op_name: str):
203 + return "reduce_kernel" in op_name
204 +
205 + headers = list(trace_df)
206 + ops = copy.deepcopy(headers)
207 +
208 + attention_ops = list(filter(lambda x: is_attention_block(x), ops))
209 + ops = list(filter(lambda x: x not in attention_ops, ops))
210 +
211 + quant_ops = list(filter(lambda x: is_quant(x), ops))
212 + ops = list(filter(lambda x: x not in quant_ops, ops))
213 +
214 + gemm_ops = list(filter(lambda x: is_gemm_op(x), ops))
215 + ops = list(filter(lambda x: x not in gemm_ops, ops))
216 +
217 + rms_norm_ops = list(filter(lambda x: is_rms_norm(x), ops))
218 + ops = list(filter(lambda x: x not in rms_norm_ops, ops))
219 +
220 + vocab_embed_ops = list(filter(lambda x: is_vocab_embedding_op(x), ops))
221 + ops = list(filter(lambda x: x not in vocab_embed_ops, ops))
222 +
223 + mem_ops = list(filter(lambda x: is_mem_op(x), ops))
224 + ops = list(filter(lambda x: x not in mem_ops, ops))
225 +
226 + elementwise_ops = list(filter(lambda x: is_elementwise_op(x), ops))
227 + ops = list(filter(lambda x: x not in elementwise_ops, ops))
228 +
229 + nccl_all_reduce_ops = list(filter(lambda x: is_nccl_all_reduce(x), ops))
230 + ops = list(filter(lambda x: x not in nccl_all_reduce_ops, ops))
231 +
232 + nccl_gather_ops = list(filter(lambda x: is_nccl_gather(x), ops))
233 + ops = list(filter(lambda x: x not in nccl_gather_ops, ops))
234 +
235 + nccl_broadcast_ops = list(filter(lambda x: is_nccl_broadcast(x), ops))
236 + ops = list(filter(lambda x: x not in nccl_broadcast_ops, ops))
237 +
238 + nccl_other_ops = list(filter(lambda x: is_nccl_op(x), ops))
239 + ops = list(filter(lambda x: x not in nccl_other_ops, ops))
240 +
241 + cross_device_reduce_1stage_ops = list(
242 + filter(lambda x: is_cross_device_reduce_1stage(x), ops))
243 + ops = list(filter(lambda x: x not in cross_device_reduce_1stage_ops, ops))
244 +
245 + cross_device_reduce_2stage_ops = list(
246 + filter(lambda x: is_cross_device_reduce_2stage(x), ops))
247 + ops = list(filter(lambda x: x not in cross_device_reduce_2stage_ops, ops))
248 +
249 + custom_ar_all_reduce_unreg_ops = list(
250 + filter(lambda x: is_custom_ar_all_reduce_unreg(x), ops))
251 + ops = list(filter(lambda x: x not in custom_ar_all_reduce_unreg_ops, ops))
252 +
253 + reduce_kernel_ops = list(filter(lambda x: is_reduce_kernel(x), ops))
254 + ops = list(filter(lambda x: x not in reduce_kernel_ops, ops))
255 +
256 + if len(attention_ops):
257 + trace_df['attention'] = trace_df[attention_ops].agg("sum", axis=1)
258 + if len(quant_ops):
259 + trace_df['quant_ops'] = trace_df[quant_ops].agg("sum", axis=1)
260 + if len(gemm_ops):
261 + trace_df['gemm_ops'] = trace_df[gemm_ops].agg("sum", axis=1)
262 + if len(rms_norm_ops):
263 + trace_df['rms_norm_ops'] = trace_df[rms_norm_ops].agg("sum", axis=1)
264 + if len(vocab_embed_ops):
265 + trace_df['vocab_embed_ops'] = trace_df[vocab_embed_ops].agg("sum",
266 + axis=1)
267 + if len(mem_ops):
268 + trace_df['mem_ops'] = trace_df[mem_ops].agg("sum", axis=1)
269 + if len(elementwise_ops):
270 + trace_df['elementwise_ops'] = trace_df[elementwise_ops].agg("sum",
271 + axis=1)
272 +
273 + if len(nccl_all_reduce_ops):
274 + trace_df['nccl_all_reduce_ops'] = trace_df[nccl_all_reduce_ops].agg(
275 + "sum", axis=1)
276 + if len(nccl_gather_ops):
277 + trace_df['nccl_gather_ops'] = trace_df[nccl_gather_ops].agg("sum",
278 + axis=1)
279 + if len(nccl_broadcast_ops):
280 + trace_df['nccl_broadcast_ops'] = trace_df[nccl_broadcast_ops].agg(
281 + "sum", axis=1)
282 + if len(nccl_other_ops):
283 + trace_df['nccl_other_ops'] = trace_df[nccl_other_ops].agg("sum",
284 + axis=1)
285 +
286 + if len(cross_device_reduce_1stage_ops):
287 + trace_df['cross_device_reduce_1stage_ops'] = trace_df[
288 + cross_device_reduce_1stage_ops].agg("sum", axis=1)
289 + if len(cross_device_reduce_2stage_ops):
290 + trace_df['cross_device_reduce_2stage_ops'] = trace_df[
291 + cross_device_reduce_2stage_ops].agg("sum", axis=1)
292 + if len(custom_ar_all_reduce_unreg_ops):
293 + trace_df['custom_ar_all_reduce_unreg_ops'] = trace_df[
294 + custom_ar_all_reduce_unreg_ops].agg("sum", axis=1)
295 + if len(reduce_kernel_ops):
296 + trace_df['reduce_kernel_ops'] = trace_df[reduce_kernel_ops].agg("sum",
297 + axis=1)
298 +
299 + trace_df.drop(
300 + attention_ops + quant_ops + gemm_ops + rms_norm_ops + vocab_embed_ops +
301 + mem_ops + elementwise_ops + nccl_all_reduce_ops + nccl_gather_ops +
302 + nccl_broadcast_ops + nccl_other_ops + cross_device_reduce_1stage_ops +
303 + cross_device_reduce_2stage_ops + custom_ar_all_reduce_unreg_ops +
304 + reduce_kernel_ops,
305 + axis=1,
306 + inplace=True)
307 + return trace_df
308 +
309 +
310 +## Data plotting utils ####
311 +
312 +
313 +def plot_trace_df(traces_df: pd.DataFrame,
314 + plot_metric: str,
315 + plot_title: str,
316 + output: Optional[Path] = None):
317 +
318 + phases = traces_df['phase'].unique()
319 + traces_df = traces_df.pivot_table(index="phase",
320 + columns="name",
321 + values=plot_metric,
322 + aggfunc="sum")
323 +
324 + traces_df = group_trace_by_operations(traces_df)
325 +
326 + # Make the figure
327 + fig, ax = plt.subplots(1, figsize=(5, 8), sharex=True)
328 +
329 + # Draw the stacked bars
330 + ops = list(traces_df)
331 + bottom = [0] * len(phases)
332 + for op in ops:
333 + values = [traces_df[op][phase] for phase in phases]
334 + values = list(map(lambda x: 0.0 if math.isnan(x) else x, values))
335 + ax.bar(phases, values, label=op, bottom=bottom)
336 + bottom = [bottom[j] + values[j] for j in range(len(phases))]
337 +
338 + # Write the values as text on the bars
339 + for bar in ax.patches:
340 + if bar.get_height() != 0:
341 + ax.text(bar.get_x() + bar.get_width() / 2,
342 + bar.get_height() / 2 + bar.get_y(),
343 + f"{round(bar.get_height(), 2)}",
344 + ha='center',
345 + color='w',
346 + weight='bold',
347 + size=5)
348 +
349 + # Setup legend
350 + handles, labels = plt.gca().get_legend_handles_labels()
351 + legend = fig.legend(handles,
352 + labels,
353 + loc='center left',
354 + bbox_to_anchor=(1, 1))
355 + shorten_plot_legend_strings(legend, 50)
356 +
357 + # Setup labels and title
358 + plt.setp(ax.get_xticklabels(), rotation=90)
359 + ax.set_ylabel(plot_metric)
360 + plt.suptitle(plot_title)
361 +
362 + plt.savefig(output, bbox_inches='tight')
363 + print("Created: ", output)
364 +
365 +
366 +def main(
367 + json_trace: Path,
368 + output_directory: Path,
369 + depth: int, # Fetch/Plot operations at this depth of the Json tree
370 + plot_metric: str,
371 + make_names_unique: bool,
372 + top_k: int,
373 + json_nodes_to_fold: List[str]):
374 +
375 + def prepare_data(profile_json: dict, step_keys: List[str]) -> pd.DataFrame:
376 +
377 + def get_entries_and_traces(key: str):
378 + entries_and_traces: List[Tuple[Any, Any]] = []
379 + for root in profile_json[key]["summary_stats"]:
380 + # Fold nodes in the traces as per user request. i.e. simply
381 + # make the requested nodes leaf-nodes.
382 + root = fold_nodes(root, json_nodes_to_fold)
383 + get_entries_at_depth(depth, entries_and_traces, root)
384 + return entries_and_traces
385 +
386 + def keep_only_top_entries(df: pd.DataFrame,
387 + metric: str,
388 + top_k: int = 9) -> pd.DataFrame:
389 + df.loc[df.nsmallest(len(df) - top_k + 1, metric).index,
390 + ["name"]] = "others"
391 + return df
392 +
393 + # Get data for each key
394 + traces = list(map(lambda x: get_entries_and_traces(x), step_keys))
395 +
396 + # Attempt some cleanup
397 + if make_names_unique:
398 + for trace in traces:
399 + attempt_to_make_names_unique(trace)
400 +
401 + # To pandas dataframe
402 + trace_dfs = list(
403 + map(lambda t: pd.DataFrame([entry for entry, _ in t]).fillna(0),
404 + traces))
405 +
406 + # Respect top_k
407 + if top_k:
408 + trace_dfs = list(
409 + map(
410 + lambda trace_df: keep_only_top_entries(
411 + trace_df, "cuda_time_us", top_k), trace_dfs))
412 +
413 + # Fill in information about the step-keys
414 + for trace_df, step_key in zip(trace_dfs, step_keys):
415 + trace_df['phase'] = step_key
416 +
417 + # Combine all data frames so they can be put in a single plot
418 + traces_df = pd.concat(trace_dfs)
419 +
420 + # Add a derived metric `cuda_time_ms`
421 + traces_df["cuda_time_ms"] = traces_df["cuda_time_us"] / 1000
422 + traces_df = traces_df.fillna(0)
423 +
424 + return traces_df
425 +
426 + def make_plot_title_suffix(profile_json: dict) -> str:
427 + context = profile_json["context"]
428 + sparsity = context.get('sparsity', None)
429 + return (f"{context['model']}\n"
430 + f"Batch={context['batch_size']}, "
431 + f"PromptLen={context['prompt_len']}, "
432 + f"OutputLen={context['output_len']},"
433 + f"NumGpus={context['tensor_parallel_size']}"
434 + f"{', Sparsity ' + sparsity if sparsity else ''}")
435 +
436 + profile_json = None
437 + with open(json_trace, "r") as f:
438 + profile_json = json.load(f)
439 + assert profile_json is not None
440 +
441 + # Get all `llm.generate.step()` profile
442 + step_traces = list(profile_json.keys())
443 + assert (step_traces[0] == 'context')
444 + step_traces = step_traces[1:] # have only prefill and decodes
445 + prefills = list(filter(lambda x: "prefill" in x, step_traces))
446 + all_decodes = list(filter(lambda x: "decode" in x, step_traces))
447 + assert len(prefills) + len(all_decodes) == len(step_traces)
448 + assert len(prefills) == 1
449 +
450 + decodes = all_decodes[::args.step_plot_interval]
451 + if decodes[-1] != all_decodes[-1]:
452 + # Always have the last decode
453 + decodes.append(all_decodes[-1])
454 +
455 + prefill_traces = prepare_data(profile_json, prefills)
456 + decode_traces = prepare_data(profile_json, decodes)
457 +
458 + plot_title_suffix = make_plot_title_suffix(profile_json)
459 +
460 + plot_trace_df(prefill_traces, plot_metric, "prefill " + plot_title_suffix,
461 + output_directory / Path("prefill.png"))
462 + plot_trace_df(decode_traces, plot_metric, "decodes " + plot_title_suffix,
463 + output_directory / Path("decode_steps.png"))
464 +
465 +
466 +if __name__ == "__main__":
467 + parser = argparse.ArgumentParser()
468 +
469 + parser.add_argument(
470 + "--json-trace",
471 + type=str,
472 + required=True,
473 + help="json trace file output by examples/offline_profile.py")
474 + parser.add_argument("--output-directory",
475 + type=str,
476 + required=False,
477 + help="Directory to output plots")
478 + parser.add_argument("--level",
479 + type=str,
480 + default="module",
481 + choices=["module", "kernel"])
482 + parser.add_argument("--top-k",
483 + type=int,
484 + default=12,
485 + help="Only graph the top `top_k` entries by time.")
486 + parser.add_argument("--fold-json-node",
487 + nargs='+',
488 + default=['Sampler', 'LogitsProcessor'],
489 + help='Do not plot the children of these nodes. Let, \
490 + the node represent the aggregate of all its \
491 + children')
492 + parser.add_argument("--plot-metric",
493 + type=str,
494 + default="cuda_time_ms",
495 + help='Metric to plot. some options are cuda_time_ms, \
496 + pct_cuda_time')
497 + parser.add_argument(
498 + "--step-plot-interval",
499 + type=int,
500 + default=4,
501 + help="For every `step_plot_interval` steps, plot 1 step")
502 +
503 + args = parser.parse_args()
504 +
505 + # Prepare/Extract relevant args
506 + make_names_unique = False
507 + if args.level == "module":
508 + depth = -2
509 + make_names_unique = True
510 + elif args.level == "kernel":
511 + depth = -1
512 + else:
513 + raise Exception(f"Unexpected level value ({args.level})")
514 +
515 + output_directory = args.output_directory if args.output_directory else Path(
516 + args.json_trace).parent
517 +
518 + if not os.path.exists(output_directory):
519 + os.makedirs(output_directory)
520 +
521 + main(Path(args.json_trace), output_directory, depth, args.plot_metric,
522 + make_names_unique, args.top_k, args.fold_json_node)
vllm/profiler/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from .layerwise_profile import layerwise_profile
2 +
3 +__all__ = [
4 + "layerwise_profile",
5 +]
vllm/profiler/layerwise_profile.py new
+354
@@ -0,0 +1,354 @@
1 +import copy
2 +from collections import defaultdict
3 +from dataclasses import asdict, dataclass, field
4 +from typing import Callable, Dict, List, Optional, Tuple, TypeAlias, Union
5 +
6 +import pandas as pd
7 +from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
8 +from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent
9 +from torch.autograd.profiler import FunctionEvent
10 +from torch.profiler import ProfilerActivity, profile
11 +
12 +from vllm.profiler.utils import (TablePrinter, event_has_module,
13 + event_is_torch_op, event_module_repr,
14 + event_torch_op_stack_trace, indent_string)
15 +
16 +
17 +@dataclass
18 +class _ModuleTreeNode:
19 + event: _ProfilerEvent
20 + parent: Optional['_ModuleTreeNode'] = None
21 + children: List['_ModuleTreeNode'] = field(default_factory=list)
22 + trace: str = ""
23 +
24 + @property
25 + def is_leaf(self):
26 + return (self.event.children is None or len(self.event.children) == 0)
27 +
28 + @property
29 + def is_torch_op(self):
30 + return event_is_torch_op(self.event)
31 +
32 + @property
33 + def is_cuda(self):
34 + return (self.event.tag == _EventType.Kineto
35 + and self.event.typed[1].device_type == DeviceType.CUDA)
36 +
37 +
38 +@dataclass
39 +class SummaryStatsEntry:
40 + name: str
41 + cuda_time_us: float
42 + pct_cuda_time: float
43 + invocations: int
44 +
45 +
46 +@dataclass
47 +class ModelStatsEntry:
48 + name: str
49 + cpu_time_us: float
50 + cuda_time_us: float
51 + pct_cuda_time: float
52 + trace: str
53 +
54 +
55 +StatsEntry: TypeAlias = Union[ModelStatsEntry, SummaryStatsEntry]
56 +
57 +
58 +@dataclass
59 +class _StatsTreeNode:
60 + entry: StatsEntry
61 + children: List[StatsEntry]
62 + parent: Optional[StatsEntry]
63 +
64 +
65 +@dataclass
66 +class LayerwiseProfileResults(profile):
67 + _kineto_results: _ProfilerResult
68 + _kineto_event_correlation_map: Dict[int,
69 + List[_KinetoEvent]] = field(init=False)
70 + _event_correlation_map: Dict[int, List[FunctionEvent]] = field(init=False)
71 + _module_tree: List[_ModuleTreeNode] = field(init=False)
72 + _model_stats_tree: List[_StatsTreeNode] = field(init=False)
73 + _summary_stats_tree: List[_StatsTreeNode] = field(init=False)
74 +
75 + def __post_init__(self):
76 + self._build_correlation_map()
77 + self._build_module_tree()
78 + self._build_stats_trees()
79 +
80 + def print_model_table(self, column_widths: Dict[str, int] = None):
81 + _column_widths = dict(name=60,
82 + cpu_time_us=12,
83 + cuda_time_us=12,
84 + pct_cuda_time=12,
85 + trace=60)
86 + if column_widths:
87 + _column_widths.update(**column_widths)
88 + filtered_model_table = [
89 + (depth, row)
90 + for depth, row in self._flatten_stats_tree(self._model_stats_tree)
91 + if row.cuda_time_us > 0 or row.cpu_time_us > 0
92 + ]
93 + TablePrinter(ModelStatsEntry, _column_widths).print_table(
94 + self._indent_row_names_based_on_depth(
95 + filtered_model_table,
96 + indent_style=lambda indent: "|" + "-" * indent + " "))
97 +
98 + def print_summary_table(self, column_widths: Dict[str, int] = None):
99 + _column_widths = dict(name=80,
100 + cuda_time_us=12,
101 + pct_cuda_time=12,
102 + invocations=15)
103 + if column_widths:
104 + _column_widths.update(**column_widths)
105 + filtered_summary_table = [(depth, row)
106 + for depth, row in self._flatten_stats_tree(
107 + self._summary_stats_tree)
108 + if row.cuda_time_us > 0]
109 + TablePrinter(SummaryStatsEntry, _column_widths).print_table(
110 + self._indent_row_names_based_on_depth(
111 + filtered_summary_table,
112 + indent_style=lambda indent: "|" + "-" * indent + " "))
113 +
114 + def export_model_stats_table_csv(self, filename: str):
115 + df = pd.DataFrame([
116 + asdict(row)
117 + for _, row in self._flatten_stats_tree(self._model_stats_tree)
118 + ])
119 + df.to_csv(filename)
120 +
121 + def export_summary_stats_table_csv(self, filename: str):
122 + df = pd.DataFrame([
123 + asdict(row)
124 + for _, row in self._flatten_stats_tree(self._summary_stats_tree)
125 + ])
126 + df.to_csv(filename)
127 +
128 + def convert_stats_to_dict(self) -> str:
129 + return {
130 + "summary_stats":
131 + self._convert_stats_tree_to_dict(self._summary_stats_tree),
132 + "model_stats":
133 + self._convert_stats_tree_to_dict(self._model_stats_tree)
134 + }
135 +
136 + @staticmethod
137 + def _indent_row_names_based_on_depth(depths_rows: List[Tuple[int,
138 + StatsEntry]],
139 + indent_style: Union[Callable[[int],
140 + str],
141 + str] = " "):
142 + indented_rows = []
143 + for depth, row in depths_rows:
144 + if row.cuda_time_us == 0:
145 + continue
146 + indented_row = copy.deepcopy(row)
147 + indented_row.name = indent_string(indented_row.name, depth,
148 + indent_style)
149 + indented_rows.append(indented_row)
150 + return indented_rows
151 +
152 + def _build_correlation_map(self):
153 + self._kineto_event_correlation_map = defaultdict(list)
154 + for event in self._kineto_results.events():
155 + self._kineto_event_correlation_map[event.correlation_id()].append(
156 + event)
157 +
158 + def _build_module_tree(self):
159 + self._module_tree = []
160 + event_tree = self._kineto_results.experimental_event_tree()
161 +
162 + def _df_traversal(event: _ProfilerEvent,
163 + curr_node: Optional[_ModuleTreeNode] = None):
164 +
165 + # For the tensor parallel case for now only look at task 1
166 + if event.start_tid != 1:
167 + return
168 +
169 + if event_has_module(event):
170 + node = _ModuleTreeNode(event=event, parent=curr_node)
171 + if curr_node:
172 + curr_node.children.append(node)
173 + else:
174 + self._module_tree.append(node)
175 + curr_node = node
176 +
177 + is_leaf = (event.children is None or len(event.children) == 0)
178 + if is_leaf and curr_node:
179 + node = _ModuleTreeNode(
180 + event=event,
181 + parent=curr_node,
182 + trace=event_torch_op_stack_trace(
183 + event, until=lambda x: event_has_module(x)))
184 + curr_node.children.append(node)
185 + curr_node = node
186 +
187 + for child in event.children:
188 + _df_traversal(child, curr_node)
189 +
190 + for root in event_tree:
191 + _df_traversal(root)
192 +
193 + def _get_kineto_gpu_event(self, node: _ModuleTreeNode):
194 + if node.event.tag != _EventType.Kineto:
195 + return None
196 + correlated_kineto_events = self._kineto_event_correlation_map.get(
197 + node.event.correlation_id, [])
198 + iterator = (x for x in correlated_kineto_events
199 + if x.device_type() == DeviceType.CUDA
200 + and x.name() == node.event.name)
201 + return next(iterator, None)
202 +
203 + def _cumulative_cuda_time(self, node: _ModuleTreeNode):
204 + 'Return cuda time in microseconds'
205 +
206 + def _cumulative_cuda_time_recursive(node: _ModuleTreeNode):
207 + if node.is_leaf and (gpu_kineto_event :=
208 + self._get_kineto_gpu_event(node)):
209 + return gpu_kineto_event.duration_ns() / 1000.0
210 + else:
211 + cumulative_cuda_time = 0
212 + for child in node.children:
213 + cumulative_cuda_time += _cumulative_cuda_time_recursive(
214 + child)
215 + return cumulative_cuda_time
216 +
217 + return _cumulative_cuda_time_recursive(node)
218 +
219 + def _total_cuda_time(self):
220 + return sum(
221 + [self._cumulative_cuda_time(root) for root in self._module_tree])
222 +
223 + def _build_stats_trees(self):
224 + summary_dict: Dict[str, self.StatsTreeNode] = {}
225 + total_cuda_time = self._total_cuda_time()
226 +
227 + def pct_cuda_time(cuda_time_us):
228 + return (cuda_time_us / total_cuda_time) * 100
229 +
230 + def build_summary_stats_tree_df(
231 + node: _ModuleTreeNode,
232 + parent: Optional[_StatsTreeNode] = None,
233 + summary_trace: Tuple[str] = ()):
234 +
235 + if event_has_module(node.event):
236 + name = event_module_repr(node.event)
237 + cuda_time_us = self._cumulative_cuda_time(node)
238 + elif (gpu_kineto_event := self._get_kineto_gpu_event(node)):
239 + name = gpu_kineto_event.name()
240 + cuda_time_us = gpu_kineto_event.duration_ns() / 1000.0
241 + else:
242 + return None
243 +
244 + summary_trace = summary_trace + (name, )
245 + if summary_trace in summary_dict:
246 + entry = summary_dict[summary_trace].entry
247 + entry.cuda_time_us += cuda_time_us
248 + entry.invocations += 1
249 + entry.pct_cuda_time = pct_cuda_time(entry.cuda_time_us)
250 + else:
251 + new_node = _StatsTreeNode(entry=SummaryStatsEntry(
252 + name=name,
253 + cuda_time_us=cuda_time_us,
254 + pct_cuda_time=pct_cuda_time(cuda_time_us),
255 + invocations=1),
256 + children=[],
257 + parent=parent)
258 + if parent:
259 + parent.children.append(new_node)
260 + summary_dict[summary_trace] = new_node
261 +
262 + for child in node.children:
263 + build_summary_stats_tree_df(child, summary_dict[summary_trace],
264 + summary_trace)
265 +
266 + return summary_dict[summary_trace]
267 +
268 + self._summary_stats_tree = []
269 + for root in self._module_tree:
270 + self._summary_stats_tree.append(build_summary_stats_tree_df(root))
271 +
272 + def build_model_stats_tree_df(node: _ModuleTreeNode,
273 + parent: Optional[_StatsTreeNode] = None):
274 + if event_has_module(node.event, ):
275 + name = event_module_repr(node.event)
276 + cuda_time_us = self._cumulative_cuda_time(node)
277 + cpu_time_us = node.event.duration_time_ns / 1000
278 + trace = ""
279 + elif (gpu_kineto_event := self._get_kineto_gpu_event(node)):
280 + name = gpu_kineto_event.name()
281 + cuda_time_us = gpu_kineto_event.duration_ns() / 1000.0
282 + cpu_time_us = 0
283 + trace = node.trace
284 + else:
285 + return None
286 +
287 + new_node = _StatsTreeNode(entry=ModelStatsEntry(
288 + name=name,
289 + cpu_time_us=cpu_time_us,
290 + cuda_time_us=cuda_time_us,
291 + pct_cuda_time=pct_cuda_time(cuda_time_us),
292 + trace=trace),
293 + parent=parent,
294 + children=[])
295 + if parent:
296 + parent.children.append(new_node)
297 +
298 + for child in node.children:
299 + build_model_stats_tree_df(child, new_node)
300 +
301 + return new_node
302 +
303 + self._model_stats_tree = []
304 + for root in self._module_tree:
305 + self._model_stats_tree.append(build_model_stats_tree_df(root))
306 +
307 + def _flatten_stats_tree(
308 + self, tree: List[_StatsTreeNode]) -> List[Tuple[int, StatsEntry]]:
309 + entries: List[Tuple[int, StatsEntry]] = []
310 +
311 + def df_traversal(node: _StatsTreeNode, depth=0):
312 + entries.append((depth, node.entry))
313 + for child in node.children:
314 + df_traversal(child, depth=depth + 1)
315 +
316 + for root in tree:
317 + df_traversal(root)
318 +
319 + return entries
320 +
321 + def _convert_stats_tree_to_dict(self,
322 + tree: List[_StatsTreeNode]) -> List[Dict]:
323 + root_dicts: List[Dict] = []
324 +
325 + def df_traversal(node: _StatsTreeNode, curr_json_list: List[Dict]):
326 + curr_json_list.append({
327 + "entry": asdict(node.entry),
328 + "children": []
329 + })
330 + for child in node.children:
331 + df_traversal(child, curr_json_list[-1]["children"])
332 +
333 + for root in tree:
334 + df_traversal(root, root_dicts)
335 +
336 + return root_dicts
337 +
338 +
339 +class layerwise_profile(profile):
340 +
341 + def __init__(self):
342 + super().__init__(
343 + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
344 + record_shapes=True,
345 + with_stack=True,
346 + with_modules=True,
347 + experimental_config=_ExperimentalConfig(verbose=True))
348 +
349 + def __enter__(self):
350 + return super().__enter__()
351 +
352 + def __exit__(self, exc_type, exc_val, exc_tb):
353 + super().__exit__(exc_type, exc_val, exc_tb)
354 + self.results = LayerwiseProfileResults(self.profiler.kineto_results)
vllm/profiler/utils.py new
+145
@@ -0,0 +1,145 @@
1 +import dataclasses
2 +from typing import Callable, Dict, List, Type, Union
3 +
4 +from torch._C._profiler import _EventType, _ProfilerEvent, _TensorMetadata
5 +
6 +#
7 +# String / Print Manipulation
8 +#
9 +
10 +
11 +def trim_string_front(string, width):
12 + if len(string) > width:
13 + offset = len(string) - width + 3
14 + string = string[offset:]
15 + if len(string) > 3:
16 + string = "..." + string[3:]
17 + return string
18 +
19 +
20 +def trim_string_back(string, width):
21 + if len(string) > width:
22 + offset = len(string) - width + 3
23 + string = string[:-offset]
24 + if len(string) > 3:
25 + string = string + "..."
26 + return string
27 +
28 +
29 +class TablePrinter:
30 +
31 + def __init__(self, row_cls: Type[dataclasses.dataclass],
32 + column_widths: Dict[str, int]):
33 + self.row_cls = row_cls
34 + self.fieldnames = [x.name for x in dataclasses.fields(row_cls)]
35 + self.column_widths = column_widths
36 + assert set(self.column_widths.keys()) == set(self.fieldnames)
37 +
38 + def print_table(self, rows: List[dataclasses.dataclass]):
39 + self._print_header()
40 + self._print_line()
41 + for row in rows:
42 + self._print_row(row)
43 +
44 + def _print_header(self):
45 + for i, f in enumerate(self.fieldnames):
46 + last = (i == len(self.fieldnames) - 1)
47 + col_width = self.column_widths[f]
48 + print(trim_string_back(f, col_width).ljust(col_width),
49 + end=" | " if not last else "\n")
50 +
51 + def _print_row(self, row):
52 + assert isinstance(row, self.row_cls)
53 +
54 + for i, f in enumerate(self.fieldnames):
55 + last = (i == len(self.fieldnames) - 1)
56 + col_width = self.column_widths[f]
57 + val = getattr(row, f)
58 +
59 + val_str = ""
60 + if isinstance(val, str):
61 + val_str = trim_string_back(val, col_width).ljust(col_width)
62 + elif type(val) in [float, int]:
63 + val_str = f"{float(val):>.2f}".rjust(col_width)
64 + else:
65 + val_str = f"{val}".rjust(col_width)
66 + print(val_str, end=" | " if not last else "\n")
67 +
68 + def _print_line(self):
69 + total_col_width = 0
70 + for column_width in self.column_widths.values():
71 + total_col_width += column_width
72 + print("=" * (total_col_width + 3 * (len(self.column_widths) - 1)))
73 +
74 +
75 +def indent_string(string: str,
76 + indent: int,
77 + indent_style: Union[Callable[[int], str], str] = " ") -> str:
78 + if indent:
79 + if isinstance(indent_style, str):
80 + return indent_style * indent + string
81 + else:
82 + return indent_style(indent) + string
83 + else:
84 + return string
85 +
86 +
87 +#
88 +# _ProfilerEvent utils
89 +#
90 +
91 +
92 +def event_has_module(event: _ProfilerEvent) -> bool:
93 + event_type, typed_event = event.typed
94 + if event_type == _EventType.PyCall:
95 + return typed_event.module is not None
96 + return False
97 +
98 +
99 +def event_is_torch_op(event: _ProfilerEvent) -> bool:
100 + return event.tag == _EventType.TorchOp
101 +
102 +
103 +def event_arg_repr(arg) -> str:
104 + if arg is None or type(arg) in [float, int, bool, str]:
105 + return f"{arg}"
106 + elif isinstance(arg, list):
107 + return f"[{', '.join([event_arg_repr(x) for x in arg])}]"
108 + elif isinstance(arg, tuple):
109 + return f"({', '.join([event_arg_repr(x) for x in arg])})"
110 + else:
111 + assert isinstance(arg,
112 + _TensorMetadata), f"Unsupported type: {type(arg)}"
113 + sizes_str = ', '.join([str(x) for x in arg.sizes])
114 + return f"{str(arg.dtype).replace('torch.', '')}[{sizes_str}]"
115 +
116 +
117 +def event_torch_op_repr(event: _ProfilerEvent) -> str:
118 + assert event.tag == _EventType.TorchOp
119 + args_str = ', '.join([event_arg_repr(x) for x in event.typed[1].inputs])
120 + return f"{event.name}({args_str})".replace("aten::", "")
121 +
122 +
123 +def event_module_repr(event: _ProfilerEvent) -> str:
124 + assert event_has_module(event)
125 + module = event.typed[1].module
126 + if module.parameters and len(module.parameters) > 0:
127 + args_str = ', '.join(
128 + [f'{x[0]}={event_arg_repr(x[1])}' for x in module.parameters])
129 + return f"{module.cls_name}({args_str})"
130 + else:
131 + return module.cls_name
132 +
133 +
134 +def event_torch_op_stack_trace(curr_event: _ProfilerEvent,
135 + until: Callable[[_ProfilerEvent], bool]) -> str:
136 + trace = ""
137 + curr_event = curr_event.parent
138 + while curr_event and not until(curr_event):
139 + if event_is_torch_op(curr_event):
140 + if len(trace) > 0:
141 + trace += " <- "
142 + trace += event_torch_op_repr(curr_event)
143 + curr_event = curr_event.parent
144 +
145 + return trace
vllm/worker/model_runner.py
+4 -4
@@ -1742,10 +1742,13 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1742 return [output]
1743
1744
1745 -class CUDAGraphRunner:
1745 +# NOTE: this is nn.Module so the profiler can properly capture/group
1746 +# kernels calls made within the graph
1747 +class CUDAGraphRunner(nn.Module):
1748
1749 def __init__(self, model: nn.Module, backend_name: str,
1750 attn_state: AttentionState, is_encoder_decoder_model: bool):
1751 + super().__init__()
1752 self.model = model
1753 self.backend_name = backend_name
1754 self.attn_state = attn_state
@@ -1892,9 +1895,6 @@ class CUDAGraphRunner:
1895
1896 return self.output_buffers
1897
1895 - def __call__(self, *args, **kwargs):
1896 - return self.forward(*args, **kwargs)
1897 -
1898
1899 def _get_graph_batch_size(batch_size: int) -> int:
1900 """Returns the padded batch size given actual batch size.