A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[misc] split engine_model into json file for nsys profile tool (#23117)
Signed-off-by: Grace Ho <grho@nvidia.com> Signed-off-by: Grace Ho <146482179+gracehonv@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Grace Ho committed
Aug 19, 2025 at 00:44 UTC
01a08739e006c9711fd9edf23fbe7cf23d43df2d
3 files changed
+135
-186
tools/profiler/nsys_profile_tools/README.md
+26
-27
@@ -36,8 +36,7 @@ profiling and analyzing nsys profile output.
36
## Notes
37
38
- Make sure you have pandas installed.
39
-- Make sure nsys is installed, and specify the path to the `nsys` command with
40
- `--nsys_cmd` if it is not in your PATH.
39
+- Make sure [nsys](https://developer.nvidia.com/nsight-systems/get-started) is installed, and specify the path to the `nsys` command with `--nsys_cmd` if it is not in your PATH.
40
- For more details on available engines and models, see the help string in
41
the script or run:
42
@@ -135,34 +134,31 @@ time which would cause a difference for the overall category.
134
135
## Example 3: add new classification for a new model
136
138
-Suppose there's a new model ABC that is available for engine DEF, and say there
139
-are 4 kernels to be classified into "gemm" and "attn", where the gemm kernels
137
+To create a new engine DEF with model ABC, just add another json file in the same directory as
138
+gputrc2graph.py with the same format as the other json files. The script will automatically pick up all the json files in the same directory as engine/model specifications.
139
+
140
+Then, for this new model, suppose there are 4 kernels to be classified into "gemm" and "attn", where the gemm kernels
141
have names with "*H*" or "*I*" in them, and attn kernels have names with "*J*"
141
-or "*K*" in them, add a new entry like so:
142
-
143
-```python
144
-engine_model = {
145
- 'DEF': {
146
- 'ABC': {
147
- 'layer_anno': {
148
- 'Stage': {
149
- '.*': 'layer',
150
- },
151
- 'Substage': {
152
- 'H|I': 'gemm',
153
- 'J|K': 'attn',
154
- 'CUDA mem': 'non-gpu-H_D_memops',
155
- '.*': 'misc'
156
- }
157
- }
158
- },
159
- }
160
- 'vllm': {...}
142
+or "*K*" in them, just add another .json file in the same directory as
143
+gputrc2graph.py with the same format as the other json files, like the following:
144
+
145
+```json
146
+{
147
+ "DEF": {
148
+ "ABC": {
149
+ "H|I": "gemm",
150
+ "J|K": "attn",
151
+ "CUDA mem": "non-gpu-H_D_memops",
152
+ ".*": "misc"
153
+ }
154
+ }
155
+}
156
```
157
163
-Basically Substage is a dictionary with a list of key/value pairs, where the
164
-keys are regex's of the kernel names to be classified, and values are the
165
-classification bins which one wishes to compare across engines/models.
158
+Each entry in the dictionary consists of:
159
+
160
+- key: a regex used to classify the kernels
161
+- value: the category to classify the kernels into.
162
163
The last 2 entries are common for all engine/models, consisting of CUDA memory
164
operations and a 'misc' for anything that's leftover and can't be classified.
@@ -173,3 +169,6 @@ like the following:
169
```bash
170
--infile new.nsys-rep,DEF,ABC,<runtime>
171
```
172
+
173
+If the engine_DEF.json file already exists, just add the model as a new node in
174
+the existing engine file, after the other models.
tools/profiler/nsys_profile_tools/gputrc2graph.py
+46
-159
@@ -15,132 +15,18 @@ logger = logging.getLogger(__name__)
15
16
17
# helper data class for annotating kernels
18
-class EngineModelData:
19
- # engine + model mappings
20
- engine_model = {
21
- 'vllm': {
22
- 'llama': {
23
- 'layer_anno': {
24
- 'Stage': {
25
- '.*': 'layer',
26
- },
27
- 'Substage': {
28
- 'gemm': 'gemm',
29
- 'fused_moe_kernel|GroupProblemShape|group_gemm_starts':
30
- 'moe_gemm', #llama4
31
- 'moe|sigmoid': 'moe', #llama4
32
- 'CatArrayBatched|prepare_inputs': 'prepare_next',
33
- 'flash': 'attn',
34
- 'ncclDevKernel|cross_device_reduce':
35
- 'nccl_and_custom_ar',
36
- '_norm_': 'norm',
37
- 'act_and_mul_': 'silu',
38
- 'rotary_embedding_kernel': 'rope',
39
- 'SoftMax': 'softmax',
40
- 'elementwise': 'elementwise',
41
- 'fp8_quant': 'quantize',
42
- 'reduce_kernel': 'reduce',
43
- 'triton': 'triton_kernel',
44
- 'CUDA mem': 'non-gpu-H_D_memops',
45
- '.*': 'misc'
46
- }
47
- }
48
- },
49
- 'ds': {
50
- 'layer_anno': {
51
- 'Stage': {
52
- '.*': 'layer',
53
- },
54
- 'Substage': {
55
- 'block_fp8|gemm_fp8_blockwise':
56
- 'block_fp8_gemm',
57
- 'fused_moe_kernel|_group_gemm|GroupProblemShape|GemmUniversal':
58
- 'moe_gemm',
59
- 'gemm|matmul|nvjet':
60
- 'gemm',
61
- 'moe|sigmoid|expert':
62
- 'moe',
63
- '_fwd_|FlashAttn|_mla_|_attn_':
64
- 'attn',
65
- 'CatArrayBatched':
66
- 'prepare_next',
67
- 'ncclDevKernel|cross_device_reduce':
68
- 'nccl_and_custom_ar',
69
- 'Norm|_norm_':
70
- 'norm',
71
- 'sbtopk':
72
- 'topk',
73
- 'act_and_mul_':
74
- 'activation',
75
- 'compute_position_kernel':
76
- 'rope',
77
- 'elementwise':
78
- 'elementwise',
79
- 'fp8_quant|quant_fp8|cvt_fp16_to_fp4':
80
- 'quantize',
81
- 'reduce':
82
- 'reduce',
83
- 'SoftMax':
84
- 'softmax',
85
- 'triton':
86
- 'triton_kernel',
87
- 'CUDA mem':
88
- 'non-gpu-H_D_memops',
89
- '.*':
90
- 'misc'
91
- }
92
- }
93
- },
94
- 'gpt-oss': {
95
- 'layer_anno': {
96
- 'Stage': {
97
- '.*': 'layer',
98
- },
99
- 'Substage': {
100
- 'block_fp8|gemm_fp8_blockwise':
101
- 'block_fp8_gemm',
102
- 'fused_moe_kernel|_group_gemm|GroupProblemShape|GemmUniversal|bmm_'
103
- # this section is triton_moe_gemm
104
- '|matmul_ogs_|_topk_forward|_combined_routing'
105
- '|_sum_bitmatrix_rows|_compute_writeback_idx':
106
- 'moe_gemm',
107
- 'gemm|matmul|nvjet':
108
- 'gemm',
109
- 'moe|sigmoid|expert|splitKreduce':
110
- 'moe',
111
- '_fwd_|FlashAttn|_mla_|_attn_|_flash_|flash::prepare_varlen|fmha':
112
- 'attn',
113
- 'CatArrayBatched':
114
- 'prepare_next',
115
- 'ncclDevKernel|cross_device_reduce':
116
- 'nccl_and_custom_ar',
117
- 'Norm|_norm_':
118
- 'norm',
119
- 'sbtopk':
120
- 'topk',
121
- 'act_and_mul_':
122
- 'activation',
123
- 'compute_position_kernel':
124
- 'rope',
125
- 'elementwise':
126
- 'elementwise',
127
- 'fp8_quant|quant_fp8|cvt_fp16_to_fp4|quantize':
128
- 'quantize',
129
- 'reduce':
130
- 'reduce',
131
- 'SoftMax':
132
- 'softmax',
133
- 'triton':
134
- 'triton_kernel',
135
- 'CUDA mem':
136
- 'non-gpu-H_D_memops',
137
- '.*':
138
- 'misc'
139
- }
140
- }
141
- }
142
- },
143
- }
18
+def load_engine_model():
19
+ """ returns engine_model built from all json files in the current dir """
20
+ import glob
21
+ import json
22
+ engine_model = {}
23
+
24
+ json_files = glob.glob(
25
+ os.path.join(os.path.dirname(__file__) or ".", "*.json"))
26
+ for fname in json_files:
27
+ with open(fname, encoding="utf-8") as f:
28
+ engine_model.update(json.load(f))
29
+ return engine_model
30
31
32
class GPUTrace2Graph:
@@ -148,8 +34,7 @@ class GPUTrace2Graph:
34
Parses output of nsys report, generates csv and bar chart output
35
"""
36
151
- def __init__(self, nsys_cmd):
152
- self.nsys_cmd = nsys_cmd
37
+ def __init__(self):
38
import pandas as pd # avoid importing till needed
39
self.pd = pd
40
self.pd.options.mode.copy_on_write = True
@@ -227,7 +112,7 @@ class GPUTrace2Graph:
112
title = 'Model_Engine'
113
x = 'Model_Engine'
114
y = 'Elapsed Time (sec)'
230
- color = 'Substage'
115
+ color = 'Category'
116
""" generate kernel mapping table """
117
# Sort Model_Engine categories by last field after underscore
118
df['Model_Engine'] = self.pd.Categorical(
@@ -249,14 +134,13 @@ class GPUTrace2Graph:
134
Generate data table with columns per Model_Engine into result.html
135
"""
136
pivot_df = df.pivot_table(values='Elapsed Time (sec)',
252
- index='Substage',
137
+ index='Category',
138
columns='Model_Engine',
139
aggfunc='sum',
140
observed=False).round(2)
141
# Add sum row at bottom
142
pivot_df.loc['total_elapsed_sec'] = pivot_df.sum()
143
pivot_df.fillna('').to_html('temp.html')
259
- print('got')
144
with (open(f'{output_name}.html', 'a', encoding='utf-8') as
145
outfile, open('temp.html', encoding='utf-8') as infile):
146
outfile.write(infile.read())
@@ -264,23 +148,22 @@ class GPUTrace2Graph:
148
149
print(f'Finished generating: \n'
150
f' {output_name}.html for stack bar chart \n'
267
- f' {output_name}.csv for Kernel-Substage mapping')
151
+ f' {output_name}.csv for Kernel-Category mapping')
152
153
def anno_gpu_kernname(self, df, mapping):
270
- """ add "stage" and "substage" columns """
154
+ """ add "Category" column """
155
272
- def anno_gpu_kernname_helper(name, stage):
273
- for kern_name, val in mapping['layer_anno'][stage].items():
156
+ def anno_gpu_kernname_helper(name):
157
+ for kern_name, val in mapping.items():
158
if re.search(kern_name, name):
159
return val
160
277
- for stage in ['Stage', 'Substage']:
278
- df[stage] = df['Name'].apply(anno_gpu_kernname_helper, stage=stage)
161
+ df['Category'] = df['Name'].apply(anno_gpu_kernname_helper)
162
163
def make_nongpu_row(self, df, nongpu_sec):
164
""" this will append non-gpu time entry at end of df """
165
nongpu_row = self.pd.DataFrame([df.iloc[-1]])
283
- nongpu_row['Substage'] = nongpu_row['Name'] = 'CPU(non-GPU)'
166
+ nongpu_row['Category'] = nongpu_row['Name'] = 'CPU(non-GPU)'
167
nongpu_row['Instances'] = 1
168
nongpu_row['Elapsed Time (sec)'] = nongpu_sec
169
return (nongpu_row)
@@ -302,7 +185,7 @@ class GPUTrace2Graph:
185
logger.info('generating %s', new_file)
186
return True
187
305
- def gen_sum_file(self, file):
188
+ def gen_sum_file(self, file, nsys_cmd):
189
"""
190
generates sum file from nsys trace with times per kernel and
191
returns the name of the sum file
@@ -318,17 +201,21 @@ class GPUTrace2Graph:
201
sum_file = f'{file_dir}/{file_name}_cuda_gpu_kernel_tracesum.csv'
202
if self.should_gen_file(nsys_stats_file, file):
203
cmd = [
321
- self.nsys_cmd, 'stats', '-r', 'cuda_gpu_trace', file, '-o',
204
+ nsys_cmd, 'stats', '-r', 'cuda_gpu_trace', file, '-o',
205
f'{file_dir}/{file_name}'
206
]
207
cmd_str = ' '.join(cmd)
208
logger.info('+ %s', cmd_str)
209
+ # estimate time based on calibrated 240M/min
210
+ file_size_mb = os.path.getsize(file) / 1e6
211
+ logger.info(
212
+ 'nsys stats for %.2f MB file expected to take %.2f min',
213
+ file_size_mb, file_size_mb / 240)
214
try:
327
- subprocess.run(cmd)
215
+ subprocess.run(cmd, check=True)
216
except Exception:
329
- logger.error(
330
- "%s failed, specify --nsys_cmd for correct nsys path",
331
- cmd_str)
217
+ logger.error("%s failed; Use --nsys_cmd to specify nsys path",
218
+ cmd_str)
219
exit(1)
220
logger.info('generating non-overalapped sum %s', sum_file)
221
self.gen_nonoverlapped_sum_from_gputrace(nsys_stats_file, sum_file)
@@ -336,7 +223,7 @@ class GPUTrace2Graph:
223
logger.info('Finished generating %s', sum_file)
224
return sum_file
225
339
- def gen_graph(self, in_file, out_dir, title):
226
+ def gen_graph(self, in_file, out_dir, title, nsys_cmd, engine_model):
227
""" generates graph and csv file from in_file into out_dir """
228
# Initialize an empty DataFrame to store combined data
229
combined_df = self.pd.DataFrame()
@@ -345,17 +232,16 @@ class GPUTrace2Graph:
232
file_name = os.path.basename(file)
233
if not file_dir:
234
file_dir = '.'
348
- sum_file = self.gen_sum_file(file)
235
+ sum_file = self.gen_sum_file(file, nsys_cmd)
236
# read kernel summary file
237
df = self.pd.read_csv(sum_file)
238
# annotate kernel to their categories
352
- assert EngineModelData.engine_model.get(engine)
353
- assert EngineModelData.engine_model[engine].get(model)
239
+ assert engine_model.get(engine), f'engine {engine} unknown'
240
+ assert engine_model[engine].get(model), f'model {model} unknown'
241
# remove nsys-rep from file_name for shorter x-label
242
file_name = file_name.replace('.nsys-rep', '')
243
df['Model_Engine'] = f'{model}_{engine}_{file_name}_{idx}'
357
- self.anno_gpu_kernname(df,
358
- EngineModelData.engine_model[engine][model])
244
+ self.anno_gpu_kernname(df, engine_model[engine][model])
245
# patch in non-gpu time
246
gpu_sec = round(df['Elapsed Time (sec)'].sum(), 1)
247
total_sec = round(float(total_sec), 1)
@@ -393,12 +279,12 @@ def main():
279
"--out_dir results/ --title \"Model=gpt-oss vLLM chart\""),
280
formatter_class=argparse.RawDescriptionHelpFormatter)
281
396
- # Build help string showing available engine/model combinations
397
- engine_model_help = []
398
- for engine, models in EngineModelData.engine_model.items():
399
- model_list = list(models.keys())
400
- engine_model_help.append(f"{engine}:[{','.join(model_list)}]")
401
- engine_model_str = ' '.join(engine_model_help)
282
+ # load supported engine_model
283
+ engine_model_supported = load_engine_model()
284
+ # Get a string representation of supported engine/model combinations
285
+ engine_model_supported_str = ', '.join(
286
+ f"{engine}:[{', '.join(models.keys())}]"
287
+ for engine, models in engine_model_supported.items())
288
parser.add_argument(
289
'--in_file',
290
type=parse_tuple,
@@ -408,7 +294,7 @@ def main():
294
'separated by space. Elapsed_nonprofiled_sec is runtime without '
295
'profiling used to calculate non-gpu time. Specify 0 to use '
296
'elapsed time from nsys-rep but that might inflate non-gpu time. '
411
- f'Available engine:[model] are: {engine_model_str} '
297
+ f'Available engine:[model] are: {engine_model_supported_str} '
298
f'Example: --infile d1.nsys-rep,vllm,llama,100 '
299
'd2.nsys-rep,vllm,gpt-oss,102'),
300
required=True)
@@ -418,8 +304,9 @@ def main():
304
help=('nsys cmd, e.g. /usr/bin/nsys, Default: nsys'),
305
default="nsys")
306
args = parser.parse_args()
421
- gputrace = GPUTrace2Graph(args.nsys_cmd)
422
- gputrace.gen_graph(args.in_file, args.out_dir, args.title)
307
+ gputrace = GPUTrace2Graph()
308
+ gputrace.gen_graph(args.in_file, args.out_dir, args.title, args.nsys_cmd,
309
+ engine_model_supported)
310
311
312
if __name__ == '__main__':
tools/profiler/nsys_profile_tools/vllm_engine_model.json
new
+63
@@ -0,0 +1,63 @@
1
+{
2
+ "vllm": {
3
+ "llama": {
4
+ "fused_moe_kernel|GroupProblemShape|group_gemm_starts|bmm_|GemmUniversal": "moe_gemm",
5
+ "gemm|nvjet": "gemm",
6
+ "moe|sigmoid": "moe",
7
+ "CatArrayBatched|prepare_inputs": "prepare_next",
8
+ "ncclDevKernel|cross_device_reduce": "nccl_and_custom_ar",
9
+ "_norm_|Norm": "norm",
10
+ "act_and_mul_": "activation",
11
+ "Rotary": "rope",
12
+ "SoftMax": "softmax",
13
+ "flash|fmha": "attn",
14
+ "elementwise": "elementwise",
15
+ "fp8_quant|cvt_": "quantize",
16
+ "reduce_kernel": "reduce",
17
+ "triton": "triton_kernel",
18
+ "CUDA mem": "non-gpu-H_D_memops",
19
+ ".*": "misc"
20
+ },
21
+ "ds": {
22
+ "block_fp8|gemm_fp8_blockwise": "block_fp8_gemm",
23
+ "fused_moe_kernel|_group_gemm|GroupProblemShape|GemmUniversal|bmm_": "moe_gemm",
24
+ "gemm|matmul|nvjet": "gemm",
25
+ "moe|sigmoid|expert": "moe",
26
+ "CatArrayBatched": "prepare_next",
27
+ "ncclDevKernel|cross_device_reduce": "nccl_and_custom_ar",
28
+ "Norm|_norm_": "norm",
29
+ "sbtopk": "topk",
30
+ "act_and_mul_": "activation",
31
+ "compute_position_kernel": "rope",
32
+ "elementwise": "elementwise",
33
+ "fp8_quant|quant_fp8|cvt_": "quantize",
34
+ "reduce": "reduce",
35
+ "SoftMax": "softmax",
36
+ "_fwd_|FlashAttn|_mla_|_attn_|fmha": "attn",
37
+ "triton": "triton_kernel",
38
+ "topk": "topk",
39
+ "CUDA mem": "non-gpu-H_D_memops",
40
+ ".*": "misc"
41
+ },
42
+ "gpt-oss": {
43
+ "block_fp8|gemm_fp8_blockwise": "block_fp8_gemm",
44
+ "fused_moe_kernel|_group_gemm|GroupProblemShape|GemmUniversal|bmm_|matmul_ogs_|_topk_forward|_combined_routing|_sum_bitmatrix_rows|_compute_writeback_idx": "moe_gemm",
45
+ "gemm|matmul|nvjet": "gemm",
46
+ "moe|sigmoid|expert|splitKreduce": "moe",
47
+ "CatArrayBatched": "prepare_next",
48
+ "ncclDevKernel|cross_device_reduce": "nccl_and_custom_ar",
49
+ "Norm|_norm_": "norm",
50
+ "topk": "topk",
51
+ "act_and_mul_": "activation",
52
+ "compute_position_kernel": "rope",
53
+ "elementwise": "elementwise",
54
+ "fp8_quant|quant_fp8|cvt_|quantize": "quantize",
55
+ "reduce": "reduce",
56
+ "SoftMax": "softmax",
57
+ "_fwd_|FlashAttn|_mla_|_attn_|_flash_|flash::prepare_varlen|fmha": "attn",
58
+ "triton": "triton_kernel",
59
+ "CUDA mem": "non-gpu-H_D_memops",
60
+ ".*": "misc"
61
+ }
62
+ }
63
+}
\ No newline at end of file