A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[misc] Layerwise profile updates (#10242)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Dec 16, 2024 at 13:14 UTC
efbce85f4d375d7851a491a0126a224e25d9f91d
5 files changed
+314
-47
.buildkite/test-pipeline.yaml
+1
-1
@@ -201,7 +201,7 @@ steps:
201
- python3 offline_inference_classification.py
202
- python3 offline_inference_embedding.py
203
- python3 offline_inference_scoring.py
204
- - python3 offline_profile.py --model facebook/opt-125m
204
+ - python3 offline_profile.py --model facebook/opt-125m run_num_steps --num-steps 2
205
206
- label: Prefix Caching Test # 9min
207
mirror_hardwares: [amd]
examples/offline_profile.py
+206
-30
@@ -4,9 +4,10 @@ import os
4
import sys
5
from argparse import RawTextHelpFormatter
6
from dataclasses import asdict, dataclass
7
-from typing import Optional
7
+from typing import Any, Dict, Generator, List, Optional, TypeAlias
8
9
import torch
10
+import tqdm
11
12
from vllm import LLM, SamplingParams
13
from vllm.engine.arg_utils import EngineArgs
@@ -15,16 +16,21 @@ from vllm.utils import FlexibleArgumentParser
16
17
BATCH_SIZE_DEFAULT = 1
18
PROMPT_LEN_DEFAULT = 256
18
-OUTPUT_LEN_DEFAULT = 2
19
20
21
@dataclass
22
class ProfileContext:
23
engine_args: EngineArgs
24
prompt_len: int
25
- output_len: int
25
batch_size: int
27
- save_chrome_traces_folder: Optional[str]
26
+
27
+ # The profiler can run in 2 modes,
28
+ # 1. Run profiler for user specified num_steps
29
+ num_steps: Optional[int] = None
30
+ # 2. Run profiler until all requests complete
31
+ complete_num_requests_per_step: Optional[int] = None
32
+
33
+ save_chrome_traces_folder: Optional[str] = None
34
35
36
def get_dtype(dtype: str):
@@ -34,23 +40,155 @@ def get_dtype(dtype: str):
40
return dtype
41
42
43
+OutputLen_NumReqs_Map: TypeAlias = Dict[int, int]
44
+def compute_request_output_lengths(batch_size: int, step_requests: List[int]) \
45
+ -> OutputLen_NumReqs_Map:
46
+ """
47
+ Given the number of requests, batch_size, and the number of requests
48
+ that each engine-step should process, step_requests, determine the
49
+ output lengths of the requests such that step_request is honoured.
50
+
51
+ Example:
52
+ if batch size = 128 and step_request = [128, 128, 96, 64, 32, 1]
53
+ then return,
54
+ {2 : 32, 3 : 32, 4 : 32, 5 : 31, 6 : 1}, meaning,
55
+ 32 requests should have output length 2,
56
+ 32 requests should have output length 3,
57
+ 32 requests should have output length 4,
58
+ 31 requests should have output length 5,
59
+ 1 request should have output length 6.
60
+
61
+ Args:
62
+ batch_size (int): Number of requests submitted for profile. This is
63
+ args.batch_size.
64
+ step_requests (List[int]): step_requests[i] is the number of requests
65
+ that the ith engine step should process.
66
+
67
+ Returns:
68
+ OutputLen_NumReqs_Map : A dictionary with output-length as keys and the
69
+ number of requests required to have that output-length as values.
70
+ """
71
+ ol_nr: OutputLen_NumReqs_Map = {}
72
+
73
+ # Number of request that are assigned an output-length
74
+ num_reqs_assigned: int = 0
75
+ num_steps: int = len(step_requests)
76
+
77
+ # sanity check. The first step (prefill-step), must process all requests.
78
+ assert step_requests[0] == batch_size
79
+
80
+ # Begin assignments from the last step.
81
+ output_length: int = num_steps
82
+ for num_requests_at_step in reversed(step_requests):
83
+ if num_reqs_assigned == batch_size:
84
+ break
85
+
86
+ assert num_reqs_assigned < batch_size
87
+
88
+ # Remove the number of requests that have been determined
89
+ # to participate in this step and beyond.
90
+ num_reqs_unassigned_at_step = num_requests_at_step - num_reqs_assigned
91
+ assert num_reqs_unassigned_at_step >= 0
92
+
93
+ if num_reqs_unassigned_at_step > 0:
94
+ ol_nr[output_length] = num_reqs_unassigned_at_step
95
+ num_reqs_assigned += num_reqs_unassigned_at_step
96
+
97
+ output_length -= 1
98
+
99
+ # sanity checks.
100
+ assert sum(ol_nr.values()) == batch_size, \
101
+ ("Number of requests in output-length assignment does not match "
102
+ f"batch-size.\n batch size {batch_size} - "
103
+ f"step requests {step_requests} - assignments {ol_nr}")
104
+
105
+ # Check that the output-length is in [1, num-steps]. Output length must be
106
+ # at least 1 as all requests must participate in the prefill-step.
107
+ assert all(ol >= 1 and ol <= num_steps for ol in ol_nr), \
108
+ ("Output lengths of requests should be in range "
109
+ f"[1, num-engine-steps].\n batch size {batch_size} - "
110
+ f"step requests {step_requests} - assignments {ol_nr}")
111
+
112
+ return ol_nr
113
+
114
+
115
+def determine_requests_per_step(context: ProfileContext) -> List[int]:
116
+ """
117
+ Determine number of requests each engine step should process.
118
+ If context.num_steps is set, then all engine steps process the
119
+ same number of requests and the output list is of length
120
+ context.num_steps.
121
+
122
+ If context.complete_num_requests_per_step is set, then each decode step
123
+ processes fewer and fewer requests until there are no requests to process.
124
+ In this case, the output list is as big as the number of steps
125
+ required to process all requests.
126
+
127
+ Args:
128
+ context: ProfileContext object.
129
+
130
+ Returns:
131
+ List[int]: Number of requests to process for all engine-steps.
132
+ output[i], contains the number of requests that the ith step
133
+ should process.
134
+ """
135
+ if context.num_steps:
136
+ # All requests must run until num_engine_steps. This implies
137
+ # that their output lengths must be equal to num_engine_steps.
138
+ return [context.batch_size] * context.num_steps
139
+
140
+ assert context.complete_num_requests_per_step and \
141
+ context.complete_num_requests_per_step > 0, \
142
+ (f"Expected a positive complete_num_requests_per_step argument."
143
+ f"Instead got {context.complete_num_requests_per_step}")
144
+
145
+ # We start dropping after the first decode step.
146
+ step_requests = [
147
+ context.batch_size, # prefill
148
+ context.batch_size, # decode
149
+ ]
150
+
151
+ num_running_requests = context.batch_size
152
+ num_running_requests -= context.complete_num_requests_per_step
153
+ while num_running_requests > 0:
154
+ step_requests.append(num_running_requests)
155
+ num_running_requests -= context.complete_num_requests_per_step
156
+
157
+ if step_requests[-1] != 1:
158
+ # have 1 request running at the last step. This is often
159
+ # useful
160
+ step_requests.append(1)
161
+
162
+ return step_requests
163
+
164
+
165
def run_profile(context: ProfileContext, csv_output: Optional[str],
166
json_output: Optional[str]):
167
print("Run profile with:")
168
for key, value in asdict(context).items():
169
print(f" {key} = {value}")
170
171
+ requests_per_step: List[int] = determine_requests_per_step(context)
172
+
173
+ ol_nr: OutputLen_NumReqs_Map = compute_request_output_lengths(
174
+ context.batch_size, requests_per_step)
175
+
176
+ num_steps_to_profile: int = len(requests_per_step)
177
+ max_output_len: int = max(ol_nr.keys())
178
+ assert max_output_len >= 1
179
+
180
# Create sampling params
44
- sampling_params = SamplingParams(temperature=0.8,
45
- top_p=0.95,
46
- max_tokens=args.output_len,
47
- ignore_eos=True)
181
+ sampling_params = SamplingParams(
182
+ temperature=0.8,
183
+ top_p=0.95,
184
+ # max_tokens is set on a per-request basis.
185
+ max_tokens=None,
186
+ ignore_eos=True)
187
188
# Create LLM
189
llm = LLM(**asdict(context.engine_args))
190
batch_size = context.batch_size
191
prompt_len = context.prompt_len
53
- output_len = context.output_len
192
193
scheduler_config = llm.llm_engine.scheduler_config
194
max_model_len = llm.llm_engine.model_config.max_model_len
@@ -65,7 +203,7 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
203
f"choose a smaller batch size or prompt length, or increase "
204
f"--max-num-batched-tokens")
205
sys.exit(-1)
68
- if batch_size >= max_num_seqs:
206
+ if batch_size > max_num_seqs:
207
print(
208
f"ERROR: chosen batch_size ({batch_size}) is larger than "
209
f"max_num_seqs ({max_num_seqs}) and therefore cannot be run in a "
@@ -73,16 +211,26 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
211
sys.exit(-1)
212
print("llm.llm_engine.model_config.max_model_len: ",
213
llm.llm_engine.model_config.max_model_len)
76
- if prompt_len + output_len > llm.llm_engine.model_config.max_model_len:
77
- print(
78
- f"ERROR: chosen prompt_len + output_len ({prompt_len} + "
79
- f"{output_len} = {prompt_len + output_len}) is larger than the "
80
- f"model's max_model_len ({max_model_len}), please choose a smaller "
81
- f"prompt_len or output_len, or increase --max-model-len")
214
+ if prompt_len + max_output_len > llm.llm_engine.model_config.max_model_len:
215
+ print(f"ERROR: chosen prompt_len + max_output_len ({prompt_len} + "
216
+ f"{max_output_len} = {prompt_len + max_output_len}) is larger "
217
+ f"than the model's max_model_len ({max_model_len}), please "
218
+ f"choose a smaller prompt_len or max_output_len, or increase "
219
+ f"--max-model-len")
220
sys.exit(-1)
221
222
def add_requests():
223
+
224
+ def get_output_len_generator() -> Generator[int, Any, Any]:
225
+ for output_len, num_reqs in ol_nr.items():
226
+ for _ in range(num_reqs):
227
+ yield output_len
228
+
229
+ output_len_generator = get_output_len_generator()
230
for i in range(batch_size):
231
+ sampling_params.max_tokens = next(output_len_generator)
232
+ assert isinstance(sampling_params.max_tokens, int)
233
+
234
prompt_token_ids = torch.randint(
235
llm.llm_engine.model_config.get_vocab_size(),
236
size=(prompt_len, )).tolist()
@@ -110,8 +258,11 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
258
llm.llm_engine.step() # First step is prefill
259
260
decode_profs = []
113
- for x in range(args.output_len - 1):
114
- with layerwise_profile() as decode_prof:
261
+ for _ in tqdm.tqdm(range(num_steps_to_profile - 1)):
262
+ num_running_seqs = llm.llm_engine.scheduler[
263
+ 0].get_num_unfinished_seq_groups()
264
+ with layerwise_profile(
265
+ num_running_seqs=num_running_seqs) as decode_prof:
266
llm.llm_engine.step()
267
decode_profs.append(decode_prof)
268
@@ -154,7 +305,8 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
305
decode_results_list[0].print_summary_table()
306
307
if csv_output:
157
- csv_filename_base = csv_output.rstrip(".csv")
308
+ csv_filename_base = csv_output[:-4] \
309
+ if csv_output.endswith('.csv') else csv_output
310
prefill_results.export_model_stats_table_csv(
311
csv_filename_base + "_prefill_model_table.csv")
312
prefill_results.export_summary_stats_table_csv(
@@ -187,10 +339,10 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
339
for idx, dr in enumerate(decode_results_list):
340
json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
341
190
- for idx, dr in enumerate(decode_results_list[1:]):
191
- json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
192
-
193
- with open(json_output.rstrip(".json") + ".json", "w+") as f:
342
+ # Add .json to json_output filename if it doesn't exist already.
343
+ json_output_file = json_output if json_output.endswith(
344
+ '.json') else json_output + '.json'
345
+ with open(json_output_file, "w+") as f:
346
json.dump(json_dict, f, indent=2)
347
pass
348
@@ -214,7 +366,7 @@ Profile a model
366
python examples/offline_profile.py \\
367
--model neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 --batch-size 4 \\
368
--prompt-len 512 --max-num-batched-tokens 8196 --json Llama31-8b-FP8 \\
217
- --enforce-eager
369
+ --enforce-eager run_num_steps -n 2
370
```
371
372
then you can use various tools to analyze the json output
@@ -261,17 +413,41 @@ Profile a model
413
default=BATCH_SIZE_DEFAULT,
414
help=f"Number of requests to run as a single batch, "
415
f"default={BATCH_SIZE_DEFAULT}")
264
- parser.add_argument(
265
- "--output-len",
416
+
417
+ subparsers = parser.add_subparsers(dest="cmd")
418
+
419
+ run_num_steps_parser = subparsers.add_parser(
420
+ "run_num_steps",
421
+ help="This variation profiles n engine.step() invocations.")
422
+ run_num_steps_parser.add_argument(
423
+ '-n',
424
+ '--num-steps',
425
type=int,
267
- default=OUTPUT_LEN_DEFAULT,
268
- help="Number of llm steps to run (includes prefill and decode) "
269
- "- default={OUTPUT_LEN_DEFAULT}")
426
+ help="Number of engine steps to profile.\n"
427
+ "Setting it to 1, profiles only the prefill step.\n"
428
+ "Setting it to 2, profiles the prefill and first decode step\n"
429
+ "Setting it to 3, profiles the prefill, 1st and 2nd decode steps\n"
430
+ "and so on ...")
431
+
432
+ run_to_completion_parser = subparsers.add_parser(
433
+ "run_to_completion",
434
+ help="This variation profiles all the engine.step() invocations"
435
+ "until the engine exhausts all submitted requests.")
436
+ run_to_completion_parser.add_argument(
437
+ '-n',
438
+ '--complete-num-requests-per-step',
439
+ type=int,
440
+ help=
441
+ "Complete complete_num_requests_per_step requests every decode step."
442
+ "For e.g., with batch_size 128 and complete_num_requests_per_step 32,"
443
+ "the profiler is run for 6 engine steps, with the steps processing, "
444
+ "128, 128, 96, 64, 32, 1 requests respectively.\n"
445
+ "Note that we tack-on a one-request step at the end as it is often "
446
+ "useful.")
447
448
EngineArgs.add_cli_args(parser)
449
450
args = parser.parse_args()
274
-
451
context = ProfileContext(
452
engine_args=EngineArgs.from_cli_args(args),
453
**{
tools/profiler/print_layerwise_table.py
+7
-2
@@ -34,9 +34,10 @@ if __name__ == "__main__":
34
"examples/offline_profile.py")
35
parser.add_argument("--phase",
36
type=str,
37
- choices=["prefill", "decode_1"],
37
required=True,
39
- help="The phase to print the table for.")
38
+ help="The phase to print the table for. This is either"
39
+ "prefill or decode_n, where n is the decode step "
40
+ "number")
41
parser.add_argument("--table",
42
type=str,
43
choices=["summary", "model"],
@@ -49,6 +50,10 @@ if __name__ == "__main__":
50
with open(args.json_trace) as f:
51
profile_data = json.load(f)
52
53
+ assert args.phase in profile_data, \
54
+ (f"Cannot find phase {args.phase} in profile data. Choose one among"
55
+ f'{[x for x in profile_data.keys() if "prefill" in x or "decode" in x]}') #noqa
56
+
57
if args.table == "summary":
58
entries_and_depths = flatten_entries(
59
SummaryStatsEntry, profile_data[args.phase]["summary_stats"])
tools/profiler/visualize_layerwise_profile.py
+80
-12
@@ -151,16 +151,31 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
151
"scaled_int8_quant" in op_name:
152
return True
153
154
+ # LoRA ops
155
+ def is_sgmv_shrink(op_name: str):
156
+ return "sgmv_shrink" in op_name
157
+
158
+ def is_sgmv_expand(op_name: str):
159
+ return "sgmv_expand" in op_name
160
+
161
+ def is_bgmv_shrink(op_name: str):
162
+ return "bgmv_shrink" in op_name
163
+
164
+ def is_bgmv_expand(op_name: str):
165
+ return "bgmv_expand" in op_name
166
+
167
+ def is_cutlass_gemm_op(op_name: str):
168
+ return "void cutlass::Kernel" in op_name or \
169
+ "void cutlass::device_kernel" in op_name
170
+
171
def is_gemm_op(op_name: str):
172
if is_quant(op_name):
173
return False
157
- if "xmma_gemm" in op_name or \
174
+ return is_cutlass_gemm_op(op_name) or \
175
+ "xmma_gemm" in op_name or \
176
"gemv2T_kernel" in op_name or \
177
"splitKreduce" in op_name or \
160
- "void cutlass::Kernel" in op_name or \
161
- "void cutlass::device_kernel" in op_name or \
162
- "s16816gemm" in op_name:
163
- return True
178
+ "s16816gemm" in op_name
179
180
def is_elementwise_op(op_name: str):
181
return "elementwise_kernel" in op_name
@@ -211,6 +226,18 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
226
quant_ops = list(filter(lambda x: is_quant(x), ops))
227
ops = list(filter(lambda x: x not in quant_ops, ops))
228
229
+ sgmv_shrink_ops = list(filter(lambda x: is_sgmv_shrink(x), ops))
230
+ ops = list(filter(lambda x: x not in sgmv_shrink_ops, ops))
231
+ sgmv_expand_ops = list(filter(lambda x: is_sgmv_expand(x), ops))
232
+ ops = list(filter(lambda x: x not in sgmv_expand_ops, ops))
233
+ bgmv_shrink_ops = list(filter(lambda x: is_bgmv_shrink(x), ops))
234
+ ops = list(filter(lambda x: x not in bgmv_shrink_ops, ops))
235
+ bgmv_expand_ops = list(filter(lambda x: is_bgmv_expand(x), ops))
236
+ ops = list(filter(lambda x: x not in bgmv_expand_ops, ops))
237
+
238
+ cutlass_gemm_ops = list(filter(lambda x: is_cutlass_gemm_op(x), ops))
239
+ ops = list(filter(lambda x: x not in cutlass_gemm_ops, ops))
240
+
241
gemm_ops = list(filter(lambda x: is_gemm_op(x), ops))
242
ops = list(filter(lambda x: x not in gemm_ops, ops))
243
@@ -257,6 +284,24 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
284
trace_df['attention'] = trace_df[attention_ops].agg("sum", axis=1)
285
if len(quant_ops):
286
trace_df['quant_ops'] = trace_df[quant_ops].agg("sum", axis=1)
287
+
288
+ if len(sgmv_shrink_ops):
289
+ trace_df['sgmv_shrink_ops'] = trace_df[sgmv_shrink_ops].agg("sum",
290
+ axis=1)
291
+ if len(sgmv_expand_ops):
292
+ trace_df['sgmv_expand_ops'] = trace_df[sgmv_expand_ops].agg("sum",
293
+ axis=1)
294
+ if len(bgmv_shrink_ops):
295
+ trace_df['bgmv_shrink_ops'] = trace_df[bgmv_shrink_ops].agg("sum",
296
+ axis=1)
297
+ if len(bgmv_expand_ops):
298
+ trace_df['bgmv_expand_ops'] = trace_df[bgmv_expand_ops].agg("sum",
299
+ axis=1)
300
+
301
+ if len(cutlass_gemm_ops):
302
+ trace_df['cutlass_gemm_ops'] = trace_df[cutlass_gemm_ops].agg("sum",
303
+ axis=1)
304
+
305
if len(gemm_ops):
306
trace_df['gemm_ops'] = trace_df[gemm_ops].agg("sum", axis=1)
307
if len(rms_norm_ops):
@@ -296,7 +341,9 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
341
trace_df['reduce_kernel_ops'] = trace_df[reduce_kernel_ops].agg("sum",
342
axis=1)
343
299
- trace_df.drop(attention_ops + quant_ops + gemm_ops + rms_norm_ops +
344
+ trace_df.drop(attention_ops + quant_ops + sgmv_shrink_ops +
345
+ sgmv_expand_ops + bgmv_shrink_ops + bgmv_expand_ops +
346
+ cutlass_gemm_ops + gemm_ops + rms_norm_ops +
347
vocab_embed_ops + mem_ops + elementwise_ops +
348
nccl_all_reduce_ops + nccl_gather_ops + nccl_broadcast_ops +
349
nccl_other_ops + cross_device_reduce_1stage_ops +
@@ -315,7 +362,14 @@ def plot_trace_df(traces_df: pd.DataFrame,
362
plot_title: str,
363
output: Optional[Path] = None):
364
365
+ def get_phase_description(traces_df: pd.DataFrame, phase: str) -> str:
366
+ phase_df = traces_df.query(f'phase == "{phase}"')
367
+ descs = phase_df['phase_desc'].to_list()
368
+ assert all([desc == descs[0] for desc in descs])
369
+ return descs[0]
370
+
371
phases = traces_df['phase'].unique()
372
+ phase_descs = [get_phase_description(traces_df, p) for p in phases]
373
traces_df = traces_df.pivot_table(index="phase",
374
columns="name",
375
values=plot_metric,
@@ -324,7 +378,8 @@ def plot_trace_df(traces_df: pd.DataFrame,
378
traces_df = group_trace_by_operations(traces_df)
379
380
# Make the figure
327
- fig, ax = plt.subplots(1, figsize=(5, 8), sharex=True)
381
+ fig_size_x = max(5, len(phases))
382
+ fig, ax = plt.subplots(1, figsize=(fig_size_x, 8), sharex=True)
383
384
# Draw the stacked bars
385
ops = list(traces_df)
@@ -332,7 +387,7 @@ def plot_trace_df(traces_df: pd.DataFrame,
387
for op in ops:
388
values = [traces_df[op][phase] for phase in phases]
389
values = list(map(lambda x: 0.0 if math.isnan(x) else x, values))
335
- ax.bar(phases, values, label=op, bottom=bottom)
390
+ ax.bar(phase_descs, values, label=op, bottom=bottom)
391
bottom = [bottom[j] + values[j] for j in range(len(phases))]
392
393
# Write the values as text on the bars
@@ -390,6 +445,14 @@ def main(
445
["name"]] = "others"
446
return df
447
448
+ def get_phase_description(key: str) -> str:
449
+ num_running_seqs = profile_json[key]['metadata'][
450
+ 'num_running_seqs']
451
+ if num_running_seqs is not None:
452
+ return f"{key}-seqs-{num_running_seqs}"
453
+ else:
454
+ return key
455
+
456
# Get data for each key
457
traces = list(map(lambda x: get_entries_and_traces(x), step_keys))
458
@@ -413,6 +476,7 @@ def main(
476
# Fill in information about the step-keys
477
for trace_df, step_key in zip(trace_dfs, step_keys):
478
trace_df['phase'] = step_key
479
+ trace_df['phase_desc'] = get_phase_description(step_key)
480
481
# Combine all data frames so they can be put in a single plot
482
traces_df = pd.concat(trace_dfs)
@@ -426,12 +490,16 @@ def main(
490
def make_plot_title_suffix(profile_json: dict) -> str:
491
context = profile_json["context"]
492
sparsity = context.get('sparsity', None)
429
- return (f"{context['model']}\n"
493
+ run_type = \
494
+ f'Run {context["num_steps"]} steps' if context['num_steps'] else \
495
+ (f'Complete {context["complete_num_requests_per_step"]} per '
496
+ f'step; Run till completion')
497
+ return (f"{context['engine_args']['model']}\n"
498
f"Batch={context['batch_size']}, "
499
f"PromptLen={context['prompt_len']}, "
432
- f"OutputLen={context['output_len']},"
433
- f"NumGpus={context['tensor_parallel_size']}"
434
- f"{', Sparsity ' + sparsity if sparsity else ''}")
500
+ f"NumGpus={context['engine_args']['tensor_parallel_size']}"
501
+ f"{', Sparsity ' + sparsity if sparsity else ''}\n"
502
+ f"Run Type: {run_type}")
503
504
profile_json = None
505
with open(json_trace) as f:
vllm/profiler/layerwise_profile.py
+20
-2
@@ -72,6 +72,9 @@ class LayerwiseProfileResults(profile):
72
_model_stats_tree: List[_StatsTreeNode] = field(init=False)
73
_summary_stats_tree: List[_StatsTreeNode] = field(init=False)
74
75
+ # profile metadata
76
+ num_running_seqs: Optional[int] = None
77
+
78
def __post_init__(self):
79
self._build_correlation_map()
80
self._build_module_tree()
@@ -127,6 +130,9 @@ class LayerwiseProfileResults(profile):
130
131
def convert_stats_to_dict(self) -> str:
132
return {
133
+ "metadata": {
134
+ "num_running_seqs": self.num_running_seqs
135
+ },
136
"summary_stats":
137
self._convert_stats_tree_to_dict(self._summary_stats_tree),
138
"model_stats":
@@ -338,7 +344,15 @@ class LayerwiseProfileResults(profile):
344
345
class layerwise_profile(profile):
346
341
- def __init__(self):
347
+ def __init__(self, num_running_seqs: Optional[int] = None):
348
+ """
349
+ layerwise profile constructor.
350
+
351
+ Args:
352
+ num_running_seqs (Optional[int], optional): When given,
353
+ num_running_seqs will be passed to LayerProfileResults for metadata
354
+ update. Defaults to None.
355
+ """
356
super().__init__(
357
activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
358
record_shapes=True,
@@ -346,9 +360,13 @@ class layerwise_profile(profile):
360
with_modules=True,
361
experimental_config=_ExperimentalConfig(verbose=True))
362
363
+ self.num_running_seqs = num_running_seqs
364
+
365
def __enter__(self):
366
return super().__enter__()
367
368
def __exit__(self, exc_type, exc_val, exc_tb):
369
super().__exit__(exc_type, exc_val, exc_tb)
354
- self.results = LayerwiseProfileResults(self.profiler.kineto_results)
370
+ self.results = LayerwiseProfileResults(
371
+ self.profiler.kineto_results,
372
+ num_running_seqs=self.num_running_seqs)