@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / efbce85f4d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[misc] Layerwise profile updates (#10242)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Dec 16, 2024 at 13:14 UTC efbce85f4d375d7851a491a0126a224e25d9f91d
5 files changed +314 -47
.buildkite/test-pipeline.yaml
+1 -1
@@ -201,7 +201,7 @@ steps:
201 - python3 offline_inference_classification.py
202 - python3 offline_inference_embedding.py
203 - python3 offline_inference_scoring.py
204 - - python3 offline_profile.py --model facebook/opt-125m
204 + - python3 offline_profile.py --model facebook/opt-125m run_num_steps --num-steps 2
205
206 - label: Prefix Caching Test # 9min
207 mirror_hardwares: [amd]
examples/offline_profile.py
+206 -30
@@ -4,9 +4,10 @@ import os
4 import sys
5 from argparse import RawTextHelpFormatter
6 from dataclasses import asdict, dataclass
7 -from typing import Optional
7 +from typing import Any, Dict, Generator, List, Optional, TypeAlias
8
9 import torch
10 +import tqdm
11
12 from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import EngineArgs
@@ -15,16 +16,21 @@ from vllm.utils import FlexibleArgumentParser
16
17 BATCH_SIZE_DEFAULT = 1
18 PROMPT_LEN_DEFAULT = 256
18 -OUTPUT_LEN_DEFAULT = 2
19
20
21 @dataclass
22 class ProfileContext:
23 engine_args: EngineArgs
24 prompt_len: int
25 - output_len: int
25 batch_size: int
27 - save_chrome_traces_folder: Optional[str]
26 +
27 + # The profiler can run in 2 modes,
28 + # 1. Run profiler for user specified num_steps
29 + num_steps: Optional[int] = None
30 + # 2. Run profiler until all requests complete
31 + complete_num_requests_per_step: Optional[int] = None
32 +
33 + save_chrome_traces_folder: Optional[str] = None
34
35
36 def get_dtype(dtype: str):
@@ -34,23 +40,155 @@ def get_dtype(dtype: str):
40 return dtype
41
42
43 +OutputLen_NumReqs_Map: TypeAlias = Dict[int, int]
44 +def compute_request_output_lengths(batch_size: int, step_requests: List[int]) \
45 + -> OutputLen_NumReqs_Map:
46 + """
47 + Given the number of requests, batch_size, and the number of requests
48 + that each engine-step should process, step_requests, determine the
49 + output lengths of the requests such that step_request is honoured.
50 +
51 + Example:
52 + if batch size = 128 and step_request = [128, 128, 96, 64, 32, 1]
53 + then return,
54 + {2 : 32, 3 : 32, 4 : 32, 5 : 31, 6 : 1}, meaning,
55 + 32 requests should have output length 2,
56 + 32 requests should have output length 3,
57 + 32 requests should have output length 4,
58 + 31 requests should have output length 5,
59 + 1 request should have output length 6.
60 +
61 + Args:
62 + batch_size (int): Number of requests submitted for profile. This is
63 + args.batch_size.
64 + step_requests (List[int]): step_requests[i] is the number of requests
65 + that the ith engine step should process.
66 +
67 + Returns:
68 + OutputLen_NumReqs_Map : A dictionary with output-length as keys and the
69 + number of requests required to have that output-length as values.
70 + """
71 + ol_nr: OutputLen_NumReqs_Map = {}
72 +
73 + # Number of request that are assigned an output-length
74 + num_reqs_assigned: int = 0
75 + num_steps: int = len(step_requests)
76 +
77 + # sanity check. The first step (prefill-step), must process all requests.
78 + assert step_requests[0] == batch_size
79 +
80 + # Begin assignments from the last step.
81 + output_length: int = num_steps
82 + for num_requests_at_step in reversed(step_requests):
83 + if num_reqs_assigned == batch_size:
84 + break
85 +
86 + assert num_reqs_assigned < batch_size
87 +
88 + # Remove the number of requests that have been determined
89 + # to participate in this step and beyond.
90 + num_reqs_unassigned_at_step = num_requests_at_step - num_reqs_assigned
91 + assert num_reqs_unassigned_at_step >= 0
92 +
93 + if num_reqs_unassigned_at_step > 0:
94 + ol_nr[output_length] = num_reqs_unassigned_at_step
95 + num_reqs_assigned += num_reqs_unassigned_at_step
96 +
97 + output_length -= 1
98 +
99 + # sanity checks.
100 + assert sum(ol_nr.values()) == batch_size, \
101 + ("Number of requests in output-length assignment does not match "
102 + f"batch-size.\n batch size {batch_size} - "
103 + f"step requests {step_requests} - assignments {ol_nr}")
104 +
105 + # Check that the output-length is in [1, num-steps]. Output length must be
106 + # at least 1 as all requests must participate in the prefill-step.
107 + assert all(ol >= 1 and ol <= num_steps for ol in ol_nr), \
108 + ("Output lengths of requests should be in range "
109 + f"[1, num-engine-steps].\n batch size {batch_size} - "
110 + f"step requests {step_requests} - assignments {ol_nr}")
111 +
112 + return ol_nr
113 +
114 +
115 +def determine_requests_per_step(context: ProfileContext) -> List[int]:
116 + """
117 + Determine number of requests each engine step should process.
118 + If context.num_steps is set, then all engine steps process the
119 + same number of requests and the output list is of length
120 + context.num_steps.
121 +
122 + If context.complete_num_requests_per_step is set, then each decode step
123 + processes fewer and fewer requests until there are no requests to process.
124 + In this case, the output list is as big as the number of steps
125 + required to process all requests.
126 +
127 + Args:
128 + context: ProfileContext object.
129 +
130 + Returns:
131 + List[int]: Number of requests to process for all engine-steps.
132 + output[i], contains the number of requests that the ith step
133 + should process.
134 + """
135 + if context.num_steps:
136 + # All requests must run until num_engine_steps. This implies
137 + # that their output lengths must be equal to num_engine_steps.
138 + return [context.batch_size] * context.num_steps
139 +
140 + assert context.complete_num_requests_per_step and \
141 + context.complete_num_requests_per_step > 0, \
142 + (f"Expected a positive complete_num_requests_per_step argument."
143 + f"Instead got {context.complete_num_requests_per_step}")
144 +
145 + # We start dropping after the first decode step.
146 + step_requests = [
147 + context.batch_size, # prefill
148 + context.batch_size, # decode
149 + ]
150 +
151 + num_running_requests = context.batch_size
152 + num_running_requests -= context.complete_num_requests_per_step
153 + while num_running_requests > 0:
154 + step_requests.append(num_running_requests)
155 + num_running_requests -= context.complete_num_requests_per_step
156 +
157 + if step_requests[-1] != 1:
158 + # have 1 request running at the last step. This is often
159 + # useful
160 + step_requests.append(1)
161 +
162 + return step_requests
163 +
164 +
165 def run_profile(context: ProfileContext, csv_output: Optional[str],
166 json_output: Optional[str]):
167 print("Run profile with:")
168 for key, value in asdict(context).items():
169 print(f" {key} = {value}")
170
171 + requests_per_step: List[int] = determine_requests_per_step(context)
172 +
173 + ol_nr: OutputLen_NumReqs_Map = compute_request_output_lengths(
174 + context.batch_size, requests_per_step)
175 +
176 + num_steps_to_profile: int = len(requests_per_step)
177 + max_output_len: int = max(ol_nr.keys())
178 + assert max_output_len >= 1
179 +
180 # Create sampling params
44 - sampling_params = SamplingParams(temperature=0.8,
45 - top_p=0.95,
46 - max_tokens=args.output_len,
47 - ignore_eos=True)
181 + sampling_params = SamplingParams(
182 + temperature=0.8,
183 + top_p=0.95,
184 + # max_tokens is set on a per-request basis.
185 + max_tokens=None,
186 + ignore_eos=True)
187
188 # Create LLM
189 llm = LLM(**asdict(context.engine_args))
190 batch_size = context.batch_size
191 prompt_len = context.prompt_len
53 - output_len = context.output_len
192
193 scheduler_config = llm.llm_engine.scheduler_config
194 max_model_len = llm.llm_engine.model_config.max_model_len
@@ -65,7 +203,7 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
203 f"choose a smaller batch size or prompt length, or increase "
204 f"--max-num-batched-tokens")
205 sys.exit(-1)
68 - if batch_size >= max_num_seqs:
206 + if batch_size > max_num_seqs:
207 print(
208 f"ERROR: chosen batch_size ({batch_size}) is larger than "
209 f"max_num_seqs ({max_num_seqs}) and therefore cannot be run in a "
@@ -73,16 +211,26 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
211 sys.exit(-1)
212 print("llm.llm_engine.model_config.max_model_len: ",
213 llm.llm_engine.model_config.max_model_len)
76 - if prompt_len + output_len > llm.llm_engine.model_config.max_model_len:
77 - print(
78 - f"ERROR: chosen prompt_len + output_len ({prompt_len} + "
79 - f"{output_len} = {prompt_len + output_len}) is larger than the "
80 - f"model's max_model_len ({max_model_len}), please choose a smaller "
81 - f"prompt_len or output_len, or increase --max-model-len")
214 + if prompt_len + max_output_len > llm.llm_engine.model_config.max_model_len:
215 + print(f"ERROR: chosen prompt_len + max_output_len ({prompt_len} + "
216 + f"{max_output_len} = {prompt_len + max_output_len}) is larger "
217 + f"than the model's max_model_len ({max_model_len}), please "
218 + f"choose a smaller prompt_len or max_output_len, or increase "
219 + f"--max-model-len")
220 sys.exit(-1)
221
222 def add_requests():
223 +
224 + def get_output_len_generator() -> Generator[int, Any, Any]:
225 + for output_len, num_reqs in ol_nr.items():
226 + for _ in range(num_reqs):
227 + yield output_len
228 +
229 + output_len_generator = get_output_len_generator()
230 for i in range(batch_size):
231 + sampling_params.max_tokens = next(output_len_generator)
232 + assert isinstance(sampling_params.max_tokens, int)
233 +
234 prompt_token_ids = torch.randint(
235 llm.llm_engine.model_config.get_vocab_size(),
236 size=(prompt_len, )).tolist()
@@ -110,8 +258,11 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
258 llm.llm_engine.step() # First step is prefill
259
260 decode_profs = []
113 - for x in range(args.output_len - 1):
114 - with layerwise_profile() as decode_prof:
261 + for _ in tqdm.tqdm(range(num_steps_to_profile - 1)):
262 + num_running_seqs = llm.llm_engine.scheduler[
263 + 0].get_num_unfinished_seq_groups()
264 + with layerwise_profile(
265 + num_running_seqs=num_running_seqs) as decode_prof:
266 llm.llm_engine.step()
267 decode_profs.append(decode_prof)
268
@@ -154,7 +305,8 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
305 decode_results_list[0].print_summary_table()
306
307 if csv_output:
157 - csv_filename_base = csv_output.rstrip(".csv")
308 + csv_filename_base = csv_output[:-4] \
309 + if csv_output.endswith('.csv') else csv_output
310 prefill_results.export_model_stats_table_csv(
311 csv_filename_base + "_prefill_model_table.csv")
312 prefill_results.export_summary_stats_table_csv(
@@ -187,10 +339,10 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
339 for idx, dr in enumerate(decode_results_list):
340 json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
341
190 - for idx, dr in enumerate(decode_results_list[1:]):
191 - json_dict[f"decode_{idx + 1}"] = dr.convert_stats_to_dict()
192 -
193 - with open(json_output.rstrip(".json") + ".json", "w+") as f:
342 + # Add .json to json_output filename if it doesn't exist already.
343 + json_output_file = json_output if json_output.endswith(
344 + '.json') else json_output + '.json'
345 + with open(json_output_file, "w+") as f:
346 json.dump(json_dict, f, indent=2)
347 pass
348
@@ -214,7 +366,7 @@ Profile a model
366 python examples/offline_profile.py \\
367 --model neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 --batch-size 4 \\
368 --prompt-len 512 --max-num-batched-tokens 8196 --json Llama31-8b-FP8 \\
217 - --enforce-eager
369 + --enforce-eager run_num_steps -n 2
370 ```
371
372 then you can use various tools to analyze the json output
@@ -261,17 +413,41 @@ Profile a model
413 default=BATCH_SIZE_DEFAULT,
414 help=f"Number of requests to run as a single batch, "
415 f"default={BATCH_SIZE_DEFAULT}")
264 - parser.add_argument(
265 - "--output-len",
416 +
417 + subparsers = parser.add_subparsers(dest="cmd")
418 +
419 + run_num_steps_parser = subparsers.add_parser(
420 + "run_num_steps",
421 + help="This variation profiles n engine.step() invocations.")
422 + run_num_steps_parser.add_argument(
423 + '-n',
424 + '--num-steps',
425 type=int,
267 - default=OUTPUT_LEN_DEFAULT,
268 - help="Number of llm steps to run (includes prefill and decode) "
269 - "- default={OUTPUT_LEN_DEFAULT}")
426 + help="Number of engine steps to profile.\n"
427 + "Setting it to 1, profiles only the prefill step.\n"
428 + "Setting it to 2, profiles the prefill and first decode step\n"
429 + "Setting it to 3, profiles the prefill, 1st and 2nd decode steps\n"
430 + "and so on ...")
431 +
432 + run_to_completion_parser = subparsers.add_parser(
433 + "run_to_completion",
434 + help="This variation profiles all the engine.step() invocations"
435 + "until the engine exhausts all submitted requests.")
436 + run_to_completion_parser.add_argument(
437 + '-n',
438 + '--complete-num-requests-per-step',
439 + type=int,
440 + help=
441 + "Complete complete_num_requests_per_step requests every decode step."
442 + "For e.g., with batch_size 128 and complete_num_requests_per_step 32,"
443 + "the profiler is run for 6 engine steps, with the steps processing, "
444 + "128, 128, 96, 64, 32, 1 requests respectively.\n"
445 + "Note that we tack-on a one-request step at the end as it is often "
446 + "useful.")
447
448 EngineArgs.add_cli_args(parser)
449
450 args = parser.parse_args()
274 -
451 context = ProfileContext(
452 engine_args=EngineArgs.from_cli_args(args),
453 **{
tools/profiler/print_layerwise_table.py
+7 -2
@@ -34,9 +34,10 @@ if __name__ == "__main__":
34 "examples/offline_profile.py")
35 parser.add_argument("--phase",
36 type=str,
37 - choices=["prefill", "decode_1"],
37 required=True,
39 - help="The phase to print the table for.")
38 + help="The phase to print the table for. This is either"
39 + "prefill or decode_n, where n is the decode step "
40 + "number")
41 parser.add_argument("--table",
42 type=str,
43 choices=["summary", "model"],
@@ -49,6 +50,10 @@ if __name__ == "__main__":
50 with open(args.json_trace) as f:
51 profile_data = json.load(f)
52
53 + assert args.phase in profile_data, \
54 + (f"Cannot find phase {args.phase} in profile data. Choose one among"
55 + f'{[x for x in profile_data.keys() if "prefill" in x or "decode" in x]}') #noqa
56 +
57 if args.table == "summary":
58 entries_and_depths = flatten_entries(
59 SummaryStatsEntry, profile_data[args.phase]["summary_stats"])
tools/profiler/visualize_layerwise_profile.py
+80 -12
@@ -151,16 +151,31 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
151 "scaled_int8_quant" in op_name:
152 return True
153
154 + # LoRA ops
155 + def is_sgmv_shrink(op_name: str):
156 + return "sgmv_shrink" in op_name
157 +
158 + def is_sgmv_expand(op_name: str):
159 + return "sgmv_expand" in op_name
160 +
161 + def is_bgmv_shrink(op_name: str):
162 + return "bgmv_shrink" in op_name
163 +
164 + def is_bgmv_expand(op_name: str):
165 + return "bgmv_expand" in op_name
166 +
167 + def is_cutlass_gemm_op(op_name: str):
168 + return "void cutlass::Kernel" in op_name or \
169 + "void cutlass::device_kernel" in op_name
170 +
171 def is_gemm_op(op_name: str):
172 if is_quant(op_name):
173 return False
157 - if "xmma_gemm" in op_name or \
174 + return is_cutlass_gemm_op(op_name) or \
175 + "xmma_gemm" in op_name or \
176 "gemv2T_kernel" in op_name or \
177 "splitKreduce" in op_name or \
160 - "void cutlass::Kernel" in op_name or \
161 - "void cutlass::device_kernel" in op_name or \
162 - "s16816gemm" in op_name:
163 - return True
178 + "s16816gemm" in op_name
179
180 def is_elementwise_op(op_name: str):
181 return "elementwise_kernel" in op_name
@@ -211,6 +226,18 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
226 quant_ops = list(filter(lambda x: is_quant(x), ops))
227 ops = list(filter(lambda x: x not in quant_ops, ops))
228
229 + sgmv_shrink_ops = list(filter(lambda x: is_sgmv_shrink(x), ops))
230 + ops = list(filter(lambda x: x not in sgmv_shrink_ops, ops))
231 + sgmv_expand_ops = list(filter(lambda x: is_sgmv_expand(x), ops))
232 + ops = list(filter(lambda x: x not in sgmv_expand_ops, ops))
233 + bgmv_shrink_ops = list(filter(lambda x: is_bgmv_shrink(x), ops))
234 + ops = list(filter(lambda x: x not in bgmv_shrink_ops, ops))
235 + bgmv_expand_ops = list(filter(lambda x: is_bgmv_expand(x), ops))
236 + ops = list(filter(lambda x: x not in bgmv_expand_ops, ops))
237 +
238 + cutlass_gemm_ops = list(filter(lambda x: is_cutlass_gemm_op(x), ops))
239 + ops = list(filter(lambda x: x not in cutlass_gemm_ops, ops))
240 +
241 gemm_ops = list(filter(lambda x: is_gemm_op(x), ops))
242 ops = list(filter(lambda x: x not in gemm_ops, ops))
243
@@ -257,6 +284,24 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
284 trace_df['attention'] = trace_df[attention_ops].agg("sum", axis=1)
285 if len(quant_ops):
286 trace_df['quant_ops'] = trace_df[quant_ops].agg("sum", axis=1)
287 +
288 + if len(sgmv_shrink_ops):
289 + trace_df['sgmv_shrink_ops'] = trace_df[sgmv_shrink_ops].agg("sum",
290 + axis=1)
291 + if len(sgmv_expand_ops):
292 + trace_df['sgmv_expand_ops'] = trace_df[sgmv_expand_ops].agg("sum",
293 + axis=1)
294 + if len(bgmv_shrink_ops):
295 + trace_df['bgmv_shrink_ops'] = trace_df[bgmv_shrink_ops].agg("sum",
296 + axis=1)
297 + if len(bgmv_expand_ops):
298 + trace_df['bgmv_expand_ops'] = trace_df[bgmv_expand_ops].agg("sum",
299 + axis=1)
300 +
301 + if len(cutlass_gemm_ops):
302 + trace_df['cutlass_gemm_ops'] = trace_df[cutlass_gemm_ops].agg("sum",
303 + axis=1)
304 +
305 if len(gemm_ops):
306 trace_df['gemm_ops'] = trace_df[gemm_ops].agg("sum", axis=1)
307 if len(rms_norm_ops):
@@ -296,7 +341,9 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
341 trace_df['reduce_kernel_ops'] = trace_df[reduce_kernel_ops].agg("sum",
342 axis=1)
343
299 - trace_df.drop(attention_ops + quant_ops + gemm_ops + rms_norm_ops +
344 + trace_df.drop(attention_ops + quant_ops + sgmv_shrink_ops +
345 + sgmv_expand_ops + bgmv_shrink_ops + bgmv_expand_ops +
346 + cutlass_gemm_ops + gemm_ops + rms_norm_ops +
347 vocab_embed_ops + mem_ops + elementwise_ops +
348 nccl_all_reduce_ops + nccl_gather_ops + nccl_broadcast_ops +
349 nccl_other_ops + cross_device_reduce_1stage_ops +
@@ -315,7 +362,14 @@ def plot_trace_df(traces_df: pd.DataFrame,
362 plot_title: str,
363 output: Optional[Path] = None):
364
365 + def get_phase_description(traces_df: pd.DataFrame, phase: str) -> str:
366 + phase_df = traces_df.query(f'phase == "{phase}"')
367 + descs = phase_df['phase_desc'].to_list()
368 + assert all([desc == descs[0] for desc in descs])
369 + return descs[0]
370 +
371 phases = traces_df['phase'].unique()
372 + phase_descs = [get_phase_description(traces_df, p) for p in phases]
373 traces_df = traces_df.pivot_table(index="phase",
374 columns="name",
375 values=plot_metric,
@@ -324,7 +378,8 @@ def plot_trace_df(traces_df: pd.DataFrame,
378 traces_df = group_trace_by_operations(traces_df)
379
380 # Make the figure
327 - fig, ax = plt.subplots(1, figsize=(5, 8), sharex=True)
381 + fig_size_x = max(5, len(phases))
382 + fig, ax = plt.subplots(1, figsize=(fig_size_x, 8), sharex=True)
383
384 # Draw the stacked bars
385 ops = list(traces_df)
@@ -332,7 +387,7 @@ def plot_trace_df(traces_df: pd.DataFrame,
387 for op in ops:
388 values = [traces_df[op][phase] for phase in phases]
389 values = list(map(lambda x: 0.0 if math.isnan(x) else x, values))
335 - ax.bar(phases, values, label=op, bottom=bottom)
390 + ax.bar(phase_descs, values, label=op, bottom=bottom)
391 bottom = [bottom[j] + values[j] for j in range(len(phases))]
392
393 # Write the values as text on the bars
@@ -390,6 +445,14 @@ def main(
445 ["name"]] = "others"
446 return df
447
448 + def get_phase_description(key: str) -> str:
449 + num_running_seqs = profile_json[key]['metadata'][
450 + 'num_running_seqs']
451 + if num_running_seqs is not None:
452 + return f"{key}-seqs-{num_running_seqs}"
453 + else:
454 + return key
455 +
456 # Get data for each key
457 traces = list(map(lambda x: get_entries_and_traces(x), step_keys))
458
@@ -413,6 +476,7 @@ def main(
476 # Fill in information about the step-keys
477 for trace_df, step_key in zip(trace_dfs, step_keys):
478 trace_df['phase'] = step_key
479 + trace_df['phase_desc'] = get_phase_description(step_key)
480
481 # Combine all data frames so they can be put in a single plot
482 traces_df = pd.concat(trace_dfs)
@@ -426,12 +490,16 @@ def main(
490 def make_plot_title_suffix(profile_json: dict) -> str:
491 context = profile_json["context"]
492 sparsity = context.get('sparsity', None)
429 - return (f"{context['model']}\n"
493 + run_type = \
494 + f'Run {context["num_steps"]} steps' if context['num_steps'] else \
495 + (f'Complete {context["complete_num_requests_per_step"]} per '
496 + f'step; Run till completion')
497 + return (f"{context['engine_args']['model']}\n"
498 f"Batch={context['batch_size']}, "
499 f"PromptLen={context['prompt_len']}, "
432 - f"OutputLen={context['output_len']},"
433 - f"NumGpus={context['tensor_parallel_size']}"
434 - f"{', Sparsity ' + sparsity if sparsity else ''}")
500 + f"NumGpus={context['engine_args']['tensor_parallel_size']}"
501 + f"{', Sparsity ' + sparsity if sparsity else ''}\n"
502 + f"Run Type: {run_type}")
503
504 profile_json = None
505 with open(json_trace) as f:
vllm/profiler/layerwise_profile.py
+20 -2
@@ -72,6 +72,9 @@ class LayerwiseProfileResults(profile):
72 _model_stats_tree: List[_StatsTreeNode] = field(init=False)
73 _summary_stats_tree: List[_StatsTreeNode] = field(init=False)
74
75 + # profile metadata
76 + num_running_seqs: Optional[int] = None
77 +
78 def __post_init__(self):
79 self._build_correlation_map()
80 self._build_module_tree()
@@ -127,6 +130,9 @@ class LayerwiseProfileResults(profile):
130
131 def convert_stats_to_dict(self) -> str:
132 return {
133 + "metadata": {
134 + "num_running_seqs": self.num_running_seqs
135 + },
136 "summary_stats":
137 self._convert_stats_tree_to_dict(self._summary_stats_tree),
138 "model_stats":
@@ -338,7 +344,15 @@ class LayerwiseProfileResults(profile):
344
345 class layerwise_profile(profile):
346
341 - def __init__(self):
347 + def __init__(self, num_running_seqs: Optional[int] = None):
348 + """
349 + layerwise profile constructor.
350 +
351 + Args:
352 + num_running_seqs (Optional[int], optional): When given,
353 + num_running_seqs will be passed to LayerProfileResults for metadata
354 + update. Defaults to None.
355 + """
356 super().__init__(
357 activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA],
358 record_shapes=True,
@@ -346,9 +360,13 @@ class layerwise_profile(profile):
360 with_modules=True,
361 experimental_config=_ExperimentalConfig(verbose=True))
362
363 + self.num_running_seqs = num_running_seqs
364 +
365 def __enter__(self):
366 return super().__enter__()
367
368 def __exit__(self, exc_type, exc_val, exc_tb):
369 super().__exit__(exc_type, exc_val, exc_tb)
354 - self.results = LayerwiseProfileResults(self.profiler.kineto_results)
370 + self.results = LayerwiseProfileResults(
371 + self.profiler.kineto_results,
372 + num_running_seqs=self.num_running_seqs)