@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0974c9bc5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Bugfix] Fix incorrect types in LayerwiseProfileResults (#12196)

Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>

Yuan Tang committed Jan 20, 2025 at 01:59 UTC 0974c9bc5c0252ecb25f440139936529657452ab
1 file changed +3 -3
vllm/profiler/layerwise_profile.py
+3 -3
@@ -1,7 +1,7 @@
1 import copy
2 from collections import defaultdict
3 from dataclasses import asdict, dataclass, field
4 -from typing import Callable, Dict, List, Optional, Tuple, TypeAlias, Union
4 +from typing import Any, Callable, Dict, List, Optional, Tuple, TypeAlias, Union
5
6 import pandas as pd
7 from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
@@ -128,7 +128,7 @@ class LayerwiseProfileResults(profile):
128 ])
129 df.to_csv(filename)
130
131 - def convert_stats_to_dict(self) -> str:
131 + def convert_stats_to_dict(self) -> dict[str, Any]:
132 return {
133 "metadata": {
134 "num_running_seqs": self.num_running_seqs
@@ -227,7 +227,7 @@ class LayerwiseProfileResults(profile):
227 [self._cumulative_cuda_time(root) for root in self._module_tree])
228
229 def _build_stats_trees(self):
230 - summary_dict: Dict[str, self.StatsTreeNode] = {}
230 + summary_dict: Dict[str, _StatsTreeNode] = {}
231 total_cuda_time = self._total_cuda_time()
232
233 def pct_cuda_time(cuda_time_us):