A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Bugfix] Fix incorrect types in LayerwiseProfileResults (#12196)
Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>
Yuan Tang committed
Jan 20, 2025 at 01:59 UTC
0974c9bc5c0252ecb25f440139936529657452ab
1 file changed
+3
-3
vllm/profiler/layerwise_profile.py
+3
-3
@@ -1,7 +1,7 @@
1
import copy
2
from collections import defaultdict
3
from dataclasses import asdict, dataclass, field
4
-from typing import Callable, Dict, List, Optional, Tuple, TypeAlias, Union
4
+from typing import Any, Callable, Dict, List, Optional, Tuple, TypeAlias, Union
5
6
import pandas as pd
7
from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
@@ -128,7 +128,7 @@ class LayerwiseProfileResults(profile):
128
])
129
df.to_csv(filename)
130
131
- def convert_stats_to_dict(self) -> str:
131
+ def convert_stats_to_dict(self) -> dict[str, Any]:
132
return {
133
"metadata": {
134
"num_running_seqs": self.num_running_seqs
@@ -227,7 +227,7 @@ class LayerwiseProfileResults(profile):
227
[self._cumulative_cuda_time(root) for root in self._module_tree])
228
229
def _build_stats_trees(self):
230
- summary_dict: Dict[str, self.StatsTreeNode] = {}
230
+ summary_dict: Dict[str, _StatsTreeNode] = {}
231
total_cuda_time = self._total_cuda_time()
232
233
def pct_cuda_time(cuda_time_us):