@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a03ea40792
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[3/N][torch.compile] consolidate custom op logging (#10399)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 18, 2024 at 15:14 UTC a03ea40792201ac8ff547d37d9f9255b347b9ccd
3 files changed +20 -5
vllm/config.py
+10 -2
@@ -4,8 +4,9 @@ import json
4 import warnings
5 from dataclasses import dataclass, field, replace
6 from pathlib import Path
7 -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, Final, List,
8 - Literal, Mapping, Optional, Set, Tuple, Type, Union)
7 +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,
8 + Final, List, Literal, Mapping, Optional, Set, Tuple, Type,
9 + Union)
10
11 import torch
12 from pydantic import BaseModel, Field, PrivateAttr
@@ -2169,6 +2170,10 @@ class CompilationConfig(BaseModel):
2170 compile_sizes: List[int] = PrivateAttr
2171 capture_sizes: List[int] = PrivateAttr
2172
2173 + # keep track of enabled and disabled custom ops
2174 + enabled_custom_ops: Counter[str] = PrivateAttr
2175 + disabled_custom_ops: Counter[str] = PrivateAttr
2176 +
2177 def model_post_init(self, __context: Any) -> None:
2178 self.level = envs.VLLM_TORCH_COMPILE_LEVEL
2179
@@ -2190,6 +2195,9 @@ class CompilationConfig(BaseModel):
2195 func = __import__(module).__dict__[func_name]
2196 self.inductor_compile_config[k] = func
2197
2198 + self.enabled_custom_ops = Counter()
2199 + self.disabled_custom_ops = Counter()
2200 +
2201 def init_backend(self) -> Union[str, Callable]:
2202 if self.level == CompilationLevel.NO_COMPILATION:
2203 raise ValueError("No compilation level is set.")
vllm/model_executor/custom_op.py
+6 -3
@@ -61,10 +61,13 @@ class CustomOp(nn.Module):
61 def dispatch_forward(self):
62 # NOTE(woosuk): Here we assume that vLLM was built for only one
63 # specific backend. Currently, we do not support dynamic dispatching.
64 -
64 + compilation_config = get_current_vllm_config().compilation_config
65 enabled = self.enabled()
66 - logger.debug("custom op %s %s", self.__class__.name,
67 - "enabled" if enabled else "disabled")
66 + if enabled:
67 + compilation_config.enabled_custom_ops.update([self.__class__.name])
68 + else:
69 + compilation_config.disabled_custom_ops.update(
70 + [self.__class__.name])
71
72 if not enabled:
73 return self.forward_native
vllm/plugins/__init__.py
+4
@@ -80,6 +80,10 @@ def set_current_vllm_config(vllm_config: VllmConfig):
80 _current_vllm_config = vllm_config
81 yield
82 finally:
83 + logger.debug("enabled custom ops: %s",
84 + vllm_config.compilation_config.enabled_custom_ops)
85 + logger.debug("disabled custom ops: %s",
86 + vllm_config.compilation_config.disabled_custom_ops)
87 _current_vllm_config = old_vllm_config
88
89