A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[3/N][torch.compile] consolidate custom op logging (#10399)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 18, 2024 at 15:14 UTC
a03ea40792201ac8ff547d37d9f9255b347b9ccd
3 files changed
+20
-5
vllm/config.py
+10
-2
@@ -4,8 +4,9 @@ import json
4
import warnings
5
from dataclasses import dataclass, field, replace
6
from pathlib import Path
7
-from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Dict, Final, List,
8
- Literal, Mapping, Optional, Set, Tuple, Type, Union)
7
+from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Counter, Dict,
8
+ Final, List, Literal, Mapping, Optional, Set, Tuple, Type,
9
+ Union)
10
11
import torch
12
from pydantic import BaseModel, Field, PrivateAttr
@@ -2169,6 +2170,10 @@ class CompilationConfig(BaseModel):
2170
compile_sizes: List[int] = PrivateAttr
2171
capture_sizes: List[int] = PrivateAttr
2172
2173
+ # keep track of enabled and disabled custom ops
2174
+ enabled_custom_ops: Counter[str] = PrivateAttr
2175
+ disabled_custom_ops: Counter[str] = PrivateAttr
2176
+
2177
def model_post_init(self, __context: Any) -> None:
2178
self.level = envs.VLLM_TORCH_COMPILE_LEVEL
2179
@@ -2190,6 +2195,9 @@ class CompilationConfig(BaseModel):
2195
func = __import__(module).__dict__[func_name]
2196
self.inductor_compile_config[k] = func
2197
2198
+ self.enabled_custom_ops = Counter()
2199
+ self.disabled_custom_ops = Counter()
2200
+
2201
def init_backend(self) -> Union[str, Callable]:
2202
if self.level == CompilationLevel.NO_COMPILATION:
2203
raise ValueError("No compilation level is set.")
vllm/model_executor/custom_op.py
+6
-3
@@ -61,10 +61,13 @@ class CustomOp(nn.Module):
61
def dispatch_forward(self):
62
# NOTE(woosuk): Here we assume that vLLM was built for only one
63
# specific backend. Currently, we do not support dynamic dispatching.
64
-
64
+ compilation_config = get_current_vllm_config().compilation_config
65
enabled = self.enabled()
66
- logger.debug("custom op %s %s", self.__class__.name,
67
- "enabled" if enabled else "disabled")
66
+ if enabled:
67
+ compilation_config.enabled_custom_ops.update([self.__class__.name])
68
+ else:
69
+ compilation_config.disabled_custom_ops.update(
70
+ [self.__class__.name])
71
72
if not enabled:
73
return self.forward_native
vllm/plugins/__init__.py
+4
@@ -80,6 +80,10 @@ def set_current_vllm_config(vllm_config: VllmConfig):
80
_current_vllm_config = vllm_config
81
yield
82
finally:
83
+ logger.debug("enabled custom ops: %s",
84
+ vllm_config.compilation_config.enabled_custom_ops)
85
+ logger.debug("disabled custom ops: %s",
86
+ vllm_config.compilation_config.disabled_custom_ops)
87
_current_vllm_config = old_vllm_config
88
89