A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Kernel][Misc] dynamo support for ScalarType (#7594)
bnellnm committed
Aug 16, 2024 at 16:59 UTC
7759ae958ffb9015cd28137c380ffdcbe7648511
2 files changed
+149
-24
csrc/core/scalar_type.hpp
+30
@@ -313,6 +313,8 @@ class ScalarType {
313
// have ScalarType inherit from torch::CustomClassHolder and have a constexpr
314
// constructor at the same time (torch::CustomClassHolder does not have a
315
// constexpr destructor)
316
+// See also:
317
+// https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA
318
class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
319
public:
320
ScalarTypeTorch(int64_t exponent, int64_t mantissa, int64_t bias,
@@ -382,6 +384,29 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
384
exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
385
}
386
387
+ // This needs to be implemented and throw a TypeError in order for
388
+ // PyTorch's opcheck to work on ops that use ScalarTypes.
389
+ int64_t len() const {
390
+ throw c10::TypeError("__len__ not implemented");
391
+ return 0;
392
+ }
393
+
394
+ // Serialize a ScalarType into a tuple of pairs. Where each pair
395
+ // is a (fieldname, value).
396
+ // For simplicity, we are just going to convert to a ScalarTypeId.
397
+ std::tuple<std::tuple<std::string, int64_t>> obj_flatten() const {
398
+ return {{"ScalarType", id()}};
399
+ }
400
+
401
+ // Deserialize a scalar type that has been serialized by obj_flatten,
402
+ // ostensibly from a tuple of (member name, value) pairs, but in reality
403
+ // just a ScalarTypeId.
404
+ static SelfPtr obj_unflatten(
405
+ std::tuple<std::tuple<std::string, int64_t>> const& flat_type) {
406
+ return c10::make_intrusive<Self>(
407
+ from_id(std::get<1>(std::get<0>(flat_type))));
408
+ }
409
+
410
template <typename T>
411
static void bind_readonly_property(torch::class_<Self>& cls,
412
std::string const& name, T Base::*field) {
@@ -457,6 +482,7 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
482
self.get()->min());
483
});
484
485
+ bind_function(cls, "__len__", &ScalarTypeTorch::len);
486
bind_function(cls, "__str__", &Base::str);
487
bind_function(cls, "__eq__", [](SelfPtr const& self, SelfPtr const& other) {
488
return *self == *other;
@@ -465,6 +491,10 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
491
return "ScalarType." + self.get()->str();
492
});
493
494
+ bind_function(cls, "__obj_flatten__", &ScalarTypeTorch::obj_flatten);
495
+ bind_static_function(cls, "__obj_unflatten__",
496
+ &ScalarTypeTorch::obj_unflatten);
497
+
498
// Bind static functions (convenience constructors)
499
bind_static_function(cls, "int_", &ScalarTypeTorch::int_);
500
bind_static_function(cls, "uint", &ScalarTypeTorch::uint);
vllm/_core_ext.py
+119
-24
@@ -1,6 +1,6 @@
1
import importlib.util
2
from enum import Enum
3
-from typing import TYPE_CHECKING, Optional, Union
3
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
4
5
import torch
6
@@ -31,14 +31,14 @@ if TYPE_CHECKING or not core_C_available:
31
@dataclass(frozen=True)
32
class ScalarType:
33
"""
34
- ScalarType can represent a wide range of floating point and integer
35
- types, in particular it can be used to represent sub-byte data types
36
- (something that torch.dtype currently does not support). It is also
34
+ ScalarType can represent a wide range of floating point and integer
35
+ types, in particular it can be used to represent sub-byte data types
36
+ (something that torch.dtype currently does not support). It is also
37
capable of representing types with a bias, i.e.:
38
- `stored_value = value + bias`,
39
- this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40
- of 8). The implementation for this class can be found in
41
- csrc/core/scalar_type.hpp, these type signatures should be kept in sync
38
+ `stored_value = value + bias`,
39
+ this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40
+ of 8). The implementation for this class can be found in
41
+ csrc/core/scalar_type.hpp, these type signatures should be kept in sync
42
with that file.
43
"""
44
@@ -51,15 +51,15 @@ if TYPE_CHECKING or not core_C_available:
51
mantissa: int
52
"""
53
Number of bits in the mantissa if this is a floating point type,
54
- or the number bits representing an integer excluding the sign bit if
54
+ or the number bits representing an integer excluding the sign bit if
55
this an integer type.
56
"""
57
58
bias: int
59
"""
60
- bias used to encode the values in this scalar type
61
- (value = stored_value - bias, default 0) for example if we store the
62
- type as an unsigned integer with a bias of 128 then the value 0 will be
60
+ bias used to encode the values in this scalar type
61
+ (value = stored_value - bias, default 0) for example if we store the
62
+ type as an unsigned integer with a bias of 128 then the value 0 will be
63
stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
64
"""
65
@@ -73,7 +73,7 @@ if TYPE_CHECKING or not core_C_available:
73
74
nan_repr: int = NanRepr.IEEE_754.value
75
"""
76
- How NaNs are represent in this scalar type, returns NanRepr value.
76
+ How NaNs are represent in this scalar type, returns NanRepr value.
77
(not applicable for integer types)
78
"""
79
@@ -83,14 +83,14 @@ if TYPE_CHECKING or not core_C_available:
83
84
def min(self) -> Union[int, float]:
85
"""
86
- Min representable value for this scalar type.
86
+ Min representable value for this scalar type.
87
(accounting for bias if there is one)
88
"""
89
raise NotImplementedError
90
91
def max(self) -> Union[int, float]:
92
"""
93
- Max representable value for this scalar type.
93
+ Max representable value for this scalar type.
94
(accounting for bias if there is one)
95
"""
96
raise NotImplementedError
@@ -103,28 +103,28 @@ if TYPE_CHECKING or not core_C_available:
103
"""
104
...
105
106
- def is_floating_point(self):
106
+ def is_floating_point(self) -> bool:
107
"If the type is a floating point type"
108
return self.exponent != 0
109
110
- def is_integer(self):
110
+ def is_integer(self) -> bool:
111
"If the type is an integer type"
112
return self.exponent == 0
113
114
- def has_bias(self):
114
+ def has_bias(self) -> bool:
115
"If the type has a non-zero bias"
116
return self.bias != 0
117
118
- def has_infs(self):
118
+ def has_infs(self) -> bool:
119
"If the type is floating point and supports infinity"
120
return not self._finite_values_only
121
122
- def has_nans(self):
122
+ def has_nans(self) -> bool:
123
return self.nan_repr != NanRepr.NONE.value
124
125
def is_ieee_754(self) -> bool:
126
"""
127
- If the type is a floating point type that follows IEEE 754
127
+ If the type is a floating point type that follows IEEE 754
128
conventions
129
"""
130
return self.nan_repr == NanRepr.IEEE_754.value and \
@@ -136,6 +136,11 @@ if TYPE_CHECKING or not core_C_available:
136
def __repr__(self) -> str:
137
raise NotImplementedError
138
139
+ # __len__ needs to be defined (and has to throw TypeError) for pytorch's
140
+ # opcheck to work.
141
+ def __len__(self) -> int:
142
+ raise TypeError
143
+
144
#
145
# Convenience Constructors
146
#
@@ -153,16 +158,16 @@ if TYPE_CHECKING or not core_C_available:
158
@classmethod
159
def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
160
"""
156
- Create a standard floating point type
161
+ Create a standard floating point type
162
(i.e. follows IEEE 754 conventions).
163
"""
164
return cls(exponent, mantissa, 0, True)
165
166
@classmethod
167
def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
163
- nan_repr: int):
168
+ nan_repr: int) -> 'ScalarType':
169
"""
165
- Create a non-standard floating point type
170
+ Create a non-standard floating point type
171
(i.e. does not follow IEEE 754 conventions).
172
"""
173
return cls(exponent, mantissa, 0, True, finite_values_only,
@@ -175,3 +180,93 @@ elif core_C_available:
180
logger.warning("Failed to import from vllm._core_C with %r", e)
181
182
ScalarType = torch.classes._core_C.ScalarType
183
+
184
+ # Needed for dynamo support of ScalarType.
185
+ @torch._library.register_fake_class("_core_C::ScalarType")
186
+ class FakeScalarType:
187
+
188
+ def __init__(self, scalar_type):
189
+ self.ScalarType = scalar_type
190
+
191
+ def bias_getter(self) -> int:
192
+ return self.ScalarType.bias
193
+
194
+ def exponent_getter(self) -> int:
195
+ return self.ScalarType.exponent
196
+
197
+ def mantissa_getter(self) -> int:
198
+ return self.ScalarType.mantissa
199
+
200
+ def signed_getter(self) -> bool:
201
+ return self.ScalarType.signed
202
+
203
+ def size_bits_getter(self) -> int:
204
+ return self.ScalarType.size_bits
205
+
206
+ @property
207
+ def size_bits(self) -> int:
208
+ return self.ScalarType.size_bits
209
+
210
+ def min(self) -> Union[int, float]:
211
+ return self.ScalarType.min()
212
+
213
+ def max(self) -> Union[int, float]:
214
+ return self.ScalarType.max()
215
+
216
+ def is_signed(self) -> bool:
217
+ return self.ScalarType.is_signed()
218
+
219
+ def is_floating_point(self) -> bool:
220
+ return self.ScalarType.is_floating_point()
221
+
222
+ def is_integer(self) -> bool:
223
+ return self.ScalarType.is_integer()
224
+
225
+ def has_bias(self) -> bool:
226
+ return self.ScalarType.has_bias()
227
+
228
+ def has_infs(self) -> bool:
229
+ return self.ScalarType.has_infs()
230
+
231
+ def has_nans(self) -> bool:
232
+ return self.ScalarType.has_nans()
233
+
234
+ def is_ieee_754(self) -> bool:
235
+ return self.ScalarType.is_ieee_754()
236
+
237
+ def __str__(self) -> str:
238
+ return self.ScalarType.__str__()
239
+
240
+ def __repr__(self) -> str:
241
+ return self.ScalarType.__repr__()
242
+
243
+ def __len__(self) -> int:
244
+ return self.ScalarType.__len__()
245
+
246
+ def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
247
+ return torch.classes._core_C.ScalarType.__obj_flatten__(
248
+ self.ScalarType)
249
+
250
+ @classmethod
251
+ def __obj_unflatten__(
252
+ cls, flat_type: Tuple[Tuple[str, Any], ...]) -> 'ScalarType':
253
+ return cls(
254
+ torch.classes._core_C.ScalarType.__obj_unflatten__(flat_type))
255
+
256
+ @classmethod
257
+ def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
258
+ return ScalarType.int_(size_bits, bias)
259
+
260
+ @classmethod
261
+ def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262
+ return ScalarType.uint(size_bits, bias)
263
+
264
+ @classmethod
265
+ def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
266
+ return ScalarType.float_IEEE754(exponent, mantissa)
267
+
268
+ @classmethod
269
+ def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
270
+ nan_repr: int) -> 'ScalarType':
271
+ return ScalarType.float_(exponent, mantissa, finite_values_only,
272
+ nan_repr)