@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7759ae958f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Kernel][Misc] dynamo support for ScalarType (#7594)

bnellnm committed Aug 16, 2024 at 16:59 UTC 7759ae958ffb9015cd28137c380ffdcbe7648511
2 files changed +149 -24
csrc/core/scalar_type.hpp
+30
@@ -313,6 +313,8 @@ class ScalarType {
313 // have ScalarType inherit from torch::CustomClassHolder and have a constexpr
314 // constructor at the same time (torch::CustomClassHolder does not have a
315 // constexpr destructor)
316 +// See also:
317 +// https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA
318 class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
319 public:
320 ScalarTypeTorch(int64_t exponent, int64_t mantissa, int64_t bias,
@@ -382,6 +384,29 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
384 exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
385 }
386
387 + // This needs to be implemented and throw a TypeError in order for
388 + // PyTorch's opcheck to work on ops that use ScalarTypes.
389 + int64_t len() const {
390 + throw c10::TypeError("__len__ not implemented");
391 + return 0;
392 + }
393 +
394 + // Serialize a ScalarType into a tuple of pairs. Where each pair
395 + // is a (fieldname, value).
396 + // For simplicity, we are just going to convert to a ScalarTypeId.
397 + std::tuple<std::tuple<std::string, int64_t>> obj_flatten() const {
398 + return {{"ScalarType", id()}};
399 + }
400 +
401 + // Deserialize a scalar type that has been serialized by obj_flatten,
402 + // ostensibly from a tuple of (member name, value) pairs, but in reality
403 + // just a ScalarTypeId.
404 + static SelfPtr obj_unflatten(
405 + std::tuple<std::tuple<std::string, int64_t>> const& flat_type) {
406 + return c10::make_intrusive<Self>(
407 + from_id(std::get<1>(std::get<0>(flat_type))));
408 + }
409 +
410 template <typename T>
411 static void bind_readonly_property(torch::class_<Self>& cls,
412 std::string const& name, T Base::*field) {
@@ -457,6 +482,7 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
482 self.get()->min());
483 });
484
485 + bind_function(cls, "__len__", &ScalarTypeTorch::len);
486 bind_function(cls, "__str__", &Base::str);
487 bind_function(cls, "__eq__", [](SelfPtr const& self, SelfPtr const& other) {
488 return *self == *other;
@@ -465,6 +491,10 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
491 return "ScalarType." + self.get()->str();
492 });
493
494 + bind_function(cls, "__obj_flatten__", &ScalarTypeTorch::obj_flatten);
495 + bind_static_function(cls, "__obj_unflatten__",
496 + &ScalarTypeTorch::obj_unflatten);
497 +
498 // Bind static functions (convenience constructors)
499 bind_static_function(cls, "int_", &ScalarTypeTorch::int_);
500 bind_static_function(cls, "uint", &ScalarTypeTorch::uint);
vllm/_core_ext.py
+119 -24
@@ -1,6 +1,6 @@
1 import importlib.util
2 from enum import Enum
3 -from typing import TYPE_CHECKING, Optional, Union
3 +from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
4
5 import torch
6
@@ -31,14 +31,14 @@ if TYPE_CHECKING or not core_C_available:
31 @dataclass(frozen=True)
32 class ScalarType:
33 """
34 - ScalarType can represent a wide range of floating point and integer
35 - types, in particular it can be used to represent sub-byte data types
36 - (something that torch.dtype currently does not support). It is also
34 + ScalarType can represent a wide range of floating point and integer
35 + types, in particular it can be used to represent sub-byte data types
36 + (something that torch.dtype currently does not support). It is also
37 capable of representing types with a bias, i.e.:
38 - `stored_value = value + bias`,
39 - this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40 - of 8). The implementation for this class can be found in
41 - csrc/core/scalar_type.hpp, these type signatures should be kept in sync
38 + `stored_value = value + bias`,
39 + this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40 + of 8). The implementation for this class can be found in
41 + csrc/core/scalar_type.hpp, these type signatures should be kept in sync
42 with that file.
43 """
44
@@ -51,15 +51,15 @@ if TYPE_CHECKING or not core_C_available:
51 mantissa: int
52 """
53 Number of bits in the mantissa if this is a floating point type,
54 - or the number bits representing an integer excluding the sign bit if
54 + or the number bits representing an integer excluding the sign bit if
55 this an integer type.
56 """
57
58 bias: int
59 """
60 - bias used to encode the values in this scalar type
61 - (value = stored_value - bias, default 0) for example if we store the
62 - type as an unsigned integer with a bias of 128 then the value 0 will be
60 + bias used to encode the values in this scalar type
61 + (value = stored_value - bias, default 0) for example if we store the
62 + type as an unsigned integer with a bias of 128 then the value 0 will be
63 stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
64 """
65
@@ -73,7 +73,7 @@ if TYPE_CHECKING or not core_C_available:
73
74 nan_repr: int = NanRepr.IEEE_754.value
75 """
76 - How NaNs are represent in this scalar type, returns NanRepr value.
76 + How NaNs are represent in this scalar type, returns NanRepr value.
77 (not applicable for integer types)
78 """
79
@@ -83,14 +83,14 @@ if TYPE_CHECKING or not core_C_available:
83
84 def min(self) -> Union[int, float]:
85 """
86 - Min representable value for this scalar type.
86 + Min representable value for this scalar type.
87 (accounting for bias if there is one)
88 """
89 raise NotImplementedError
90
91 def max(self) -> Union[int, float]:
92 """
93 - Max representable value for this scalar type.
93 + Max representable value for this scalar type.
94 (accounting for bias if there is one)
95 """
96 raise NotImplementedError
@@ -103,28 +103,28 @@ if TYPE_CHECKING or not core_C_available:
103 """
104 ...
105
106 - def is_floating_point(self):
106 + def is_floating_point(self) -> bool:
107 "If the type is a floating point type"
108 return self.exponent != 0
109
110 - def is_integer(self):
110 + def is_integer(self) -> bool:
111 "If the type is an integer type"
112 return self.exponent == 0
113
114 - def has_bias(self):
114 + def has_bias(self) -> bool:
115 "If the type has a non-zero bias"
116 return self.bias != 0
117
118 - def has_infs(self):
118 + def has_infs(self) -> bool:
119 "If the type is floating point and supports infinity"
120 return not self._finite_values_only
121
122 - def has_nans(self):
122 + def has_nans(self) -> bool:
123 return self.nan_repr != NanRepr.NONE.value
124
125 def is_ieee_754(self) -> bool:
126 """
127 - If the type is a floating point type that follows IEEE 754
127 + If the type is a floating point type that follows IEEE 754
128 conventions
129 """
130 return self.nan_repr == NanRepr.IEEE_754.value and \
@@ -136,6 +136,11 @@ if TYPE_CHECKING or not core_C_available:
136 def __repr__(self) -> str:
137 raise NotImplementedError
138
139 + # __len__ needs to be defined (and has to throw TypeError) for pytorch's
140 + # opcheck to work.
141 + def __len__(self) -> int:
142 + raise TypeError
143 +
144 #
145 # Convenience Constructors
146 #
@@ -153,16 +158,16 @@ if TYPE_CHECKING or not core_C_available:
158 @classmethod
159 def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
160 """
156 - Create a standard floating point type
161 + Create a standard floating point type
162 (i.e. follows IEEE 754 conventions).
163 """
164 return cls(exponent, mantissa, 0, True)
165
166 @classmethod
167 def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
163 - nan_repr: int):
168 + nan_repr: int) -> 'ScalarType':
169 """
165 - Create a non-standard floating point type
170 + Create a non-standard floating point type
171 (i.e. does not follow IEEE 754 conventions).
172 """
173 return cls(exponent, mantissa, 0, True, finite_values_only,
@@ -175,3 +180,93 @@ elif core_C_available:
180 logger.warning("Failed to import from vllm._core_C with %r", e)
181
182 ScalarType = torch.classes._core_C.ScalarType
183 +
184 + # Needed for dynamo support of ScalarType.
185 + @torch._library.register_fake_class("_core_C::ScalarType")
186 + class FakeScalarType:
187 +
188 + def __init__(self, scalar_type):
189 + self.ScalarType = scalar_type
190 +
191 + def bias_getter(self) -> int:
192 + return self.ScalarType.bias
193 +
194 + def exponent_getter(self) -> int:
195 + return self.ScalarType.exponent
196 +
197 + def mantissa_getter(self) -> int:
198 + return self.ScalarType.mantissa
199 +
200 + def signed_getter(self) -> bool:
201 + return self.ScalarType.signed
202 +
203 + def size_bits_getter(self) -> int:
204 + return self.ScalarType.size_bits
205 +
206 + @property
207 + def size_bits(self) -> int:
208 + return self.ScalarType.size_bits
209 +
210 + def min(self) -> Union[int, float]:
211 + return self.ScalarType.min()
212 +
213 + def max(self) -> Union[int, float]:
214 + return self.ScalarType.max()
215 +
216 + def is_signed(self) -> bool:
217 + return self.ScalarType.is_signed()
218 +
219 + def is_floating_point(self) -> bool:
220 + return self.ScalarType.is_floating_point()
221 +
222 + def is_integer(self) -> bool:
223 + return self.ScalarType.is_integer()
224 +
225 + def has_bias(self) -> bool:
226 + return self.ScalarType.has_bias()
227 +
228 + def has_infs(self) -> bool:
229 + return self.ScalarType.has_infs()
230 +
231 + def has_nans(self) -> bool:
232 + return self.ScalarType.has_nans()
233 +
234 + def is_ieee_754(self) -> bool:
235 + return self.ScalarType.is_ieee_754()
236 +
237 + def __str__(self) -> str:
238 + return self.ScalarType.__str__()
239 +
240 + def __repr__(self) -> str:
241 + return self.ScalarType.__repr__()
242 +
243 + def __len__(self) -> int:
244 + return self.ScalarType.__len__()
245 +
246 + def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
247 + return torch.classes._core_C.ScalarType.__obj_flatten__(
248 + self.ScalarType)
249 +
250 + @classmethod
251 + def __obj_unflatten__(
252 + cls, flat_type: Tuple[Tuple[str, Any], ...]) -> 'ScalarType':
253 + return cls(
254 + torch.classes._core_C.ScalarType.__obj_unflatten__(flat_type))
255 +
256 + @classmethod
257 + def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
258 + return ScalarType.int_(size_bits, bias)
259 +
260 + @classmethod
261 + def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262 + return ScalarType.uint(size_bits, bias)
263 +
264 + @classmethod
265 + def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
266 + return ScalarType.float_IEEE754(exponent, mantissa)
267 +
268 + @classmethod
269 + def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
270 + nan_repr: int) -> 'ScalarType':
271 + return ScalarType.float_(exponent, mantissa, finite_values_only,
272 + nan_repr)