@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c166e7e43e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] Allow ScalarType to be compiled with pytorch 2.3 and add checks for registering FakeScalarType and dynamo support. (#7886)

bnellnm committed Aug 27, 2024 at 23:13 UTC c166e7e43e7bb398835d1933a69d106b47f6cc8d
4 files changed +84 -67
csrc/core/scalar_type.hpp
+2 -1
@@ -387,7 +387,8 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
387 // This needs to be implemented and throw a TypeError in order for
388 // PyTorch's opcheck to work on ops that use ScalarTypes.
389 int64_t len() const {
390 - throw c10::TypeError("__len__ not implemented");
390 + throw c10::TypeError({__func__, __FILE__, static_cast<uint32_t>(__LINE__)},
391 + "__len__ not implemented");
392 return 0;
393 }
394
vllm/_core_ext.py
+70 -64
@@ -181,92 +181,98 @@ elif core_C_available:
181
182 ScalarType = torch.classes._core_C.ScalarType
183
184 - # Needed for dynamo support of ScalarType.
185 - @torch._library.register_fake_class("_core_C::ScalarType")
186 - class FakeScalarType:
184 + if (hasattr(torch, "_library")
185 + and hasattr(torch._library, "register_fake_class")):
186 + # Needed for dynamo support of ScalarType.
187 + @torch._library.register_fake_class("_core_C::ScalarType")
188 + class FakeScalarType:
189
188 - def __init__(self, scalar_type):
189 - self.ScalarType = scalar_type
190 + def __init__(self, scalar_type):
191 + self.ScalarType = scalar_type
192
191 - def bias_getter(self) -> int:
192 - return self.ScalarType.bias
193 + def bias_getter(self) -> int:
194 + return self.ScalarType.bias
195
194 - def exponent_getter(self) -> int:
195 - return self.ScalarType.exponent
196 + def exponent_getter(self) -> int:
197 + return self.ScalarType.exponent
198
197 - def mantissa_getter(self) -> int:
198 - return self.ScalarType.mantissa
199 + def mantissa_getter(self) -> int:
200 + return self.ScalarType.mantissa
201
200 - def signed_getter(self) -> bool:
201 - return self.ScalarType.signed
202 + def signed_getter(self) -> bool:
203 + return self.ScalarType.signed
204
203 - def size_bits_getter(self) -> int:
204 - return self.ScalarType.size_bits
205 + def size_bits_getter(self) -> int:
206 + return self.ScalarType.size_bits
207
206 - @property
207 - def size_bits(self) -> int:
208 - return self.ScalarType.size_bits
208 + @property
209 + def size_bits(self) -> int:
210 + return self.ScalarType.size_bits
211
210 - def min(self) -> Union[int, float]:
211 - return self.ScalarType.min()
212 + def min(self) -> Union[int, float]:
213 + return self.ScalarType.min()
214
213 - def max(self) -> Union[int, float]:
214 - return self.ScalarType.max()
215 + def max(self) -> Union[int, float]:
216 + return self.ScalarType.max()
217
216 - def is_signed(self) -> bool:
217 - return self.ScalarType.is_signed()
218 + def is_signed(self) -> bool:
219 + return self.ScalarType.is_signed()
220
219 - def is_floating_point(self) -> bool:
220 - return self.ScalarType.is_floating_point()
221 + def is_floating_point(self) -> bool:
222 + return self.ScalarType.is_floating_point()
223
222 - def is_integer(self) -> bool:
223 - return self.ScalarType.is_integer()
224 + def is_integer(self) -> bool:
225 + return self.ScalarType.is_integer()
226
225 - def has_bias(self) -> bool:
226 - return self.ScalarType.has_bias()
227 + def has_bias(self) -> bool:
228 + return self.ScalarType.has_bias()
229
228 - def has_infs(self) -> bool:
229 - return self.ScalarType.has_infs()
230 + def has_infs(self) -> bool:
231 + return self.ScalarType.has_infs()
232
231 - def has_nans(self) -> bool:
232 - return self.ScalarType.has_nans()
233 + def has_nans(self) -> bool:
234 + return self.ScalarType.has_nans()
235
234 - def is_ieee_754(self) -> bool:
235 - return self.ScalarType.is_ieee_754()
236 + def is_ieee_754(self) -> bool:
237 + return self.ScalarType.is_ieee_754()
238
237 - def __str__(self) -> str:
238 - return self.ScalarType.__str__()
239 + def __str__(self) -> str:
240 + return self.ScalarType.__str__()
241
240 - def __repr__(self) -> str:
241 - return self.ScalarType.__repr__()
242 + def __repr__(self) -> str:
243 + return self.ScalarType.__repr__()
244
243 - def __len__(self) -> int:
244 - return self.ScalarType.__len__()
245 + def __len__(self) -> int:
246 + return self.ScalarType.__len__()
247
246 - def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
247 - return torch.classes._core_C.ScalarType.__obj_flatten__(
248 - self.ScalarType)
248 + def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
249 + return torch.classes._core_C.ScalarType.__obj_flatten__(
250 + self.ScalarType)
251
250 - @classmethod
251 - def __obj_unflatten__(
252 - cls, flat_type: Tuple[Tuple[str, Any], ...]) -> 'ScalarType':
253 - return cls(
254 - torch.classes._core_C.ScalarType.__obj_unflatten__(flat_type))
252 + @classmethod
253 + def __obj_unflatten__(
254 + cls, flat_type: Tuple[Tuple[str, Any],
255 + ...]) -> 'ScalarType':
256 + return cls(
257 + torch.classes._core_C.ScalarType.__obj_unflatten__(
258 + flat_type))
259
256 - @classmethod
257 - def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
258 - return ScalarType.int_(size_bits, bias)
260 + @classmethod
261 + def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262 + return ScalarType.int_(size_bits, bias)
263
260 - @classmethod
261 - def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262 - return ScalarType.uint(size_bits, bias)
264 + @classmethod
265 + def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
266 + return ScalarType.uint(size_bits, bias)
267
264 - @classmethod
265 - def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
266 - return ScalarType.float_IEEE754(exponent, mantissa)
268 + @classmethod
269 + def float_IEEE754(cls, exponent: int,
270 + mantissa: int) -> 'ScalarType':
271 + return ScalarType.float_IEEE754(exponent, mantissa)
272
268 - @classmethod
269 - def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
270 - nan_repr: int) -> 'ScalarType':
271 - return ScalarType.float_(exponent, mantissa, finite_values_only,
272 - nan_repr)
273 + @classmethod
274 + def float_(cls, exponent: int, mantissa: int,
275 + finite_values_only: bool,
276 + nan_repr: int) -> 'ScalarType':
277 + return ScalarType.float_(exponent, mantissa,
278 + finite_values_only, nan_repr)
vllm/utils.py
+9
@@ -25,6 +25,7 @@ import numpy.typing as npt
25 import psutil
26 import torch
27 import torch.types
28 +from packaging.version import Version
29 from typing_extensions import ParamSpec, TypeIs, assert_never
30
31 import vllm.envs as envs
@@ -1114,3 +1115,11 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1115 """Utility function to run async task in a lock"""
1116 async with lock:
1117 return await task(*args, **kwargs)
1118 +
1119 +
1120 +# Using dynamo with vLLM doesn't really work well with PyTorch versions < 2.4.0.
1121 +# In particular, the FakeScalarType is not supported for earlier versions of
1122 +# PyTorch which breaks dynamo for any ops registered using ScalarType.
1123 +def supports_dynamo() -> bool:
1124 + base_torch_version = Version(Version(torch.__version__).base_version)
1125 + return base_torch_version >= Version("2.4.0")
vllm/worker/model_runner.py
+3 -2
@@ -44,7 +44,8 @@ from vllm.sampling_params import SamplingParams
44 from vllm.sequence import (IntermediateTensors, SamplerOutput,
45 SequenceGroupMetadata)
46 from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
47 - flatten_2d_lists, is_hip, is_pin_memory_available)
47 + flatten_2d_lists, is_hip, is_pin_memory_available,
48 + supports_dynamo)
49 from vllm.worker.model_runner_base import (
50 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
51 _add_attn_metadata_broadcastable_dict,
@@ -946,7 +947,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
947 "provided. Defaulting to scaling factors of 1.0. "
948 "This may lead to less accurate results!")
949
949 - if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE:
950 + if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
951 self.model = torch.compile(self.model,
952 fullgraph=True,
953 backend="eager")