A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] Allow ScalarType to be compiled with pytorch 2.3 and add checks for registering FakeScalarType and dynamo support. (#7886)
bnellnm committed
Aug 27, 2024 at 23:13 UTC
c166e7e43e7bb398835d1933a69d106b47f6cc8d
4 files changed
+84
-67
csrc/core/scalar_type.hpp
+2
-1
@@ -387,7 +387,8 @@ class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
387
// This needs to be implemented and throw a TypeError in order for
388
// PyTorch's opcheck to work on ops that use ScalarTypes.
389
int64_t len() const {
390
- throw c10::TypeError("__len__ not implemented");
390
+ throw c10::TypeError({__func__, __FILE__, static_cast<uint32_t>(__LINE__)},
391
+ "__len__ not implemented");
392
return 0;
393
}
394
vllm/_core_ext.py
+70
-64
@@ -181,92 +181,98 @@ elif core_C_available:
181
182
ScalarType = torch.classes._core_C.ScalarType
183
184
- # Needed for dynamo support of ScalarType.
185
- @torch._library.register_fake_class("_core_C::ScalarType")
186
- class FakeScalarType:
184
+ if (hasattr(torch, "_library")
185
+ and hasattr(torch._library, "register_fake_class")):
186
+ # Needed for dynamo support of ScalarType.
187
+ @torch._library.register_fake_class("_core_C::ScalarType")
188
+ class FakeScalarType:
189
188
- def __init__(self, scalar_type):
189
- self.ScalarType = scalar_type
190
+ def __init__(self, scalar_type):
191
+ self.ScalarType = scalar_type
192
191
- def bias_getter(self) -> int:
192
- return self.ScalarType.bias
193
+ def bias_getter(self) -> int:
194
+ return self.ScalarType.bias
195
194
- def exponent_getter(self) -> int:
195
- return self.ScalarType.exponent
196
+ def exponent_getter(self) -> int:
197
+ return self.ScalarType.exponent
198
197
- def mantissa_getter(self) -> int:
198
- return self.ScalarType.mantissa
199
+ def mantissa_getter(self) -> int:
200
+ return self.ScalarType.mantissa
201
200
- def signed_getter(self) -> bool:
201
- return self.ScalarType.signed
202
+ def signed_getter(self) -> bool:
203
+ return self.ScalarType.signed
204
203
- def size_bits_getter(self) -> int:
204
- return self.ScalarType.size_bits
205
+ def size_bits_getter(self) -> int:
206
+ return self.ScalarType.size_bits
207
206
- @property
207
- def size_bits(self) -> int:
208
- return self.ScalarType.size_bits
208
+ @property
209
+ def size_bits(self) -> int:
210
+ return self.ScalarType.size_bits
211
210
- def min(self) -> Union[int, float]:
211
- return self.ScalarType.min()
212
+ def min(self) -> Union[int, float]:
213
+ return self.ScalarType.min()
214
213
- def max(self) -> Union[int, float]:
214
- return self.ScalarType.max()
215
+ def max(self) -> Union[int, float]:
216
+ return self.ScalarType.max()
217
216
- def is_signed(self) -> bool:
217
- return self.ScalarType.is_signed()
218
+ def is_signed(self) -> bool:
219
+ return self.ScalarType.is_signed()
220
219
- def is_floating_point(self) -> bool:
220
- return self.ScalarType.is_floating_point()
221
+ def is_floating_point(self) -> bool:
222
+ return self.ScalarType.is_floating_point()
223
222
- def is_integer(self) -> bool:
223
- return self.ScalarType.is_integer()
224
+ def is_integer(self) -> bool:
225
+ return self.ScalarType.is_integer()
226
225
- def has_bias(self) -> bool:
226
- return self.ScalarType.has_bias()
227
+ def has_bias(self) -> bool:
228
+ return self.ScalarType.has_bias()
229
228
- def has_infs(self) -> bool:
229
- return self.ScalarType.has_infs()
230
+ def has_infs(self) -> bool:
231
+ return self.ScalarType.has_infs()
232
231
- def has_nans(self) -> bool:
232
- return self.ScalarType.has_nans()
233
+ def has_nans(self) -> bool:
234
+ return self.ScalarType.has_nans()
235
234
- def is_ieee_754(self) -> bool:
235
- return self.ScalarType.is_ieee_754()
236
+ def is_ieee_754(self) -> bool:
237
+ return self.ScalarType.is_ieee_754()
238
237
- def __str__(self) -> str:
238
- return self.ScalarType.__str__()
239
+ def __str__(self) -> str:
240
+ return self.ScalarType.__str__()
241
240
- def __repr__(self) -> str:
241
- return self.ScalarType.__repr__()
242
+ def __repr__(self) -> str:
243
+ return self.ScalarType.__repr__()
244
243
- def __len__(self) -> int:
244
- return self.ScalarType.__len__()
245
+ def __len__(self) -> int:
246
+ return self.ScalarType.__len__()
247
246
- def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
247
- return torch.classes._core_C.ScalarType.__obj_flatten__(
248
- self.ScalarType)
248
+ def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
249
+ return torch.classes._core_C.ScalarType.__obj_flatten__(
250
+ self.ScalarType)
251
250
- @classmethod
251
- def __obj_unflatten__(
252
- cls, flat_type: Tuple[Tuple[str, Any], ...]) -> 'ScalarType':
253
- return cls(
254
- torch.classes._core_C.ScalarType.__obj_unflatten__(flat_type))
252
+ @classmethod
253
+ def __obj_unflatten__(
254
+ cls, flat_type: Tuple[Tuple[str, Any],
255
+ ...]) -> 'ScalarType':
256
+ return cls(
257
+ torch.classes._core_C.ScalarType.__obj_unflatten__(
258
+ flat_type))
259
256
- @classmethod
257
- def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
258
- return ScalarType.int_(size_bits, bias)
260
+ @classmethod
261
+ def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262
+ return ScalarType.int_(size_bits, bias)
263
260
- @classmethod
261
- def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262
- return ScalarType.uint(size_bits, bias)
264
+ @classmethod
265
+ def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
266
+ return ScalarType.uint(size_bits, bias)
267
264
- @classmethod
265
- def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
266
- return ScalarType.float_IEEE754(exponent, mantissa)
268
+ @classmethod
269
+ def float_IEEE754(cls, exponent: int,
270
+ mantissa: int) -> 'ScalarType':
271
+ return ScalarType.float_IEEE754(exponent, mantissa)
272
268
- @classmethod
269
- def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
270
- nan_repr: int) -> 'ScalarType':
271
- return ScalarType.float_(exponent, mantissa, finite_values_only,
272
- nan_repr)
273
+ @classmethod
274
+ def float_(cls, exponent: int, mantissa: int,
275
+ finite_values_only: bool,
276
+ nan_repr: int) -> 'ScalarType':
277
+ return ScalarType.float_(exponent, mantissa,
278
+ finite_values_only, nan_repr)
vllm/utils.py
+9
@@ -25,6 +25,7 @@ import numpy.typing as npt
25
import psutil
26
import torch
27
import torch.types
28
+from packaging.version import Version
29
from typing_extensions import ParamSpec, TypeIs, assert_never
30
31
import vllm.envs as envs
@@ -1114,3 +1115,11 @@ async def _run_task_with_lock(task: Callable, lock: asyncio.Lock, *args,
1115
"""Utility function to run async task in a lock"""
1116
async with lock:
1117
return await task(*args, **kwargs)
1118
+
1119
+
1120
+# Using dynamo with vLLM doesn't really work well with PyTorch versions < 2.4.0.
1121
+# In particular, the FakeScalarType is not supported for earlier versions of
1122
+# PyTorch which breaks dynamo for any ops registered using ScalarType.
1123
+def supports_dynamo() -> bool:
1124
+ base_torch_version = Version(Version(torch.__version__).base_version)
1125
+ return base_torch_version >= Version("2.4.0")
vllm/worker/model_runner.py
+3
-2
@@ -44,7 +44,8 @@ from vllm.sampling_params import SamplingParams
44
from vllm.sequence import (IntermediateTensors, SamplerOutput,
45
SequenceGroupMetadata)
46
from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
47
- flatten_2d_lists, is_hip, is_pin_memory_available)
47
+ flatten_2d_lists, is_hip, is_pin_memory_available,
48
+ supports_dynamo)
49
from vllm.worker.model_runner_base import (
50
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
51
_add_attn_metadata_broadcastable_dict,
@@ -946,7 +947,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
947
"provided. Defaulting to scaling factors of 1.0. "
948
"This may lead to less accurate results!")
949
949
- if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE:
950
+ if envs.VLLM_TEST_DYNAMO_GRAPH_CAPTURE and supports_dynamo():
951
self.model = torch.compile(self.model,
952
fullgraph=True,
953
backend="eager")