A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix support for dimension like integers and ScalarType (#9299)
bnellnm committed
Oct 17, 2024 at 15:08 UTC
eca2c5f7c00d6c0b08051972d1e80fe822e7d1b8
22 files changed
+427
-677
.buildkite/test-pipeline.yaml
+6
-8
@@ -230,14 +230,12 @@ steps:
230
commands:
231
- pytest -v -s compile/test_basic_correctness.py
232
233
-# TODO: re-write in comparison tests, and fix symbolic shape
234
-# for quantization ops.
235
-# - label: "PyTorch Fullgraph Test" # 18min
236
-# source_file_dependencies:
237
-# - vllm/
238
-# - tests/compile
239
-# commands:
240
-# - pytest -v -s compile/test_full_graph.py
233
+- label: "PyTorch Fullgraph Test" # 18min
234
+ source_file_dependencies:
235
+ - vllm/
236
+ - tests/compile
237
+ commands:
238
+ - pytest -v -s compile/test_full_graph.py
239
240
- label: Kernels Test %N # 1h each
241
mirror_hardwares: [amd]
CMakeLists.txt
-18
@@ -83,24 +83,6 @@ endif()
83
#
84
find_package(Torch REQUIRED)
85
86
-#
87
-message(STATUS "Enabling core extension.")
88
-
89
-# Define _core_C extension
90
-# built for (almost) every target platform, (excludes TPU and Neuron)
91
-
92
-set(VLLM_EXT_SRC
93
- "csrc/core/torch_bindings.cpp")
94
-
95
-define_gpu_extension_target(
96
- _core_C
97
- DESTINATION vllm
98
- LANGUAGE CXX
99
- SOURCES ${VLLM_EXT_SRC}
100
- COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
101
- USE_SABI 3
102
- WITH_SOABI)
103
-
86
#
87
# Forward the non-CUDA device extensions to external CMake scripts.
88
#
csrc/core/scalar_type.hpp
+4
-205
@@ -1,6 +1,7 @@
1
#pragma once
2
3
-#include <torch/custom_class.h>
3
+// For TORCH_CHECK
4
+#include <torch/library.h>
5
6
namespace vllm {
7
@@ -9,12 +10,7 @@ namespace vllm {
10
// in particular it can be used to represent sub-byte data types (something
11
// that torch.dtype currently does not support).
12
//
12
-// ScalarTypeTorch is a subclass of ScalarType that is compatible with
13
-// TORCH_LIBRARY, making it accessible from Python as well meaning this class
14
-// can be used as a argument for custom operators, helping to simplify these
15
-// interfaces.
16
-//
17
-// The type definitions on the Python side can be found in: vllm/_core_ext.pyi
13
+// The type definitions on the Python side can be found in: vllm/scalar_type.py
14
// these type definitions should be kept up to date with any Python API changes
15
// here.
16
//
@@ -308,204 +304,7 @@ class ScalarType {
304
}
305
};
306
311
-// Create a TORCH_LIBRARY compatible version of ScalarType (i.e. inherit from
312
-// torch::CustomClassHolder), we use multiple inheritance here since we cannot
313
-// have ScalarType inherit from torch::CustomClassHolder and have a constexpr
314
-// constructor at the same time (torch::CustomClassHolder does not have a
315
-// constexpr destructor)
316
-// See also:
317
-// https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA
318
-class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
319
- public:
320
- ScalarTypeTorch(int64_t exponent, int64_t mantissa, int64_t bias,
321
- bool _signed)
322
- : ScalarType(exponent, mantissa, bias, _signed){};
323
-
324
- ScalarTypeTorch(ScalarType type) : ScalarType(type){};
325
-
326
- using Base = ScalarType;
327
- using Self = ScalarTypeTorch;
328
- using SelfPtr = c10::intrusive_ptr<Self>;
329
-
330
- static void check_size_bits(int64_t size_bits, bool signed_) {
331
- TORCH_CHECK(
332
- size_bits <=
333
- std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
334
- "size_bits bit width is too large to be represented");
335
- }
336
-
337
- static void check_bias(int64_t bias) {
338
- using Bias = decltype(std::declval<Self>().bias);
339
- TORCH_CHECK(bias <= std::numeric_limits<Bias>::max() &&
340
- bias >= std::numeric_limits<Bias>::min(),
341
- "bias too large or small to be represented");
342
- }
343
-
344
- static void check_exponent(int64_t exponent) {
345
- TORCH_CHECK(
346
- exponent <=
347
- std::numeric_limits<decltype(std::declval<Self>().exponent)>::max(),
348
- "exponent bit width is too large to be represented");
349
- }
350
-
351
- static void check_mantissa(int64_t mantissa) {
352
- TORCH_CHECK(
353
- mantissa <=
354
- std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
355
- "mantissa bit width is too large to be represented");
356
- }
357
-
358
- static SelfPtr int_(int64_t size_bits, c10::optional<int64_t> bias) {
359
- check_size_bits(size_bits, true);
360
- check_bias(bias.value_or(0));
361
- return c10::make_intrusive<Self>(
362
- ScalarType::int_(size_bits, bias.value_or(0)));
363
- }
364
-
365
- static SelfPtr uint(int64_t size_bits, c10::optional<int64_t> bias) {
366
- check_size_bits(size_bits, true);
367
- check_bias(bias.value_or(0));
368
- return c10::make_intrusive<Self>(
369
- ScalarType::uint(size_bits, bias.value_or(0)));
370
- }
371
-
372
- static SelfPtr float_IEEE754(int64_t exponent, int64_t mantissa) {
373
- check_mantissa(mantissa);
374
- check_exponent(exponent);
375
- return c10::make_intrusive<Self>(
376
- ScalarType::float_IEEE754(exponent, mantissa));
377
- }
378
-
379
- static SelfPtr float_(int64_t exponent, int64_t mantissa,
380
- bool finite_values_only, int64_t nan_repr) {
381
- check_mantissa(mantissa);
382
- check_exponent(exponent);
383
- return c10::make_intrusive<Self>(ScalarType::float_(
384
- exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
385
- }
386
-
387
- // This needs to be implemented and throw a TypeError in order for
388
- // PyTorch's opcheck to work on ops that use ScalarTypes.
389
- int64_t len() const {
390
- throw c10::TypeError({__func__, __FILE__, static_cast<uint32_t>(__LINE__)},
391
- "__len__ not implemented");
392
- return 0;
393
- }
394
-
395
- // Serialize a ScalarType into a tuple of pairs. Where each pair
396
- // is a (fieldname, value).
397
- // For simplicity, we are just going to convert to a ScalarTypeId.
398
- std::tuple<std::tuple<std::string, int64_t>> obj_flatten() const {
399
- return {{"ScalarType", id()}};
400
- }
401
-
402
- // Deserialize a scalar type that has been serialized by obj_flatten,
403
- // ostensibly from a tuple of (member name, value) pairs, but in reality
404
- // just a ScalarTypeId.
405
- static SelfPtr obj_unflatten(
406
- std::tuple<std::tuple<std::string, int64_t>> const& flat_type) {
407
- return c10::make_intrusive<Self>(
408
- from_id(std::get<1>(std::get<0>(flat_type))));
409
- }
410
-
411
- template <typename T>
412
- static void bind_readonly_property(torch::class_<Self>& cls,
413
- std::string const& name, T Base::*field) {
414
- auto getter_func_helper = [field = std::move(field)](SelfPtr const& self) {
415
- if constexpr (std::is_member_function_pointer_v<decltype(field)>) {
416
- return (self.get()->*field)();
417
- } else {
418
- return self.get()->*field;
419
- }
420
- };
421
-
422
- auto getter_func = [field = std::move(field),
423
- getter_func_helper = std::move(getter_func_helper)](
424
- SelfPtr const& self) {
425
- auto val = getter_func_helper(self);
426
- // upconvert uint8_t, int32_t etc. to int64_t for python
427
- if constexpr (std::is_integral_v<T>) {
428
- return static_cast<int64_t>(val);
429
- } else {
430
- return val;
431
- }
432
- };
433
-
434
- cls.def_property(name, getter_func);
435
- }
436
-
437
- template <typename MemberFunc, typename Cls>
438
- static void bind_function(torch::class_<Self>& cls, const std::string& name,
439
- MemberFunc Cls::*member) {
440
- cls.def(name, [member = std::move(member)](SelfPtr const& self) {
441
- return (self.get()->*member)();
442
- });
443
- }
444
-
445
- template <typename Func>
446
- static void bind_function(torch::class_<Self>& cls, const std::string& name,
447
- Func func) {
448
- cls.def(name, func);
449
- }
450
-
451
- template <typename Func>
452
- static void bind_static_function(torch::class_<Self>& cls,
453
- const std::string& name, Func func) {
454
- cls.def_static(name, func);
455
- }
456
-
457
- static void bind_class(torch::Library& lib) {
458
- auto cls = lib.class_<ScalarTypeTorch>("ScalarType")
459
- .def(torch::init<int64_t, int64_t, int64_t, bool>());
460
-
461
- // Bind Properties
462
- bind_readonly_property(cls, "mantissa", &Base::mantissa);
463
- bind_readonly_property(cls, "exponent", &Base::exponent);
464
- bind_readonly_property(cls, "bias", &Base::bias);
465
- bind_readonly_property(cls, "signed", &Base::is_signed);
466
- bind_readonly_property(cls, "size_bits", &Base::size_bits);
467
-
468
- // Bind member functions
469
- bind_function(cls, "is_signed", &Base::is_signed);
470
- bind_function(cls, "is_integer", &Base::is_integer);
471
- bind_function(cls, "is_floating_point", &Base::is_floating_point);
472
- bind_function(cls, "is_ieee_754", &Base::is_ieee_754);
473
- bind_function(cls, "has_nans", &Base::has_nans);
474
- bind_function(cls, "has_infs", &Base::has_infs);
475
- bind_function(cls, "has_bias", &Base::has_bias);
476
-
477
- bind_function(cls, "max", [](SelfPtr const& self) {
478
- return std::visit([](auto arg) { return c10::IValue(arg); },
479
- self.get()->max());
480
- });
481
- bind_function(cls, "min", [](SelfPtr const& self) {
482
- return std::visit([](auto arg) { return c10::IValue(arg); },
483
- self.get()->min());
484
- });
485
-
486
- bind_function(cls, "__len__", &ScalarTypeTorch::len);
487
- bind_function(cls, "__str__", &Base::str);
488
- bind_function(cls, "__eq__", [](SelfPtr const& self, SelfPtr const& other) {
489
- return *self == *other;
490
- });
491
- bind_function(cls, "__repr__", [](SelfPtr const& self) {
492
- return "ScalarType." + self.get()->str();
493
- });
494
-
495
- bind_function(cls, "__obj_flatten__", &ScalarTypeTorch::obj_flatten);
496
- bind_static_function(cls, "__obj_unflatten__",
497
- &ScalarTypeTorch::obj_unflatten);
498
-
499
- // Bind static functions (convenience constructors)
500
- bind_static_function(cls, "int_", &ScalarTypeTorch::int_);
501
- bind_static_function(cls, "uint", &ScalarTypeTorch::uint);
502
- bind_static_function(cls, "float_IEEE754", &ScalarTypeTorch::float_IEEE754);
503
- bind_static_function(cls, "float_", &ScalarTypeTorch::float_);
504
- }
505
-};
506
-
507
-using ScalarTypeId = int64_t;
508
-using ScalarTypeTorchPtr = c10::intrusive_ptr<ScalarTypeTorch>;
307
+using ScalarTypeId = ScalarType::Id;
308
309
// "rust style" names generally following:
310
// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L60-L70
csrc/core/torch_bindings.cpp
deleted
-16
@@ -1,16 +0,0 @@
1
-#include <torch/library.h>
2
-
3
-#include "scalar_type.hpp"
4
-#include "registration.h"
5
-
6
-// Note the CORE exstension will be built for (almost) all hardware targets so
7
-// new additions must account for this. (currently not built for TPU and Neuron)
8
-
9
-TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, lib) {
10
- // ScalarType, a custom class for representing data types that supports
11
- // quantized types, declared here so it can be used when creating interfaces
12
- // for custom ops.
13
- vllm::ScalarTypeTorch::bind_class(lib);
14
-}
15
-
16
-REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/moe/marlin_moe_ops.cu
+8
-7
@@ -484,21 +484,22 @@ torch::Tensor marlin_gemm_moe(
484
const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
485
torch::Tensor& b_zeros, const torch::Tensor& g_idx,
486
const torch::Tensor& perm, torch::Tensor& workspace,
487
- vllm::ScalarTypeTorchPtr const& b_q_type, int64_t size_m, int64_t size_n,
487
+ vllm::ScalarTypeId const b_q_type_id, int64_t size_m, int64_t size_n,
488
int64_t size_k, bool is_k_full, int64_t num_experts, int64_t topk,
489
int64_t moe_block_size, bool replicate_input, bool apply_weights) {
490
+ vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
491
bool has_zp = b_zeros.size(1) != 0;
492
if (has_zp) {
493
TORCH_CHECK(
493
- *b_q_type == vllm::kU4,
494
- "b_q_type must be u4 when has_zp = True. Got = ", b_q_type->str());
494
+ b_q_type == vllm::kU4,
495
+ "b_q_type must be u4 when has_zp = True. Got = ", b_q_type.str());
496
} else {
497
TORCH_CHECK(
497
- *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
498
- "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
498
+ b_q_type == vllm::kU4B8 || b_q_type == vllm::kU8B128,
499
+ "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type.str());
500
}
501
501
- int pack_factor = 32 / b_q_type->size_bits();
502
+ int pack_factor = 32 / b_q_type.size_bits();
503
504
int max_par = 4;
505
@@ -575,7 +576,7 @@ torch::Tensor marlin_gemm_moe(
576
topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
577
b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
578
expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
578
- *b_q_type, has_act_order, is_k_full, has_zp, num_groups, group_size,
579
+ b_q_type, has_act_order, is_k_full, has_zp, num_groups, group_size,
580
num_experts, topk, moe_block_size, dev,
581
at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms, max_par,
582
replicate_input, apply_weights);
csrc/moe/torch_bindings.cpp
+3
-2
@@ -13,8 +13,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
13
"marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
14
"Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
15
"b_zeros, Tensor! g_idx, Tensor! perm, Tensor! workspace, "
16
- "__torch__.torch.classes._core_C.ScalarType b_q_type, int size_m, "
17
- "int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
16
+ "int b_q_type, SymInt size_m, "
17
+ "SymInt size_n, SymInt size_k, bool is_k_full, int num_experts, int "
18
+ "topk, "
19
"int moe_block_size, bool replicate_input, bool apply_weights)"
20
" -> Tensor");
21
// conditionally compiled so impl registration is in source file
csrc/quantization/gptq_marlin/gptq_marlin.cu
+12
-11
@@ -80,7 +80,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
80
torch::Tensor& b_scales, torch::Tensor& b_zeros,
81
torch::Tensor& g_idx, torch::Tensor& perm,
82
torch::Tensor& workspace,
83
- vllm::ScalarTypeTorchPtr const& b_q_type,
83
+ vllm::ScalarTypeId const b_q_type_id,
84
int64_t size_m, int64_t size_n, int64_t size_k,
85
bool is_k_full, bool has_zp) {
86
TORCH_CHECK_NOT_IMPLEMENTED(false,
@@ -2132,22 +2132,23 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2132
torch::Tensor& b_scales, torch::Tensor& b_zeros,
2133
torch::Tensor& g_idx, torch::Tensor& perm,
2134
torch::Tensor& workspace,
2135
- vllm::ScalarTypeTorchPtr const& b_q_type,
2135
+ vllm::ScalarTypeId const& b_q_type_id,
2136
int64_t size_m, int64_t size_n, int64_t size_k,
2137
bool is_k_full, bool has_zp,
2138
bool use_fp32_reduce) {
2139
+ vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
2140
if (has_zp) {
2140
- TORCH_CHECK(*b_q_type == vllm::kU4 || *b_q_type == vllm::kU8,
2141
- "b_q_type must be u4 or u8 when has_zp = True. Got = ",
2142
- b_q_type->str());
2141
+ TORCH_CHECK(
2142
+ b_q_type == vllm::kU4 || b_q_type == vllm::kU8,
2143
+ "b_q_type must be u4 or u8 when has_zp = True. Got = ", b_q_type.str());
2144
} else {
2145
TORCH_CHECK(
2145
- *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
2146
+ b_q_type == vllm::kU4B8 || b_q_type == vllm::kU8B128,
2147
"b_q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ",
2147
- b_q_type->str());
2148
+ b_q_type.str());
2149
}
2150
2150
- int pack_factor = 32 / b_q_type->size_bits();
2151
+ int pack_factor = 32 / b_q_type.size_bits();
2152
2153
// Verify A
2154
TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
@@ -2279,7 +2280,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2280
c_tmp.data_ptr<float>(), b_scales.data_ptr<at::Half>(),
2281
b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2282
a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2282
- workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2283
+ workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2284
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2285
thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2286
} else if (a.scalar_type() == at::ScalarType::BFloat16) {
@@ -2288,7 +2289,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2289
c.data_ptr<at::BFloat16>(), c_tmp.data_ptr<float>(),
2290
b_scales.data_ptr<at::BFloat16>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2291
perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2291
- workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2292
+ workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2293
num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2294
thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2295
} else {
@@ -2302,4 +2303,4 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2303
2304
TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
2305
m.impl("gptq_marlin_gemm", &gptq_marlin_gemm);
2305
-}
\ No newline at end of file
2306
+}
csrc/quantization/machete/machete_pytorch.cu
+9
-7
@@ -38,9 +38,10 @@ static auto scalar_type_dispatch(ScalarType const& type, Fn fn) {
38
// Interface
39
//
40
41
-std::vector<std::string> supported_schedules(ScalarTypeTorchPtr const& btype) {
41
+std::vector<std::string> supported_schedules(ScalarTypeId const btype_id) {
42
#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
43
- return scalar_type_dispatch(*btype, [&](auto BType) {
43
+ vllm::ScalarType b_type = ScalarType::from_id(btype_id);
44
+ return scalar_type_dispatch(b_type, [&](auto BType) {
45
return GemmDispatcher<half_t, decltype(BType)>::supported_schedules();
46
});
47
#else
@@ -49,7 +50,7 @@ std::vector<std::string> supported_schedules(ScalarTypeTorchPtr const& btype) {
50
}
51
52
torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
52
- ScalarTypeTorchPtr const& btype,
53
+ ScalarTypeId const btype_id,
54
c10::optional<torch::Tensor> const& scales,
55
c10::optional<torch::Tensor> const& zeros,
56
c10::optional<int64_t> group_size,
@@ -57,6 +58,7 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
58
c10::optional<double> alpha, c10::optional<double> beta,
59
c10::optional<std::string> schedule) {
60
#if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
61
+ ScalarType const btype = ScalarType::from_id(btype_id);
62
auto args = PyTorchArguments{.A = A,
63
.B = B,
64
.scales = scales,
@@ -67,7 +69,7 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
69
.beta = beta,
70
.schedule = schedule};
71
70
- return scalar_type_dispatch(*btype, [&](auto BType) {
72
+ return scalar_type_dispatch(btype, [&](auto BType) {
73
return AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(
74
A.scalar_type(), "machete_gemm", [&] {
75
using ComputeType = equivalent_cutlass_type_t<scalar_t>;
@@ -79,9 +81,9 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
81
#endif
82
}
83
82
-torch::Tensor prepack_B(torch::Tensor const& B,
83
- vllm::ScalarTypeTorchPtr const& btype) {
84
- return scalar_type_dispatch(*btype, [&](auto BType) {
84
+torch::Tensor prepack_B(torch::Tensor const& B, ScalarTypeId const btype_id) {
85
+ ScalarType const btype = ScalarType::from_id(btype_id);
86
+ return scalar_type_dispatch(btype, [&](auto BType) {
87
return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
88
});
89
}
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+8
-7
@@ -89,7 +89,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
89
torch::Tensor& b_meta,
90
torch::Tensor& b_scales,
91
torch::Tensor& workspace,
92
- vllm::ScalarTypeTorchPtr const& b_q_type,
92
+ vllm::ScalarTypeId const b_q_type_id,
93
int64_t size_m, int64_t size_n,
94
int64_t size_k) {
95
TORCH_CHECK_NOT_IMPLEMENTED(
@@ -1029,13 +1029,14 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1029
torch::Tensor& b_meta,
1030
torch::Tensor& b_scales,
1031
torch::Tensor& workspace,
1032
- vllm::ScalarTypeTorchPtr const& b_q_type,
1032
+ vllm::ScalarTypeId const b_q_type_id,
1033
int64_t size_m, int64_t size_n,
1034
int64_t size_k) {
1035
+ vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
1036
// Verify num_bits
1036
- TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
1037
- "num_bits must be uint4b8 or uint8b128. Got = ", b_q_type->str());
1038
- int pack_factor = 32 / b_q_type->size_bits();
1037
+ TORCH_CHECK(b_q_type == vllm::kU4B8 || b_q_type == vllm::kU8B128,
1038
+ "num_bits must be uint4b8 or uint8b128. Got = ", b_q_type.str());
1039
+ int pack_factor = 32 / b_q_type.size_bits();
1040
1041
// Verify M
1042
TORCH_CHECK(size_m == a.size(0),
@@ -1130,8 +1131,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1131
marlin_24::marlin_cuda_2_4(
1132
a.data_ptr(), b_q_weight.data_ptr(), b_meta.data_ptr(), c.data_ptr(),
1133
b_scales.data_ptr(), size_n, size_m, size_k, workspace.data_ptr(),
1133
- b_q_type->size_bits(), groupsize, dev,
1134
- at::cuda::getCurrentCUDAStream(dev), thread_k, thread_m, sms, max_par);
1134
+ b_q_type.size_bits(), groupsize, dev, at::cuda::getCurrentCUDAStream(dev),
1135
+ thread_k, thread_m, sms, max_par);
1136
1137
return c;
1138
}
csrc/torch_bindings.cpp
+20
-25
@@ -140,13 +140,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
140
// Quantized GEMM for AWQ.
141
ops.def(
142
"awq_gemm(Tensor _in_feats, Tensor _kernel, Tensor _scaling_factors, "
143
- "Tensor _zeros, int split_k_iters) -> Tensor");
143
+ "Tensor _zeros, SymInt split_k_iters) -> Tensor");
144
ops.impl("awq_gemm", torch::kCUDA, &awq_gemm);
145
146
// Dequantization for AWQ.
147
ops.def(
148
"awq_dequantize(Tensor _kernel, Tensor _scaling_factors, "
149
- "Tensor _zeros, int split_k_iters, int thx, int thy) -> Tensor");
149
+ "Tensor _zeros, SymInt split_k_iters, int thx, int thy) -> Tensor");
150
ops.impl("awq_dequantize", torch::kCUDA, &awq_dequantize);
151
152
// Note about marlin kernel 'workspace' arguments:
@@ -166,32 +166,26 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
166
// Marlin (Dense) Optimized Quantized GEMM for GPTQ.
167
ops.def(
168
"marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
169
- "Tensor! workspace, int size_m, int size_n, int size_k) -> Tensor");
169
+ "Tensor! workspace, SymInt size_m, SymInt size_n, SymInt size_k) -> "
170
+ "Tensor");
171
// conditionally compiled so impl in source file
172
173
// Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
174
ops.def(
175
"gptq_marlin_24_gemm(Tensor a, Tensor b_q_weight, Tensor b_meta, "
176
"Tensor b_scales, Tensor workspace, "
176
- "__torch__.torch.classes._core_C.ScalarType b_q_type, "
177
- "int size_m, int size_n, int size_k) -> Tensor");
177
+ "int b_q_type, "
178
+ "SymInt size_m, SymInt size_n, SymInt size_k) -> Tensor");
179
// conditionally compiled so impl in source file
180
181
// Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
182
+ ops.def("machete_supported_schedules(int btype) -> str[]");
183
ops.def(
182
- "machete_supported_schedules("
183
- " __torch__.torch.classes._core_C.ScalarType btype"
184
- ") -> str[]");
185
- ops.def(
186
- "machete_gemm(Tensor A, Tensor B,"
187
- " __torch__.torch.classes._core_C.ScalarType btype,"
188
- " Tensor? scales, Tensor? zeros, int? group_size,"
184
+ "machete_gemm(Tensor A, Tensor B, int btype, "
185
+ " Tensor? scales, Tensor? zeros, int? group_size, "
186
" Tensor? C, float? alpha, float? beta, str? schedule)"
187
"-> Tensor");
191
- ops.def(
192
- "machete_prepack_B(Tensor B,"
193
- " __torch__.torch.classes._core_C.ScalarType btype)"
194
- "-> Tensor");
188
+ ops.def("machete_prepack_B(Tensor B, int btype) -> Tensor");
189
// conditionally compiled so impl registration is in source file
190
191
ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
@@ -201,8 +195,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
195
ops.def(
196
"gptq_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
197
"Tensor b_zeros, Tensor g_idx, Tensor perm, Tensor workspace, "
204
- "__torch__.torch.classes._core_C.ScalarType b_q_type, "
205
- "int size_m, int size_n, int size_k, bool is_k_full, "
198
+ "int b_q_type, "
199
+ "SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, "
200
"bool has_zp, bool use_fp32_reduce) -> Tensor");
201
// conditionally compiled so impl registration is in source file
202
@@ -219,32 +213,33 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
213
// conditionally compiled so impl registrations are in source file
214
215
// Dequantization for GGML.
222
- ops.def("ggml_dequantize(Tensor W, int type, int m, int n) -> Tensor");
216
+ ops.def("ggml_dequantize(Tensor W, int type, SymInt m, SymInt n) -> Tensor");
217
ops.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize);
218
219
// mmvq kernel for GGML.
220
ops.def(
227
- "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, int row) "
221
+ "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, SymInt row) "
222
"-> Tensor");
223
ops.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8);
224
225
// mmq kernel for GGML.
232
- ops.def("ggml_mul_mat_a8(Tensor W, Tensor X, int type, int row) -> Tensor");
226
+ ops.def(
227
+ "ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor");
228
ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
229
230
// fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
231
ops.def(
232
"fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
238
- "Tensor! workspace, int num_bits, int size_m, int size_n, "
239
- "int size_k) -> Tensor");
233
+ "Tensor! workspace, int num_bits, SymInt size_m, SymInt size_n, "
234
+ "SymInt size_k) -> Tensor");
235
// conditionally compiled so impl registration is in source file
236
237
// marlin_qqq_gemm for QQQ.
238
ops.def(
239
"marlin_qqq_gemm(Tensor a, Tensor b_q_weight, "
240
"Tensor s_tok, Tensor s_ch, Tensor s_group, "
246
- "Tensor! workspace, int size_m, int size_n, "
247
- "int size_k) -> Tensor");
241
+ "Tensor! workspace, SymInt size_m, SymInt size_n, "
242
+ "SymInt size_k) -> Tensor");
243
// conditionally compiled so impl registration is in source file
244
245
// CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
python_only_dev.py
-1
@@ -39,7 +39,6 @@ assert cwd != package_path, "should not import from the current directory"
39
40
files_to_copy = [
41
"vllm/_C.abi3.so",
42
- "vllm/_core_C.abi3.so",
42
"vllm/_moe_C.abi3.so",
43
"vllm/vllm_flash_attn/vllm_flash_attn_c.abi3.so",
44
"vllm/vllm_flash_attn/flash_attn_interface.py",
setup.py
-7
@@ -290,10 +290,6 @@ def _build_custom_ops() -> bool:
290
return _is_cuda() or _is_hip() or _is_cpu()
291
292
293
-def _build_core_ext() -> bool:
294
- return not (_is_neuron() or _is_tpu() or _is_openvino() or _is_xpu())
295
-
296
-
293
def get_hipcc_rocm_version():
294
# Run the hipcc --version command
295
result = subprocess.run(['hipcc', '--version'],
@@ -456,9 +452,6 @@ def get_requirements() -> List[str]:
452
453
ext_modules = []
454
459
-if _build_core_ext():
460
- ext_modules.append(CMakeExtension(name="vllm._core_C"))
461
-
455
if _is_cuda() or _is_hip():
456
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
457
tests/compile/utils.py
+4
-4
@@ -69,11 +69,11 @@ def check_full_graph_support(model,
69
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
70
os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
71
72
- # Inductor doesn't support fp8/gptq_marlin_24 yet.
72
+ # Inductor doesn't support fp8 and the base meta llama uses too
73
+ # much memory.
74
quantization = model_kwargs.get("quantization")
74
- if (quantization == "fp8" or quantization == "gptq_marlin"
75
- or quantization == "gptq_marlin_24"
76
- ) and optimization_level >= CompilationLevel.INDUCTOR:
75
+ if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
76
+ and optimization_level >= CompilationLevel.INDUCTOR):
77
return
78
79
prompts = [
tests/kernels/test_machete_gemm.py
+5
-4
@@ -80,7 +80,7 @@ def machete_quantize_and_pack(w: torch.Tensor,
80
w_q = w_q.t().contiguous().t() # convert to col major
81
w_q_machete = ops.machete_prepack_B(w_q, wtype)
82
83
- opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype))
83
+ opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype.id))
84
85
return w_ref, w_q_machete, w_s, w_zp
86
@@ -153,9 +153,10 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
153
schedule=schedule,
154
)
155
156
- opcheck(torch.ops._C.machete_gemm,
157
- (a, w_q_machete, wtype, w_s, maybe_convert_zeropoints(
158
- w_zp, w_s), group_size, None, None, None, schedule))
156
+ opcheck(
157
+ torch.ops._C.machete_gemm,
158
+ (a, w_q_machete, wtype.id, w_s, maybe_convert_zeropoints(
159
+ w_zp, w_s), group_size, None, None, None, schedule))
160
161
# Relax atol as our reduction dim becomes larger (more rounding error)
162
# Relax atol when we have zeropoints since the way machete applies
tests/kernels/test_marlin_gemm.py
+13
-3
@@ -225,7 +225,7 @@ def test_gptq_marlin_gemm(
225
opcheck(
226
torch.ops._C.gptq_marlin_gemm,
227
(a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
228
- workspace.scratch, quant_type, a_input.shape[0], b_weight.shape[1],
228
+ workspace.scratch, quant_type.id, a_input.shape[0], b_weight.shape[1],
229
a_input.shape[1], is_k_full, False, use_fp32_reduce),
230
test_utils=DEFAULT_OPCHECK_TEST_UTILS)
231
@@ -254,6 +254,16 @@ def test_gptq_marlin_gemm(
254
assert max_diff < 0.04
255
256
257
+# TODO: find better way to test this?
258
+@torch.compile(fullgraph=True)
259
+def marlin_24_gemm_tester(a_input, marlin_24_q_w_comp, marlin_24_meta,
260
+ marlin_24_s, scratch, quant_type, size_m, size_n,
261
+ size_k):
262
+ return ops.gptq_marlin_24_gemm(a_input, marlin_24_q_w_comp, marlin_24_meta,
263
+ marlin_24_s, scratch, quant_type, size_m,
264
+ size_n, size_k)
265
+
266
+
267
@pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
268
reason="Marlin is not supported on this GPU type.")
269
@pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
@@ -282,11 +292,11 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
292
293
opcheck(torch.ops._C.gptq_marlin_24_gemm,
294
(a_input, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s,
285
- workspace_24.scratch, quant_type, a_input.shape[0],
295
+ workspace_24.scratch, quant_type.id, a_input.shape[0],
296
b_weight.shape[1], a_input.shape[1]),
297
test_utils=DEFAULT_OPCHECK_TEST_UTILS)
298
289
- output = ops.gptq_marlin_24_gemm(
299
+ output = marlin_24_gemm_tester(
300
a_input,
301
marlin_24_q_w_comp,
302
marlin_24_meta,
tests/kernels/test_moe.py
+2
-2
@@ -240,8 +240,8 @@ def test_fused_marlin_moe(
240
requires_grad=False)
241
opcheck(torch.ops._moe_C.marlin_gemm_moe,
242
(a, qweight1, sorted_token_ids, topk_weights, topk_ids,
243
- scales1, zp, g_idx1, sort_indices1, workspace, quant_type, m,
244
- 2 * n, k, True, e, topk, block_size_m, True, False))
243
+ scales1, zp, g_idx1, sort_indices1, workspace, quant_type.id,
244
+ m, 2 * n, k, True, e, topk, block_size_m, True, False))
245
246
247
@pytest.mark.skip("This test is here for the sake of debugging, "
tests/test_scalartype.py
+2
-2
@@ -32,5 +32,5 @@ def test_scalar_type_min_max(type_tuple):
32
max = torch.iinfo(torch_type).max
33
34
print(t, min, max, t.min(), t.max())
35
- assert min == t.min()
36
- assert max == t.max()
35
+ assert min == t.min(), f"min: {min} != {t.min()}"
36
+ assert max == t.max(), f"max: {max} != {t.max()}"
tools/report_build_time_ninja.py
-1
@@ -16,7 +16,6 @@ Typical output looks like this:
16
2.6 weighted s to build ...torch_bindings.cpp.o (31.5 s elapsed time)
17
3.2 weighted s to build ...torch_bindings.cpp.o (38.5 s elapsed time)
18
Longest build steps for .so (linking):
19
- 0.1 weighted s to build _core_C.abi3.so (0.7 s elapsed time)
19
0.1 weighted s to build _moe_C.abi3.so (1.0 s elapsed time)
20
0.5 weighted s to build ...flash_attn_c.abi3.so (1.1 s elapsed time)
21
6.2 weighted s to build _C.abi3.so (6.2 s elapsed time)
vllm/_core_ext.py
deleted
-278
@@ -1,278 +0,0 @@
1
-import importlib.util
2
-from enum import Enum
3
-from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
4
-
5
-import torch
6
-
7
-from vllm.logger import init_logger
8
-
9
-logger = init_logger(__name__)
10
-core_C_available = importlib.util.find_spec('._core_C', 'vllm') is not None
11
-
12
-
13
-# Mirrors enum in `core/scalar_type.hpp`
14
-class NanRepr(Enum):
15
- NONE = 0 # nans are not supported
16
- IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s
17
- EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s
18
-
19
-
20
-if TYPE_CHECKING or not core_C_available:
21
- # On platforms were we cannot use/build the C++ core extension (i.e. namely
22
- # neuron and tpu), we define the mock ScalarType class here that partially
23
- # mimics the C++ ScalarType class.
24
- #
25
- # We also use this provide type signatures to the Python LSP for the methods
26
- # in the C++ ScalarType class. So these type signatures should be kept
27
- # in sync with csrc/core/scalar_type.hpp
28
-
29
- from dataclasses import dataclass
30
-
31
- @dataclass(frozen=True)
32
- class ScalarType:
33
- """
34
- ScalarType can represent a wide range of floating point and integer
35
- types, in particular it can be used to represent sub-byte data types
36
- (something that torch.dtype currently does not support). It is also
37
- capable of representing types with a bias, i.e.:
38
- `stored_value = value + bias`,
39
- this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40
- of 8). The implementation for this class can be found in
41
- csrc/core/scalar_type.hpp, these type signatures should be kept in sync
42
- with that file.
43
- """
44
-
45
- exponent: int
46
- """
47
- Number of bits in the exponent if this is a floating point type
48
- (zero if this an integer type)
49
- """
50
-
51
- mantissa: int
52
- """
53
- Number of bits in the mantissa if this is a floating point type,
54
- or the number bits representing an integer excluding the sign bit if
55
- this an integer type.
56
- """
57
-
58
- bias: int
59
- """
60
- bias used to encode the values in this scalar type
61
- (value = stored_value - bias, default 0) for example if we store the
62
- type as an unsigned integer with a bias of 128 then the value 0 will be
63
- stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
64
- """
65
-
66
- signed: bool
67
- "If the type is signed (i.e. has a sign bit)"
68
-
69
- _finite_values_only: bool = False
70
- """
71
- Private: if NANs are supported, used `has_infs()` instead.
72
- """
73
-
74
- nan_repr: int = NanRepr.IEEE_754.value
75
- """
76
- How NaNs are represent in this scalar type, returns NanRepr value.
77
- (not applicable for integer types)
78
- """
79
-
80
- @property
81
- def size_bits(self):
82
- return self.exponent + self.mantissa + int(self.signed)
83
-
84
- def min(self) -> Union[int, float]:
85
- """
86
- Min representable value for this scalar type.
87
- (accounting for bias if there is one)
88
- """
89
- raise NotImplementedError
90
-
91
- def max(self) -> Union[int, float]:
92
- """
93
- Max representable value for this scalar type.
94
- (accounting for bias if there is one)
95
- """
96
- raise NotImplementedError
97
-
98
- def is_signed(self) -> bool:
99
- """
100
- If the type is signed (i.e. has a sign bit), same as `signed`
101
- added for consistency with:
102
- https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
103
- """
104
- ...
105
-
106
- def is_floating_point(self) -> bool:
107
- "If the type is a floating point type"
108
- return self.exponent != 0
109
-
110
- def is_integer(self) -> bool:
111
- "If the type is an integer type"
112
- return self.exponent == 0
113
-
114
- def has_bias(self) -> bool:
115
- "If the type has a non-zero bias"
116
- return self.bias != 0
117
-
118
- def has_infs(self) -> bool:
119
- "If the type is floating point and supports infinity"
120
- return not self._finite_values_only
121
-
122
- def has_nans(self) -> bool:
123
- return self.nan_repr != NanRepr.NONE.value
124
-
125
- def is_ieee_754(self) -> bool:
126
- """
127
- If the type is a floating point type that follows IEEE 754
128
- conventions
129
- """
130
- return self.nan_repr == NanRepr.IEEE_754.value and \
131
- not self._finite_values_only
132
-
133
- def __str__(self) -> str:
134
- raise NotImplementedError
135
-
136
- def __repr__(self) -> str:
137
- raise NotImplementedError
138
-
139
- # __len__ needs to be defined (and has to throw TypeError) for pytorch's
140
- # opcheck to work.
141
- def __len__(self) -> int:
142
- raise TypeError
143
-
144
- #
145
- # Convenience Constructors
146
- #
147
-
148
- @classmethod
149
- def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
150
- "Create a signed integer scalar type (size_bits includes sign-bit)."
151
- return cls(size_bits - 1, size_bits, bias if bias else 0, True)
152
-
153
- @classmethod
154
- def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
155
- """Create a unsigned integer scalar type."""
156
- return cls(size_bits, size_bits, bias if bias else 0, False)
157
-
158
- @classmethod
159
- def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
160
- """
161
- Create a standard floating point type
162
- (i.e. follows IEEE 754 conventions).
163
- """
164
- return cls(exponent, mantissa, 0, True)
165
-
166
- @classmethod
167
- def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
168
- nan_repr: int) -> 'ScalarType':
169
- """
170
- Create a non-standard floating point type
171
- (i.e. does not follow IEEE 754 conventions).
172
- """
173
- return cls(exponent, mantissa, 0, True, finite_values_only,
174
- nan_repr)
175
-
176
-elif core_C_available:
177
- try:
178
- import vllm._core_C # noqa: F401
179
- except ImportError as e:
180
- logger.warning("Failed to import from vllm._core_C with %r", e)
181
-
182
- ScalarType = torch.classes._core_C.ScalarType
183
-
184
- if (hasattr(torch, "_library")
185
- and hasattr(torch._library, "register_fake_class")):
186
- # Needed for dynamo support of ScalarType.
187
- @torch._library.register_fake_class("_core_C::ScalarType")
188
- class FakeScalarType:
189
-
190
- def __init__(self, scalar_type):
191
- self.ScalarType = scalar_type
192
-
193
- def bias_getter(self) -> int:
194
- return self.ScalarType.bias
195
-
196
- def exponent_getter(self) -> int:
197
- return self.ScalarType.exponent
198
-
199
- def mantissa_getter(self) -> int:
200
- return self.ScalarType.mantissa
201
-
202
- def signed_getter(self) -> bool:
203
- return self.ScalarType.signed
204
-
205
- def size_bits_getter(self) -> int:
206
- return self.ScalarType.size_bits
207
-
208
- @property
209
- def size_bits(self) -> int:
210
- return self.ScalarType.size_bits
211
-
212
- def min(self) -> Union[int, float]:
213
- return self.ScalarType.min()
214
-
215
- def max(self) -> Union[int, float]:
216
- return self.ScalarType.max()
217
-
218
- def is_signed(self) -> bool:
219
- return self.ScalarType.is_signed()
220
-
221
- def is_floating_point(self) -> bool:
222
- return self.ScalarType.is_floating_point()
223
-
224
- def is_integer(self) -> bool:
225
- return self.ScalarType.is_integer()
226
-
227
- def has_bias(self) -> bool:
228
- return self.ScalarType.has_bias()
229
-
230
- def has_infs(self) -> bool:
231
- return self.ScalarType.has_infs()
232
-
233
- def has_nans(self) -> bool:
234
- return self.ScalarType.has_nans()
235
-
236
- def is_ieee_754(self) -> bool:
237
- return self.ScalarType.is_ieee_754()
238
-
239
- def __str__(self) -> str:
240
- return self.ScalarType.__str__()
241
-
242
- def __repr__(self) -> str:
243
- return self.ScalarType.__repr__()
244
-
245
- def __len__(self) -> int:
246
- return self.ScalarType.__len__()
247
-
248
- def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
249
- return torch.classes._core_C.ScalarType.__obj_flatten__(
250
- self.ScalarType)
251
-
252
- @classmethod
253
- def __obj_unflatten__(
254
- cls, flat_type: Tuple[Tuple[str, Any],
255
- ...]) -> 'ScalarType':
256
- return cls(
257
- torch.classes._core_C.ScalarType.__obj_unflatten__(
258
- flat_type))
259
-
260
- @classmethod
261
- def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262
- return ScalarType.int_(size_bits, bias)
263
-
264
- @classmethod
265
- def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
266
- return ScalarType.uint(size_bits, bias)
267
-
268
- @classmethod
269
- def float_IEEE754(cls, exponent: int,
270
- mantissa: int) -> 'ScalarType':
271
- return ScalarType.float_IEEE754(exponent, mantissa)
272
-
273
- @classmethod
274
- def float_(cls, exponent: int, mantissa: int,
275
- finite_values_only: bool,
276
- nan_repr: int) -> 'ScalarType':
277
- return ScalarType.float_(exponent, mantissa,
278
- finite_values_only, nan_repr)
vllm/_custom_ops.py
+31
-62
@@ -6,9 +6,9 @@ import torch
6
import torch.library
7
8
import vllm.envs as envs
9
-from vllm._core_ext import ScalarType
9
from vllm.logger import init_logger
10
from vllm.platforms import current_platform
11
+from vllm.scalar_type import ScalarType
12
13
logger = init_logger(__name__)
14
@@ -306,7 +306,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
306
workspace: torch.Tensor, b_q_type: ScalarType,
307
size_m: int, size_n: int, size_k: int) -> torch.Tensor:
308
return torch.ops._C.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
309
- workspace, b_q_type, size_m,
309
+ workspace, b_q_type.id, size_m,
310
size_n, size_k)
311
312
@@ -316,8 +316,9 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
316
def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
317
b_meta: torch.Tensor, b_scales: torch.Tensor,
318
workspace: torch.Tensor,
319
- b_q_type: ScalarType, size_m: int,
320
- size_n: int, size_k: int) -> torch.Tensor:
319
+ b_q_type: ScalarType, size_m: torch.SymInt,
320
+ size_n: torch.SymInt,
321
+ size_k: torch.SymInt) -> torch.Tensor:
322
return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
323
324
@register_fake("_C::gptq_marlin_gemm")
@@ -329,17 +330,18 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
330
perm: torch.Tensor,
331
workspace: torch.Tensor,
332
b_q_type: ScalarType,
332
- size_m: int,
333
- size_n: int,
334
- size_k: int,
333
+ size_m: torch.SymInt,
334
+ size_n: torch.SymInt,
335
+ size_k: torch.SymInt,
336
is_k_full: bool,
337
has_zp: bool = False,
338
use_fp32_reduce: bool = False) -> torch.Tensor:
339
return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
340
341
@register_fake("_C::ggml_dequantize")
341
- def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int,
342
- n: int) -> torch.Tensor:
342
+ def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int,
343
+ m: torch.SymInt,
344
+ n: torch.SymInt) -> torch.Tensor:
345
return torch.empty((m, n), dtype=torch.float16, device=W.device)
346
347
@register_fake("_C::ggml_mul_mat_vec_a8")
@@ -347,7 +349,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
349
W: torch.Tensor,
350
X: torch.Tensor,
351
quant_type: int,
350
- row: int,
352
+ row: torch.SymInt,
353
) -> torch.Tensor:
354
return torch.empty((1, row), dtype=torch.float16, device=W.device)
355
@@ -356,7 +358,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
358
W: torch.Tensor,
359
X: torch.Tensor,
360
quant_type: int,
359
- row: int,
361
+ row: torch.SymInt,
362
) -> torch.Tensor:
363
batch = X.size(0)
364
return torch.empty((batch, row), dtype=torch.float16, device=W.device)
@@ -365,8 +367,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
367
def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
368
s_tok: torch.Tensor, s_ch: torch.Tensor,
369
s_group: torch.Tensor, workspace: torch.Tensor,
368
- size_m: int, size_n: int,
369
- size_k: int) -> torch.Tensor:
370
+ size_m: torch.SymInt, size_n: torch.SymInt,
371
+ size_k: torch.SymInt) -> torch.Tensor:
372
return torch.empty((size_m, size_n),
373
dtype=torch.float16,
374
device=a.device)
@@ -374,16 +376,16 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
376
@register_fake("_C::marlin_gemm")
377
def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
378
b_scales: torch.Tensor, workspace: torch.Tensor,
377
- size_m: int, size_n: int,
378
- size_k: int) -> torch.Tensor:
379
+ size_m: torch.SymInt, size_n: torch.SymInt,
380
+ size_k: torch.SymInt) -> torch.Tensor:
381
return torch.empty((size_m, size_n),
382
dtype=torch.float16,
383
device=a.device)
384
385
@register_fake("_C::awq_dequantize")
386
def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor,
385
- zeros: torch.Tensor, split_k_iters: int, thx: int,
386
- thy: int) -> torch.Tensor:
387
+ zeros: torch.Tensor, split_k_iters: torch.SymInt,
388
+ thx: int, thy: int) -> torch.Tensor:
389
in_c = qweight.size(0)
390
qout_c = qweight.size(1)
391
out_c = qout_c * 8
@@ -394,7 +396,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
396
@register_fake("_C::awq_gemm")
397
def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor,
398
qzeros: torch.Tensor, scales: torch.Tensor,
397
- split_k_iters: int) -> torch.Tensor:
399
+ split_k_iters: torch.SymInt) -> torch.Tensor:
400
num_in_feats = input.size(0)
401
return torch.empty((split_k_iters, num_in_feats, qweight.size(1) * 8),
402
dtype=input.dtype,
@@ -429,8 +431,9 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
431
@register_fake("_C::fp8_marlin_gemm")
432
def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
433
b_scales: torch.Tensor, workspace: torch.Tensor,
432
- num_bits: int, size_m: int, size_n: int,
433
- size_k: int) -> torch.Tensor:
434
+ num_bits: int, size_m: torch.SymInt,
435
+ size_n: torch.SymInt,
436
+ size_k: torch.SymInt) -> torch.Tensor:
437
return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
438
439
@register_fake("_C::machete_gemm")
@@ -457,40 +460,6 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
460
return torch.empty_like(b_q_weight,
461
memory_format=torch.contiguous_format)
462
460
- @register_fake("_C::causal_conv1d_fwd")
461
- def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
462
- bias_: Optional[torch.Tensor],
463
- conv_states: Optional[torch.Tensor],
464
- cu_seq_len: Optional[torch.Tensor],
465
- cache_indices: Optional[torch.Tensor],
466
- has_initial_state: Optional[torch.Tensor],
467
- silu_activation: bool, pad_slot_id: int):
468
- return None
469
-
470
- @register_fake("_C::causal_conv1d_update")
471
- def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
472
- weight: torch.Tensor,
473
- bias_: Optional[torch.Tensor],
474
- silu_activation: bool,
475
- cache_seqlens: Optional[torch.Tensor],
476
- conv_state_indices: Optional[torch.Tensor],
477
- pad_slot_id: int) -> None:
478
- return None
479
-
480
- @register_fake("_C::selective_scan_fwd")
481
- def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
482
- A: torch.Tensor, B: torch.Tensor,
483
- C: torch.Tensor, D_: Optional[torch.Tensor],
484
- z_: Optional[torch.Tensor],
485
- delta_bias_: Optional[torch.Tensor],
486
- delta_softplus: bool,
487
- cu_seq_len: Optional[torch.Tensor],
488
- cache_indices: Optional[torch.Tensor],
489
- has_initial_state: Optional[torch.Tensor],
490
- ssm_states: Optional[torch.Tensor],
491
- pad_slot_id: int) -> None:
492
- return None
493
-
463
464
# cutlass
465
def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
@@ -611,7 +580,7 @@ def gptq_marlin_gemm(a: torch.Tensor,
580
has_zp: bool = False,
581
use_fp32_reduce: bool = False) -> torch.Tensor:
582
return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
614
- g_idx, perm, workspace, b_q_type,
583
+ g_idx, perm, workspace, b_q_type.id,
584
size_m, size_n, size_k, is_k_full,
585
has_zp, use_fp32_reduce)
586
@@ -627,7 +596,7 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
596
597
# machete
598
def machete_supported_schedules(b_type: ScalarType) -> List[str]:
630
- return torch.ops._C.machete_supported_schedules(b_type)
599
+ return torch.ops._C.machete_supported_schedules(b_type.id)
600
601
602
def machete_gemm(
@@ -642,13 +611,13 @@ def machete_gemm(
611
beta: Optional[float] = None,
612
schedule: Optional[str] = None,
613
) -> torch.Tensor:
645
- return torch.ops._C.machete_gemm(a, b_q, b_type, b_scales, b_zeros,
614
+ return torch.ops._C.machete_gemm(a, b_q, b_type.id, b_scales, b_zeros,
615
b_group_size, c, alpha, beta, schedule)
616
617
618
def machete_prepack_B(b_q_weight: torch.Tensor,
619
b_type: ScalarType) -> torch.Tensor:
651
- return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
620
+ return torch.ops._C.machete_prepack_B(b_q_weight, b_type.id)
621
622
623
if hasattr(torch.ops._C, "permute_cols"):
@@ -862,10 +831,10 @@ if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
831
topk_ids: torch.Tensor, b_scales: torch.Tensor,
832
b_zero_points: torch.Tensor, g_idx: torch.Tensor,
833
perm: torch.Tensor, workspace: torch.Tensor,
865
- b_q_type: ScalarType, size_m: int, size_n: int,
866
- size_k: int, is_k_full: bool, num_experts: int,
867
- topk: int, moe_block_size: int,
868
- replicate_input: bool,
834
+ b_q_type: ScalarType, size_m: torch.SymInt,
835
+ size_n: torch.SymInt, size_k: torch.SymInt,
836
+ is_k_full: bool, num_experts: int, topk: int,
837
+ moe_block_size: int, replicate_input: bool,
838
apply_weights: bool) -> torch.Tensor:
839
return torch.empty((size_m, topk, size_n),
840
dtype=a.dtype,
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+3
-3
@@ -116,7 +116,7 @@ def single_marlin_moe(
116
117
intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
118
hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
119
- w_zeros, g_idx, sort_indices, workspace, scalar_type, M, N, K,
119
+ w_zeros, g_idx, sort_indices, workspace, scalar_type.id, M, N, K,
120
is_k_full, E, topk, block_size_m, True, False)
121
122
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
@@ -272,7 +272,7 @@ def fused_marlin_moe(
272
g_idx1,
273
sort_indices1,
274
workspace,
275
- scalar_type1,
275
+ scalar_type1.id,
276
M,
277
2 * N,
278
K,
@@ -297,7 +297,7 @@ def fused_marlin_moe(
297
g_idx2,
298
sort_indices2,
299
workspace,
300
- scalar_type2,
300
+ scalar_type2.id,
301
M,
302
K,
303
N,
vllm/scalar_type.py
+297
-4
@@ -1,4 +1,298 @@
1
-from ._core_ext import NanRepr, ScalarType
1
+import functools
2
+import struct
3
+from dataclasses import dataclass
4
+from enum import Enum
5
+from typing import Optional, Union
6
+
7
+
8
+# Mirrors enum in `core/scalar_type.hpp`
9
+class NanRepr(Enum):
10
+ NONE = 0 # nans are not supported
11
+ IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s
12
+ EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s
13
+
14
+
15
+# This ScalarType class is a parallel implementation of the C++ ScalarType
16
+# class found in csrc/core/scalar_type.hpp. These two classes should be kept
17
+# in sync until the inductor fully supports custom C++ classes.
18
+@dataclass(frozen=True)
19
+class ScalarType:
20
+ """
21
+ ScalarType can represent a wide range of floating point and integer
22
+ types, in particular it can be used to represent sub-byte data types
23
+ (something that torch.dtype currently does not support). It is also
24
+ capable of representing types with a bias, i.e.:
25
+ `stored_value = value + bias`,
26
+ this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
27
+ of 8). The implementation for this class can be found in
28
+ csrc/core/scalar_type.hpp, these type signatures should be kept in sync
29
+ with that file.
30
+ """
31
+
32
+ exponent: int
33
+ """
34
+ Number of bits in the exponent if this is a floating point type
35
+ (zero if this an integer type)
36
+ """
37
+
38
+ mantissa: int
39
+ """
40
+ Number of bits in the mantissa if this is a floating point type,
41
+ or the number bits representing an integer excluding the sign bit if
42
+ this an integer type.
43
+ """
44
+
45
+ signed: bool
46
+ "If the type is signed (i.e. has a sign bit)"
47
+
48
+ bias: int
49
+ """
50
+ bias used to encode the values in this scalar type
51
+ (value = stored_value - bias, default 0) for example if we store the
52
+ type as an unsigned integer with a bias of 128 then the value 0 will be
53
+ stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
54
+ """
55
+
56
+ _finite_values_only: bool = False
57
+ """
58
+ Private: if infs are supported, used `has_infs()` instead.
59
+ """
60
+
61
+ nan_repr: NanRepr = NanRepr.IEEE_754
62
+ """
63
+ How NaNs are represent in this scalar type, returns NanRepr value.
64
+ (not applicable for integer types)
65
+ """
66
+
67
+ def _floating_point_max_int(self) -> int:
68
+ assert (
69
+ self.mantissa <= 52 and self.exponent <= 11
70
+ ), f"Cannot represent max/min as a double for type {self.__str__()}"
71
+
72
+ max_mantissa = (1 << self.mantissa) - 1
73
+ if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN:
74
+ max_mantissa = max_mantissa - 1
75
+
76
+ max_exponent = (1 << self.exponent) - 2
77
+ if (self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN
78
+ or self.nan_repr == NanRepr.NONE):
79
+ assert (
80
+ self.exponent < 11
81
+ ), f"Cannot represent max/min as a double for type {self.__str__()}"
82
+ max_exponent = max_exponent + 1
83
+
84
+ # adjust the exponent to match that of a double
85
+ # for now we assume the exponent bias is the standard 2^(e-1) -1, (where
86
+ # e is the exponent bits), there is some precedent for non-standard
87
+ # biases, example `float8_e4m3b11fnuz` here:
88
+ # https://github.com/jax-ml/ml_dtypes but to avoid premature over
89
+ # complication we are just assuming the standard exponent bias until
90
+ # there is a need to support non-standard biases
91
+ exponent_bias = (1 << (self.exponent - 1)) - 1
92
+ exponent_bias_double = (1 << 10) - 1 # double e = 11
93
+
94
+ max_exponent_double = (max_exponent - exponent_bias +
95
+ exponent_bias_double)
96
+
97
+ # shift the mantissa and exponent into the proper positions for an
98
+ # IEEE double and bitwise-or them together.
99
+ return (max_mantissa <<
100
+ (52 - self.mantissa)) | (max_exponent_double << 52)
101
+
102
+ def _floating_point_max(self) -> float:
103
+ double_raw = self._floating_point_max_int()
104
+ return struct.unpack('!d', struct.pack('!Q', double_raw))[0]
105
+
106
+ def _raw_max(self) -> Union[int, float]:
107
+ if self.is_floating_point():
108
+ return self._floating_point_max()
109
+ else:
110
+ assert (self.size_bits < 64 or self.size_bits == 64
111
+ and self.is_signed()), "Cannot represent max as an int"
112
+ return (1 << self.mantissa) - 1
113
+
114
+ def _raw_min(self) -> Union[int, float]:
115
+ if self.is_floating_point():
116
+ assert self.is_signed(
117
+ ), "We currently assume all floating point types are signed"
118
+ sign_bit_double = 1 << 63
119
+
120
+ max_raw = self._floating_point_max_int()
121
+ min_raw = max_raw | sign_bit_double
122
+ return struct.unpack('!d', struct.pack('!Q', min_raw))[0]
123
+ else:
124
+ assert (not self.is_signed() or
125
+ self.size_bits <= 64), "Cannot represent min as a int64_t"
126
+
127
+ if self.is_signed():
128
+ return -(1 << (self.size_bits - 1))
129
+ else:
130
+ return 0
131
+
132
+ @functools.cached_property
133
+ def id(self) -> int:
134
+ """
135
+ Convert the ScalarType to an int which can be passed to pytorch custom
136
+ ops. This layout of the int must be kept in sync with the C++
137
+ ScalarType's from_id method.
138
+ """
139
+ val = 0
140
+ offset = 0
141
+
142
+ def or_and_advance(member, bit_width):
143
+ nonlocal val
144
+ nonlocal offset
145
+ bit_mask = (1 << bit_width) - 1
146
+ val = val | (int(member) & bit_mask) << offset
147
+ offset = offset + bit_width
148
+
149
+ or_and_advance(self.exponent, 8)
150
+ or_and_advance(self.mantissa, 8)
151
+ or_and_advance(self.signed, 1)
152
+ or_and_advance(self.bias, 32)
153
+ or_and_advance(self._finite_values_only, 1)
154
+ or_and_advance(self.nan_repr.value, 8)
155
+
156
+ assert offset <= 64, \
157
+ f"ScalarType fields too big {offset} to fit into an int64"
158
+
159
+ return val
160
+
161
+ @property
162
+ def size_bits(self) -> int:
163
+ return self.exponent + self.mantissa + int(self.signed)
164
+
165
+ def min(self) -> Union[int, float]:
166
+ """
167
+ Min representable value for this scalar type.
168
+ (accounting for bias if there is one)
169
+ """
170
+ return self._raw_min() - self.bias
171
+
172
+ def max(self) -> Union[int, float]:
173
+ """
174
+ Max representable value for this scalar type.
175
+ (accounting for bias if there is one)
176
+ """
177
+ return self._raw_max() - self.bias
178
+
179
+ def is_signed(self) -> bool:
180
+ """
181
+ If the type is signed (i.e. has a sign bit), same as `signed`
182
+ added for consistency with:
183
+ https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
184
+ """
185
+ return self.signed
186
+
187
+ def is_floating_point(self) -> bool:
188
+ "If the type is a floating point type"
189
+ return self.exponent != 0
190
+
191
+ def is_integer(self) -> bool:
192
+ "If the type is an integer type"
193
+ return self.exponent == 0
194
+
195
+ def has_bias(self) -> bool:
196
+ "If the type has a non-zero bias"
197
+ return self.bias != 0
198
+
199
+ def has_infs(self) -> bool:
200
+ "If the type is floating point and supports infinity"
201
+ return not self._finite_values_only
202
+
203
+ def has_nans(self) -> bool:
204
+ return self.nan_repr != NanRepr.NONE.value
205
+
206
+ def is_ieee_754(self) -> bool:
207
+ """
208
+ If the type is a floating point type that follows IEEE 754
209
+ conventions
210
+ """
211
+ return self.nan_repr == NanRepr.IEEE_754.value and \
212
+ not self._finite_values_only
213
+
214
+ def __str__(self) -> str:
215
+ """
216
+ naming generally follows: https://github.com/jax-ml/ml_dtypes
217
+ for floating point types (leading f) the scheme is:
218
+ `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
219
+ flags:
220
+ - no-flags: means it follows IEEE 754 conventions
221
+ - f: means finite values only (no infinities)
222
+ - n: means nans are supported (non-standard encoding)
223
+ for integer types the scheme is:
224
+ `[u]int<size_bits>[b<bias>]`
225
+ - if bias is not present it means its zero
226
+ """
227
+ if self.is_floating_point():
228
+ ret = "float" + str(self.size_bits) + "_e" + str(
229
+ self.exponent) + "m" + str(self.mantissa)
230
+
231
+ if not self.is_ieee_754():
232
+ if self._finite_values_only:
233
+ ret = ret + "f"
234
+ if self.nan_repr != NanRepr.NONE:
235
+ ret = ret + "n"
236
+
237
+ return ret
238
+ else:
239
+ ret = ("int" if self.is_signed() else "uint") + str(self.size_bits)
240
+ if self.has_bias():
241
+ ret = ret + "b" + str(self.bias)
242
+ return ret
243
+
244
+ def __repr__(self) -> str:
245
+ return "ScalarType." + self.__str__()
246
+
247
+ # __len__ needs to be defined (and has to throw TypeError) for pytorch's
248
+ # opcheck to work.
249
+ def __len__(self) -> int:
250
+ raise TypeError
251
+
252
+ #
253
+ # Convenience Constructors
254
+ #
255
+
256
+ @classmethod
257
+ def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
258
+ "Create a signed integer scalar type (size_bits includes sign-bit)."
259
+ ret = cls(0, size_bits - 1, True, bias if bias else 0)
260
+ ret.id # noqa B018: make sure the id is cached
261
+ return ret
262
+
263
+ @classmethod
264
+ def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
265
+ """Create a unsigned integer scalar type."""
266
+ ret = cls(0, size_bits, False, bias if bias else 0)
267
+ ret.id # noqa B018: make sure the id is cached
268
+ return ret
269
+
270
+ @classmethod
271
+ def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
272
+ """
273
+ Create a standard floating point type
274
+ (i.e. follows IEEE 754 conventions).
275
+ """
276
+ assert (mantissa > 0 and exponent > 0)
277
+ ret = cls(exponent, mantissa, True, 0)
278
+ ret.id # noqa B018: make sure the id is cached
279
+ return ret
280
+
281
+ @classmethod
282
+ def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
283
+ nan_repr: NanRepr) -> 'ScalarType':
284
+ """
285
+ Create a non-standard floating point type
286
+ (i.e. does not follow IEEE 754 conventions).
287
+ """
288
+ assert (mantissa > 0 and exponent > 0)
289
+ assert (nan_repr != NanRepr.IEEE_754), (
290
+ "use `float_IEEE754` constructor for floating point types that "
291
+ "follow IEEE 754 conventions")
292
+ ret = cls(exponent, mantissa, True, 0, finite_values_only, nan_repr)
293
+ ret.id # noqa B018: make sure the id is cached
294
+ return ret
295
+
296
297
# naming generally follows: https://github.com/jax-ml/ml_dtypes
298
# for floating point types (leading f) the scheme is:
@@ -17,14 +311,13 @@ class scalar_types:
311
uint4 = ScalarType.uint(4, None)
312
int8 = ScalarType.int_(8, None)
313
uint8 = ScalarType.uint(8, None)
20
- float8_e4m3fn = ScalarType.float_(4, 3, True,
21
- NanRepr.EXTD_RANGE_MAX_MIN.value)
314
+ float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)
315
float8_e5m2 = ScalarType.float_IEEE754(5, 2)
316
float16_e8m7 = ScalarType.float_IEEE754(8, 7)
317
float16_e5m10 = ScalarType.float_IEEE754(5, 10)
318
319
# fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
27
- float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE.value)
320
+ float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE)
321
322
# "gptq" types
323
uint2b2 = ScalarType.uint(2, 2)