@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eca2c5f7c0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix support for dimension like integers and ScalarType (#9299)

bnellnm committed Oct 17, 2024 at 15:08 UTC eca2c5f7c00d6c0b08051972d1e80fe822e7d1b8
22 files changed +427 -677
.buildkite/test-pipeline.yaml
+6 -8
@@ -230,14 +230,12 @@ steps:
230 commands:
231 - pytest -v -s compile/test_basic_correctness.py
232
233 -# TODO: re-write in comparison tests, and fix symbolic shape
234 -# for quantization ops.
235 -# - label: "PyTorch Fullgraph Test" # 18min
236 -# source_file_dependencies:
237 -# - vllm/
238 -# - tests/compile
239 -# commands:
240 -# - pytest -v -s compile/test_full_graph.py
233 +- label: "PyTorch Fullgraph Test" # 18min
234 + source_file_dependencies:
235 + - vllm/
236 + - tests/compile
237 + commands:
238 + - pytest -v -s compile/test_full_graph.py
239
240 - label: Kernels Test %N # 1h each
241 mirror_hardwares: [amd]
CMakeLists.txt
-18
@@ -83,24 +83,6 @@ endif()
83 #
84 find_package(Torch REQUIRED)
85
86 -#
87 -message(STATUS "Enabling core extension.")
88 -
89 -# Define _core_C extension
90 -# built for (almost) every target platform, (excludes TPU and Neuron)
91 -
92 -set(VLLM_EXT_SRC
93 - "csrc/core/torch_bindings.cpp")
94 -
95 -define_gpu_extension_target(
96 - _core_C
97 - DESTINATION vllm
98 - LANGUAGE CXX
99 - SOURCES ${VLLM_EXT_SRC}
100 - COMPILE_FLAGS ${CXX_COMPILE_FLAGS}
101 - USE_SABI 3
102 - WITH_SOABI)
103 -
86 #
87 # Forward the non-CUDA device extensions to external CMake scripts.
88 #
csrc/core/scalar_type.hpp
+4 -205
@@ -1,6 +1,7 @@
1 #pragma once
2
3 -#include <torch/custom_class.h>
3 +// For TORCH_CHECK
4 +#include <torch/library.h>
5
6 namespace vllm {
7
@@ -9,12 +10,7 @@ namespace vllm {
10 // in particular it can be used to represent sub-byte data types (something
11 // that torch.dtype currently does not support).
12 //
12 -// ScalarTypeTorch is a subclass of ScalarType that is compatible with
13 -// TORCH_LIBRARY, making it accessible from Python as well meaning this class
14 -// can be used as a argument for custom operators, helping to simplify these
15 -// interfaces.
16 -//
17 -// The type definitions on the Python side can be found in: vllm/_core_ext.pyi
13 +// The type definitions on the Python side can be found in: vllm/scalar_type.py
14 // these type definitions should be kept up to date with any Python API changes
15 // here.
16 //
@@ -308,204 +304,7 @@ class ScalarType {
304 }
305 };
306
311 -// Create a TORCH_LIBRARY compatible version of ScalarType (i.e. inherit from
312 -// torch::CustomClassHolder), we use multiple inheritance here since we cannot
313 -// have ScalarType inherit from torch::CustomClassHolder and have a constexpr
314 -// constructor at the same time (torch::CustomClassHolder does not have a
315 -// constexpr destructor)
316 -// See also:
317 -// https://docs.google.com/document/d/18fBMPuOJ0fY5ZQ6YyrHUppw9FA332CpNtgB6SOIgyuA
318 -class ScalarTypeTorch : public torch::CustomClassHolder, public ScalarType {
319 - public:
320 - ScalarTypeTorch(int64_t exponent, int64_t mantissa, int64_t bias,
321 - bool _signed)
322 - : ScalarType(exponent, mantissa, bias, _signed){};
323 -
324 - ScalarTypeTorch(ScalarType type) : ScalarType(type){};
325 -
326 - using Base = ScalarType;
327 - using Self = ScalarTypeTorch;
328 - using SelfPtr = c10::intrusive_ptr<Self>;
329 -
330 - static void check_size_bits(int64_t size_bits, bool signed_) {
331 - TORCH_CHECK(
332 - size_bits <=
333 - std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
334 - "size_bits bit width is too large to be represented");
335 - }
336 -
337 - static void check_bias(int64_t bias) {
338 - using Bias = decltype(std::declval<Self>().bias);
339 - TORCH_CHECK(bias <= std::numeric_limits<Bias>::max() &&
340 - bias >= std::numeric_limits<Bias>::min(),
341 - "bias too large or small to be represented");
342 - }
343 -
344 - static void check_exponent(int64_t exponent) {
345 - TORCH_CHECK(
346 - exponent <=
347 - std::numeric_limits<decltype(std::declval<Self>().exponent)>::max(),
348 - "exponent bit width is too large to be represented");
349 - }
350 -
351 - static void check_mantissa(int64_t mantissa) {
352 - TORCH_CHECK(
353 - mantissa <=
354 - std::numeric_limits<decltype(std::declval<Self>().mantissa)>::max(),
355 - "mantissa bit width is too large to be represented");
356 - }
357 -
358 - static SelfPtr int_(int64_t size_bits, c10::optional<int64_t> bias) {
359 - check_size_bits(size_bits, true);
360 - check_bias(bias.value_or(0));
361 - return c10::make_intrusive<Self>(
362 - ScalarType::int_(size_bits, bias.value_or(0)));
363 - }
364 -
365 - static SelfPtr uint(int64_t size_bits, c10::optional<int64_t> bias) {
366 - check_size_bits(size_bits, true);
367 - check_bias(bias.value_or(0));
368 - return c10::make_intrusive<Self>(
369 - ScalarType::uint(size_bits, bias.value_or(0)));
370 - }
371 -
372 - static SelfPtr float_IEEE754(int64_t exponent, int64_t mantissa) {
373 - check_mantissa(mantissa);
374 - check_exponent(exponent);
375 - return c10::make_intrusive<Self>(
376 - ScalarType::float_IEEE754(exponent, mantissa));
377 - }
378 -
379 - static SelfPtr float_(int64_t exponent, int64_t mantissa,
380 - bool finite_values_only, int64_t nan_repr) {
381 - check_mantissa(mantissa);
382 - check_exponent(exponent);
383 - return c10::make_intrusive<Self>(ScalarType::float_(
384 - exponent, mantissa, finite_values_only, NanRepr(nan_repr)));
385 - }
386 -
387 - // This needs to be implemented and throw a TypeError in order for
388 - // PyTorch's opcheck to work on ops that use ScalarTypes.
389 - int64_t len() const {
390 - throw c10::TypeError({__func__, __FILE__, static_cast<uint32_t>(__LINE__)},
391 - "__len__ not implemented");
392 - return 0;
393 - }
394 -
395 - // Serialize a ScalarType into a tuple of pairs. Where each pair
396 - // is a (fieldname, value).
397 - // For simplicity, we are just going to convert to a ScalarTypeId.
398 - std::tuple<std::tuple<std::string, int64_t>> obj_flatten() const {
399 - return {{"ScalarType", id()}};
400 - }
401 -
402 - // Deserialize a scalar type that has been serialized by obj_flatten,
403 - // ostensibly from a tuple of (member name, value) pairs, but in reality
404 - // just a ScalarTypeId.
405 - static SelfPtr obj_unflatten(
406 - std::tuple<std::tuple<std::string, int64_t>> const& flat_type) {
407 - return c10::make_intrusive<Self>(
408 - from_id(std::get<1>(std::get<0>(flat_type))));
409 - }
410 -
411 - template <typename T>
412 - static void bind_readonly_property(torch::class_<Self>& cls,
413 - std::string const& name, T Base::*field) {
414 - auto getter_func_helper = [field = std::move(field)](SelfPtr const& self) {
415 - if constexpr (std::is_member_function_pointer_v<decltype(field)>) {
416 - return (self.get()->*field)();
417 - } else {
418 - return self.get()->*field;
419 - }
420 - };
421 -
422 - auto getter_func = [field = std::move(field),
423 - getter_func_helper = std::move(getter_func_helper)](
424 - SelfPtr const& self) {
425 - auto val = getter_func_helper(self);
426 - // upconvert uint8_t, int32_t etc. to int64_t for python
427 - if constexpr (std::is_integral_v<T>) {
428 - return static_cast<int64_t>(val);
429 - } else {
430 - return val;
431 - }
432 - };
433 -
434 - cls.def_property(name, getter_func);
435 - }
436 -
437 - template <typename MemberFunc, typename Cls>
438 - static void bind_function(torch::class_<Self>& cls, const std::string& name,
439 - MemberFunc Cls::*member) {
440 - cls.def(name, [member = std::move(member)](SelfPtr const& self) {
441 - return (self.get()->*member)();
442 - });
443 - }
444 -
445 - template <typename Func>
446 - static void bind_function(torch::class_<Self>& cls, const std::string& name,
447 - Func func) {
448 - cls.def(name, func);
449 - }
450 -
451 - template <typename Func>
452 - static void bind_static_function(torch::class_<Self>& cls,
453 - const std::string& name, Func func) {
454 - cls.def_static(name, func);
455 - }
456 -
457 - static void bind_class(torch::Library& lib) {
458 - auto cls = lib.class_<ScalarTypeTorch>("ScalarType")
459 - .def(torch::init<int64_t, int64_t, int64_t, bool>());
460 -
461 - // Bind Properties
462 - bind_readonly_property(cls, "mantissa", &Base::mantissa);
463 - bind_readonly_property(cls, "exponent", &Base::exponent);
464 - bind_readonly_property(cls, "bias", &Base::bias);
465 - bind_readonly_property(cls, "signed", &Base::is_signed);
466 - bind_readonly_property(cls, "size_bits", &Base::size_bits);
467 -
468 - // Bind member functions
469 - bind_function(cls, "is_signed", &Base::is_signed);
470 - bind_function(cls, "is_integer", &Base::is_integer);
471 - bind_function(cls, "is_floating_point", &Base::is_floating_point);
472 - bind_function(cls, "is_ieee_754", &Base::is_ieee_754);
473 - bind_function(cls, "has_nans", &Base::has_nans);
474 - bind_function(cls, "has_infs", &Base::has_infs);
475 - bind_function(cls, "has_bias", &Base::has_bias);
476 -
477 - bind_function(cls, "max", [](SelfPtr const& self) {
478 - return std::visit([](auto arg) { return c10::IValue(arg); },
479 - self.get()->max());
480 - });
481 - bind_function(cls, "min", [](SelfPtr const& self) {
482 - return std::visit([](auto arg) { return c10::IValue(arg); },
483 - self.get()->min());
484 - });
485 -
486 - bind_function(cls, "__len__", &ScalarTypeTorch::len);
487 - bind_function(cls, "__str__", &Base::str);
488 - bind_function(cls, "__eq__", [](SelfPtr const& self, SelfPtr const& other) {
489 - return *self == *other;
490 - });
491 - bind_function(cls, "__repr__", [](SelfPtr const& self) {
492 - return "ScalarType." + self.get()->str();
493 - });
494 -
495 - bind_function(cls, "__obj_flatten__", &ScalarTypeTorch::obj_flatten);
496 - bind_static_function(cls, "__obj_unflatten__",
497 - &ScalarTypeTorch::obj_unflatten);
498 -
499 - // Bind static functions (convenience constructors)
500 - bind_static_function(cls, "int_", &ScalarTypeTorch::int_);
501 - bind_static_function(cls, "uint", &ScalarTypeTorch::uint);
502 - bind_static_function(cls, "float_IEEE754", &ScalarTypeTorch::float_IEEE754);
503 - bind_static_function(cls, "float_", &ScalarTypeTorch::float_);
504 - }
505 -};
506 -
507 -using ScalarTypeId = int64_t;
508 -using ScalarTypeTorchPtr = c10::intrusive_ptr<ScalarTypeTorch>;
307 +using ScalarTypeId = ScalarType::Id;
308
309 // "rust style" names generally following:
310 // https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L60-L70
csrc/core/torch_bindings.cpp deleted
-16
@@ -1,16 +0,0 @@
1 -#include <torch/library.h>
2 -
3 -#include "scalar_type.hpp"
4 -#include "registration.h"
5 -
6 -// Note the CORE exstension will be built for (almost) all hardware targets so
7 -// new additions must account for this. (currently not built for TPU and Neuron)
8 -
9 -TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, lib) {
10 - // ScalarType, a custom class for representing data types that supports
11 - // quantized types, declared here so it can be used when creating interfaces
12 - // for custom ops.
13 - vllm::ScalarTypeTorch::bind_class(lib);
14 -}
15 -
16 -REGISTER_EXTENSION(TORCH_EXTENSION_NAME)
csrc/moe/marlin_moe_ops.cu
+8 -7
@@ -484,21 +484,22 @@ torch::Tensor marlin_gemm_moe(
484 const torch::Tensor& topk_ids, const torch::Tensor& b_scales,
485 torch::Tensor& b_zeros, const torch::Tensor& g_idx,
486 const torch::Tensor& perm, torch::Tensor& workspace,
487 - vllm::ScalarTypeTorchPtr const& b_q_type, int64_t size_m, int64_t size_n,
487 + vllm::ScalarTypeId const b_q_type_id, int64_t size_m, int64_t size_n,
488 int64_t size_k, bool is_k_full, int64_t num_experts, int64_t topk,
489 int64_t moe_block_size, bool replicate_input, bool apply_weights) {
490 + vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
491 bool has_zp = b_zeros.size(1) != 0;
492 if (has_zp) {
493 TORCH_CHECK(
493 - *b_q_type == vllm::kU4,
494 - "b_q_type must be u4 when has_zp = True. Got = ", b_q_type->str());
494 + b_q_type == vllm::kU4,
495 + "b_q_type must be u4 when has_zp = True. Got = ", b_q_type.str());
496 } else {
497 TORCH_CHECK(
497 - *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
498 - "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type->str());
498 + b_q_type == vllm::kU4B8 || b_q_type == vllm::kU8B128,
499 + "b_q_type must be uint4b8 or uint8b128. Got = ", b_q_type.str());
500 }
501
501 - int pack_factor = 32 / b_q_type->size_bits();
502 + int pack_factor = 32 / b_q_type.size_bits();
503
504 int max_par = 4;
505
@@ -575,7 +576,7 @@ torch::Tensor marlin_gemm_moe(
576 topk_weights.data_ptr(), topk_ids.data_ptr(), b_scales.data_ptr(),
577 b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(), a_tmp.data_ptr(),
578 expert_offsets.data_ptr(), size_m, size_n, size_k, workspace.data_ptr(),
578 - *b_q_type, has_act_order, is_k_full, has_zp, num_groups, group_size,
579 + b_q_type, has_act_order, is_k_full, has_zp, num_groups, group_size,
580 num_experts, topk, moe_block_size, dev,
581 at::cuda::getCurrentCUDAStream(dev), thread_k, thread_n, sms, max_par,
582 replicate_input, apply_weights);
csrc/moe/torch_bindings.cpp
+3 -2
@@ -13,8 +13,9 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, m) {
13 "marlin_gemm_moe(Tensor! a, Tensor! b_q_weights, Tensor! sorted_ids, "
14 "Tensor! topk_weights, Tensor! topk_ids, Tensor! b_scales, Tensor! "
15 "b_zeros, Tensor! g_idx, Tensor! perm, Tensor! workspace, "
16 - "__torch__.torch.classes._core_C.ScalarType b_q_type, int size_m, "
17 - "int size_n, int size_k, bool is_k_full, int num_experts, int topk, "
16 + "int b_q_type, SymInt size_m, "
17 + "SymInt size_n, SymInt size_k, bool is_k_full, int num_experts, int "
18 + "topk, "
19 "int moe_block_size, bool replicate_input, bool apply_weights)"
20 " -> Tensor");
21 // conditionally compiled so impl registration is in source file
csrc/quantization/gptq_marlin/gptq_marlin.cu
+12 -11
@@ -80,7 +80,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
80 torch::Tensor& b_scales, torch::Tensor& b_zeros,
81 torch::Tensor& g_idx, torch::Tensor& perm,
82 torch::Tensor& workspace,
83 - vllm::ScalarTypeTorchPtr const& b_q_type,
83 + vllm::ScalarTypeId const b_q_type_id,
84 int64_t size_m, int64_t size_n, int64_t size_k,
85 bool is_k_full, bool has_zp) {
86 TORCH_CHECK_NOT_IMPLEMENTED(false,
@@ -2132,22 +2132,23 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2132 torch::Tensor& b_scales, torch::Tensor& b_zeros,
2133 torch::Tensor& g_idx, torch::Tensor& perm,
2134 torch::Tensor& workspace,
2135 - vllm::ScalarTypeTorchPtr const& b_q_type,
2135 + vllm::ScalarTypeId const& b_q_type_id,
2136 int64_t size_m, int64_t size_n, int64_t size_k,
2137 bool is_k_full, bool has_zp,
2138 bool use_fp32_reduce) {
2139 + vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
2140 if (has_zp) {
2140 - TORCH_CHECK(*b_q_type == vllm::kU4 || *b_q_type == vllm::kU8,
2141 - "b_q_type must be u4 or u8 when has_zp = True. Got = ",
2142 - b_q_type->str());
2141 + TORCH_CHECK(
2142 + b_q_type == vllm::kU4 || b_q_type == vllm::kU8,
2143 + "b_q_type must be u4 or u8 when has_zp = True. Got = ", b_q_type.str());
2144 } else {
2145 TORCH_CHECK(
2145 - *b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
2146 + b_q_type == vllm::kU4B8 || b_q_type == vllm::kU8B128,
2147 "b_q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ",
2147 - b_q_type->str());
2148 + b_q_type.str());
2149 }
2150
2150 - int pack_factor = 32 / b_q_type->size_bits();
2151 + int pack_factor = 32 / b_q_type.size_bits();
2152
2153 // Verify A
2154 TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0),
@@ -2279,7 +2280,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2280 c_tmp.data_ptr<float>(), b_scales.data_ptr<at::Half>(),
2281 b_zeros.data_ptr(), g_idx.data_ptr(), perm.data_ptr(),
2282 a_tmp.data_ptr<at::Half>(), size_m, size_n, size_k,
2282 - workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2283 + workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2284 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2285 thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2286 } else if (a.scalar_type() == at::ScalarType::BFloat16) {
@@ -2288,7 +2289,7 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2289 c.data_ptr<at::BFloat16>(), c_tmp.data_ptr<float>(),
2290 b_scales.data_ptr<at::BFloat16>(), b_zeros.data_ptr(), g_idx.data_ptr(),
2291 perm.data_ptr(), a_tmp.data_ptr<at::BFloat16>(), size_m, size_n, size_k,
2291 - workspace.data_ptr(), *b_q_type, has_act_order, is_k_full, has_zp,
2292 + workspace.data_ptr(), b_q_type, has_act_order, is_k_full, has_zp,
2293 num_groups, group_size, dev, at::cuda::getCurrentCUDAStream(dev),
2294 thread_k, thread_n, sms, marlin::max_par, use_fp32_reduce);
2295 } else {
@@ -2302,4 +2303,4 @@ torch::Tensor gptq_marlin_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
2303
2304 TORCH_LIBRARY_IMPL_EXPAND(TORCH_EXTENSION_NAME, CUDA, m) {
2305 m.impl("gptq_marlin_gemm", &gptq_marlin_gemm);
2305 -}
\ No newline at end of file
2306 +}
csrc/quantization/machete/machete_pytorch.cu
+9 -7
@@ -38,9 +38,10 @@ static auto scalar_type_dispatch(ScalarType const& type, Fn fn) {
38 // Interface
39 //
40
41 -std::vector<std::string> supported_schedules(ScalarTypeTorchPtr const& btype) {
41 +std::vector<std::string> supported_schedules(ScalarTypeId const btype_id) {
42 #if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
43 - return scalar_type_dispatch(*btype, [&](auto BType) {
43 + vllm::ScalarType b_type = ScalarType::from_id(btype_id);
44 + return scalar_type_dispatch(b_type, [&](auto BType) {
45 return GemmDispatcher<half_t, decltype(BType)>::supported_schedules();
46 });
47 #else
@@ -49,7 +50,7 @@ std::vector<std::string> supported_schedules(ScalarTypeTorchPtr const& btype) {
50 }
51
52 torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
52 - ScalarTypeTorchPtr const& btype,
53 + ScalarTypeId const btype_id,
54 c10::optional<torch::Tensor> const& scales,
55 c10::optional<torch::Tensor> const& zeros,
56 c10::optional<int64_t> group_size,
@@ -57,6 +58,7 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
58 c10::optional<double> alpha, c10::optional<double> beta,
59 c10::optional<std::string> schedule) {
60 #if defined(__CUDACC_VER_MAJOR__) && __CUDACC_VER_MAJOR__ >= 12
61 + ScalarType const btype = ScalarType::from_id(btype_id);
62 auto args = PyTorchArguments{.A = A,
63 .B = B,
64 .scales = scales,
@@ -67,7 +69,7 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
69 .beta = beta,
70 .schedule = schedule};
71
70 - return scalar_type_dispatch(*btype, [&](auto BType) {
72 + return scalar_type_dispatch(btype, [&](auto BType) {
73 return AT_DISPATCH_SUPPORTED_COMPUTE_TYPES(
74 A.scalar_type(), "machete_gemm", [&] {
75 using ComputeType = equivalent_cutlass_type_t<scalar_t>;
@@ -79,9 +81,9 @@ torch::Tensor gemm(torch::Tensor const& A, torch::Tensor const& B,
81 #endif
82 }
83
82 -torch::Tensor prepack_B(torch::Tensor const& B,
83 - vllm::ScalarTypeTorchPtr const& btype) {
84 - return scalar_type_dispatch(*btype, [&](auto BType) {
84 +torch::Tensor prepack_B(torch::Tensor const& B, ScalarTypeId const btype_id) {
85 + ScalarType const btype = ScalarType::from_id(btype_id);
86 + return scalar_type_dispatch(btype, [&](auto BType) {
87 return PrepackBDispatcher<half_t, decltype(BType), half_t>::dispatch(B);
88 });
89 }
csrc/quantization/marlin/sparse/marlin_24_cuda_kernel.cu
+8 -7
@@ -89,7 +89,7 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
89 torch::Tensor& b_meta,
90 torch::Tensor& b_scales,
91 torch::Tensor& workspace,
92 - vllm::ScalarTypeTorchPtr const& b_q_type,
92 + vllm::ScalarTypeId const b_q_type_id,
93 int64_t size_m, int64_t size_n,
94 int64_t size_k) {
95 TORCH_CHECK_NOT_IMPLEMENTED(
@@ -1029,13 +1029,14 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1029 torch::Tensor& b_meta,
1030 torch::Tensor& b_scales,
1031 torch::Tensor& workspace,
1032 - vllm::ScalarTypeTorchPtr const& b_q_type,
1032 + vllm::ScalarTypeId const b_q_type_id,
1033 int64_t size_m, int64_t size_n,
1034 int64_t size_k) {
1035 + vllm::ScalarType const b_q_type = vllm::ScalarType::from_id(b_q_type_id);
1036 // Verify num_bits
1036 - TORCH_CHECK(*b_q_type == vllm::kU4B8 || *b_q_type == vllm::kU8B128,
1037 - "num_bits must be uint4b8 or uint8b128. Got = ", b_q_type->str());
1038 - int pack_factor = 32 / b_q_type->size_bits();
1037 + TORCH_CHECK(b_q_type == vllm::kU4B8 || b_q_type == vllm::kU8B128,
1038 + "num_bits must be uint4b8 or uint8b128. Got = ", b_q_type.str());
1039 + int pack_factor = 32 / b_q_type.size_bits();
1040
1041 // Verify M
1042 TORCH_CHECK(size_m == a.size(0),
@@ -1130,8 +1131,8 @@ torch::Tensor gptq_marlin_24_gemm(torch::Tensor& a, torch::Tensor& b_q_weight,
1131 marlin_24::marlin_cuda_2_4(
1132 a.data_ptr(), b_q_weight.data_ptr(), b_meta.data_ptr(), c.data_ptr(),
1133 b_scales.data_ptr(), size_n, size_m, size_k, workspace.data_ptr(),
1133 - b_q_type->size_bits(), groupsize, dev,
1134 - at::cuda::getCurrentCUDAStream(dev), thread_k, thread_m, sms, max_par);
1134 + b_q_type.size_bits(), groupsize, dev, at::cuda::getCurrentCUDAStream(dev),
1135 + thread_k, thread_m, sms, max_par);
1136
1137 return c;
1138 }
csrc/torch_bindings.cpp
+20 -25
@@ -140,13 +140,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
140 // Quantized GEMM for AWQ.
141 ops.def(
142 "awq_gemm(Tensor _in_feats, Tensor _kernel, Tensor _scaling_factors, "
143 - "Tensor _zeros, int split_k_iters) -> Tensor");
143 + "Tensor _zeros, SymInt split_k_iters) -> Tensor");
144 ops.impl("awq_gemm", torch::kCUDA, &awq_gemm);
145
146 // Dequantization for AWQ.
147 ops.def(
148 "awq_dequantize(Tensor _kernel, Tensor _scaling_factors, "
149 - "Tensor _zeros, int split_k_iters, int thx, int thy) -> Tensor");
149 + "Tensor _zeros, SymInt split_k_iters, int thx, int thy) -> Tensor");
150 ops.impl("awq_dequantize", torch::kCUDA, &awq_dequantize);
151
152 // Note about marlin kernel 'workspace' arguments:
@@ -166,32 +166,26 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
166 // Marlin (Dense) Optimized Quantized GEMM for GPTQ.
167 ops.def(
168 "marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
169 - "Tensor! workspace, int size_m, int size_n, int size_k) -> Tensor");
169 + "Tensor! workspace, SymInt size_m, SymInt size_n, SymInt size_k) -> "
170 + "Tensor");
171 // conditionally compiled so impl in source file
172
173 // Marlin_24 (Sparse) Optimized Quantized GEMM for GPTQ.
174 ops.def(
175 "gptq_marlin_24_gemm(Tensor a, Tensor b_q_weight, Tensor b_meta, "
176 "Tensor b_scales, Tensor workspace, "
176 - "__torch__.torch.classes._core_C.ScalarType b_q_type, "
177 - "int size_m, int size_n, int size_k) -> Tensor");
177 + "int b_q_type, "
178 + "SymInt size_m, SymInt size_n, SymInt size_k) -> Tensor");
179 // conditionally compiled so impl in source file
180
181 // Machete (Dense) Optimized Mixed Precision GEMM for Hopper.
182 + ops.def("machete_supported_schedules(int btype) -> str[]");
183 ops.def(
182 - "machete_supported_schedules("
183 - " __torch__.torch.classes._core_C.ScalarType btype"
184 - ") -> str[]");
185 - ops.def(
186 - "machete_gemm(Tensor A, Tensor B,"
187 - " __torch__.torch.classes._core_C.ScalarType btype,"
188 - " Tensor? scales, Tensor? zeros, int? group_size,"
184 + "machete_gemm(Tensor A, Tensor B, int btype, "
185 + " Tensor? scales, Tensor? zeros, int? group_size, "
186 " Tensor? C, float? alpha, float? beta, str? schedule)"
187 "-> Tensor");
191 - ops.def(
192 - "machete_prepack_B(Tensor B,"
193 - " __torch__.torch.classes._core_C.ScalarType btype)"
194 - "-> Tensor");
188 + ops.def("machete_prepack_B(Tensor B, int btype) -> Tensor");
189 // conditionally compiled so impl registration is in source file
190
191 ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
@@ -201,8 +195,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
195 ops.def(
196 "gptq_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
197 "Tensor b_zeros, Tensor g_idx, Tensor perm, Tensor workspace, "
204 - "__torch__.torch.classes._core_C.ScalarType b_q_type, "
205 - "int size_m, int size_n, int size_k, bool is_k_full, "
198 + "int b_q_type, "
199 + "SymInt size_m, SymInt size_n, SymInt size_k, bool is_k_full, "
200 "bool has_zp, bool use_fp32_reduce) -> Tensor");
201 // conditionally compiled so impl registration is in source file
202
@@ -219,32 +213,33 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
213 // conditionally compiled so impl registrations are in source file
214
215 // Dequantization for GGML.
222 - ops.def("ggml_dequantize(Tensor W, int type, int m, int n) -> Tensor");
216 + ops.def("ggml_dequantize(Tensor W, int type, SymInt m, SymInt n) -> Tensor");
217 ops.impl("ggml_dequantize", torch::kCUDA, &ggml_dequantize);
218
219 // mmvq kernel for GGML.
220 ops.def(
227 - "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, int row) "
221 + "ggml_mul_mat_vec_a8(Tensor W, Tensor X, int type, SymInt row) "
222 "-> Tensor");
223 ops.impl("ggml_mul_mat_vec_a8", torch::kCUDA, &ggml_mul_mat_vec_a8);
224
225 // mmq kernel for GGML.
232 - ops.def("ggml_mul_mat_a8(Tensor W, Tensor X, int type, int row) -> Tensor");
226 + ops.def(
227 + "ggml_mul_mat_a8(Tensor W, Tensor X, int type, SymInt row) -> Tensor");
228 ops.impl("ggml_mul_mat_a8", torch::kCUDA, &ggml_mul_mat_a8);
229
230 // fp8_marlin Optimized Quantized GEMM for FP8 weight-only.
231 ops.def(
232 "fp8_marlin_gemm(Tensor a, Tensor b_q_weight, Tensor b_scales, "
238 - "Tensor! workspace, int num_bits, int size_m, int size_n, "
239 - "int size_k) -> Tensor");
233 + "Tensor! workspace, int num_bits, SymInt size_m, SymInt size_n, "
234 + "SymInt size_k) -> Tensor");
235 // conditionally compiled so impl registration is in source file
236
237 // marlin_qqq_gemm for QQQ.
238 ops.def(
239 "marlin_qqq_gemm(Tensor a, Tensor b_q_weight, "
240 "Tensor s_tok, Tensor s_ch, Tensor s_group, "
246 - "Tensor! workspace, int size_m, int size_n, "
247 - "int size_k) -> Tensor");
241 + "Tensor! workspace, SymInt size_m, SymInt size_n, "
242 + "SymInt size_k) -> Tensor");
243 // conditionally compiled so impl registration is in source file
244
245 // CUTLASS w8a8 GEMM, supporting symmetric per-tensor or per-row/column
python_only_dev.py
-1
@@ -39,7 +39,6 @@ assert cwd != package_path, "should not import from the current directory"
39
40 files_to_copy = [
41 "vllm/_C.abi3.so",
42 - "vllm/_core_C.abi3.so",
42 "vllm/_moe_C.abi3.so",
43 "vllm/vllm_flash_attn/vllm_flash_attn_c.abi3.so",
44 "vllm/vllm_flash_attn/flash_attn_interface.py",
setup.py
-7
@@ -290,10 +290,6 @@ def _build_custom_ops() -> bool:
290 return _is_cuda() or _is_hip() or _is_cpu()
291
292
293 -def _build_core_ext() -> bool:
294 - return not (_is_neuron() or _is_tpu() or _is_openvino() or _is_xpu())
295 -
296 -
293 def get_hipcc_rocm_version():
294 # Run the hipcc --version command
295 result = subprocess.run(['hipcc', '--version'],
@@ -456,9 +452,6 @@ def get_requirements() -> List[str]:
452
453 ext_modules = []
454
459 -if _build_core_ext():
460 - ext_modules.append(CMakeExtension(name="vllm._core_C"))
461 -
455 if _is_cuda() or _is_hip():
456 ext_modules.append(CMakeExtension(name="vllm._moe_C"))
457
tests/compile/utils.py
+4 -4
@@ -69,11 +69,11 @@ def check_full_graph_support(model,
69 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(optimization_level)
70 os.environ["VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE"] = "1"
71
72 - # Inductor doesn't support fp8/gptq_marlin_24 yet.
72 + # Inductor doesn't support fp8 and the base meta llama uses too
73 + # much memory.
74 quantization = model_kwargs.get("quantization")
74 - if (quantization == "fp8" or quantization == "gptq_marlin"
75 - or quantization == "gptq_marlin_24"
76 - ) and optimization_level >= CompilationLevel.INDUCTOR:
75 + if ((quantization == "fp8" or model == "meta-llama/Meta-Llama-3-8B")
76 + and optimization_level >= CompilationLevel.INDUCTOR):
77 return
78
79 prompts = [
tests/kernels/test_machete_gemm.py
+5 -4
@@ -80,7 +80,7 @@ def machete_quantize_and_pack(w: torch.Tensor,
80 w_q = w_q.t().contiguous().t() # convert to col major
81 w_q_machete = ops.machete_prepack_B(w_q, wtype)
82
83 - opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype))
83 + opcheck(torch.ops._C.machete_prepack_B, (w_q, wtype.id))
84
85 return w_ref, w_q_machete, w_s, w_zp
86
@@ -153,9 +153,10 @@ def test_machete_all_schedules(shape, atype: torch.dtype,
153 schedule=schedule,
154 )
155
156 - opcheck(torch.ops._C.machete_gemm,
157 - (a, w_q_machete, wtype, w_s, maybe_convert_zeropoints(
158 - w_zp, w_s), group_size, None, None, None, schedule))
156 + opcheck(
157 + torch.ops._C.machete_gemm,
158 + (a, w_q_machete, wtype.id, w_s, maybe_convert_zeropoints(
159 + w_zp, w_s), group_size, None, None, None, schedule))
160
161 # Relax atol as our reduction dim becomes larger (more rounding error)
162 # Relax atol when we have zeropoints since the way machete applies
tests/kernels/test_marlin_gemm.py
+13 -3
@@ -225,7 +225,7 @@ def test_gptq_marlin_gemm(
225 opcheck(
226 torch.ops._C.gptq_marlin_gemm,
227 (a_input, marlin_q_w, marlin_s, marlin_zp, g_idx, sort_indices,
228 - workspace.scratch, quant_type, a_input.shape[0], b_weight.shape[1],
228 + workspace.scratch, quant_type.id, a_input.shape[0], b_weight.shape[1],
229 a_input.shape[1], is_k_full, False, use_fp32_reduce),
230 test_utils=DEFAULT_OPCHECK_TEST_UTILS)
231
@@ -254,6 +254,16 @@ def test_gptq_marlin_gemm(
254 assert max_diff < 0.04
255
256
257 +# TODO: find better way to test this?
258 +@torch.compile(fullgraph=True)
259 +def marlin_24_gemm_tester(a_input, marlin_24_q_w_comp, marlin_24_meta,
260 + marlin_24_s, scratch, quant_type, size_m, size_n,
261 + size_k):
262 + return ops.gptq_marlin_24_gemm(a_input, marlin_24_q_w_comp, marlin_24_meta,
263 + marlin_24_s, scratch, quant_type, size_m,
264 + size_n, size_k)
265 +
266 +
267 @pytest.mark.skipif(not is_quant_method_supported("gptq_marlin"),
268 reason="Marlin is not supported on this GPU type.")
269 @pytest.mark.parametrize("k_chunk", MARLIN_24_K_CHUNKS)
@@ -282,11 +292,11 @@ def test_gptq_marlin_24_gemm(k_chunk, n_chunk, quant_type, group_size,
292
293 opcheck(torch.ops._C.gptq_marlin_24_gemm,
294 (a_input, marlin_24_q_w_comp, marlin_24_meta, marlin_24_s,
285 - workspace_24.scratch, quant_type, a_input.shape[0],
295 + workspace_24.scratch, quant_type.id, a_input.shape[0],
296 b_weight.shape[1], a_input.shape[1]),
297 test_utils=DEFAULT_OPCHECK_TEST_UTILS)
298
289 - output = ops.gptq_marlin_24_gemm(
299 + output = marlin_24_gemm_tester(
300 a_input,
301 marlin_24_q_w_comp,
302 marlin_24_meta,
tests/kernels/test_moe.py
+2 -2
@@ -240,8 +240,8 @@ def test_fused_marlin_moe(
240 requires_grad=False)
241 opcheck(torch.ops._moe_C.marlin_gemm_moe,
242 (a, qweight1, sorted_token_ids, topk_weights, topk_ids,
243 - scales1, zp, g_idx1, sort_indices1, workspace, quant_type, m,
244 - 2 * n, k, True, e, topk, block_size_m, True, False))
243 + scales1, zp, g_idx1, sort_indices1, workspace, quant_type.id,
244 + m, 2 * n, k, True, e, topk, block_size_m, True, False))
245
246
247 @pytest.mark.skip("This test is here for the sake of debugging, "
tests/test_scalartype.py
+2 -2
@@ -32,5 +32,5 @@ def test_scalar_type_min_max(type_tuple):
32 max = torch.iinfo(torch_type).max
33
34 print(t, min, max, t.min(), t.max())
35 - assert min == t.min()
36 - assert max == t.max()
35 + assert min == t.min(), f"min: {min} != {t.min()}"
36 + assert max == t.max(), f"max: {max} != {t.max()}"
tools/report_build_time_ninja.py
-1
@@ -16,7 +16,6 @@ Typical output looks like this:
16 2.6 weighted s to build ...torch_bindings.cpp.o (31.5 s elapsed time)
17 3.2 weighted s to build ...torch_bindings.cpp.o (38.5 s elapsed time)
18 Longest build steps for .so (linking):
19 - 0.1 weighted s to build _core_C.abi3.so (0.7 s elapsed time)
19 0.1 weighted s to build _moe_C.abi3.so (1.0 s elapsed time)
20 0.5 weighted s to build ...flash_attn_c.abi3.so (1.1 s elapsed time)
21 6.2 weighted s to build _C.abi3.so (6.2 s elapsed time)
vllm/_core_ext.py deleted
-278
@@ -1,278 +0,0 @@
1 -import importlib.util
2 -from enum import Enum
3 -from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
4 -
5 -import torch
6 -
7 -from vllm.logger import init_logger
8 -
9 -logger = init_logger(__name__)
10 -core_C_available = importlib.util.find_spec('._core_C', 'vllm') is not None
11 -
12 -
13 -# Mirrors enum in `core/scalar_type.hpp`
14 -class NanRepr(Enum):
15 - NONE = 0 # nans are not supported
16 - IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s
17 - EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s
18 -
19 -
20 -if TYPE_CHECKING or not core_C_available:
21 - # On platforms were we cannot use/build the C++ core extension (i.e. namely
22 - # neuron and tpu), we define the mock ScalarType class here that partially
23 - # mimics the C++ ScalarType class.
24 - #
25 - # We also use this provide type signatures to the Python LSP for the methods
26 - # in the C++ ScalarType class. So these type signatures should be kept
27 - # in sync with csrc/core/scalar_type.hpp
28 -
29 - from dataclasses import dataclass
30 -
31 - @dataclass(frozen=True)
32 - class ScalarType:
33 - """
34 - ScalarType can represent a wide range of floating point and integer
35 - types, in particular it can be used to represent sub-byte data types
36 - (something that torch.dtype currently does not support). It is also
37 - capable of representing types with a bias, i.e.:
38 - `stored_value = value + bias`,
39 - this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
40 - of 8). The implementation for this class can be found in
41 - csrc/core/scalar_type.hpp, these type signatures should be kept in sync
42 - with that file.
43 - """
44 -
45 - exponent: int
46 - """
47 - Number of bits in the exponent if this is a floating point type
48 - (zero if this an integer type)
49 - """
50 -
51 - mantissa: int
52 - """
53 - Number of bits in the mantissa if this is a floating point type,
54 - or the number bits representing an integer excluding the sign bit if
55 - this an integer type.
56 - """
57 -
58 - bias: int
59 - """
60 - bias used to encode the values in this scalar type
61 - (value = stored_value - bias, default 0) for example if we store the
62 - type as an unsigned integer with a bias of 128 then the value 0 will be
63 - stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
64 - """
65 -
66 - signed: bool
67 - "If the type is signed (i.e. has a sign bit)"
68 -
69 - _finite_values_only: bool = False
70 - """
71 - Private: if NANs are supported, used `has_infs()` instead.
72 - """
73 -
74 - nan_repr: int = NanRepr.IEEE_754.value
75 - """
76 - How NaNs are represent in this scalar type, returns NanRepr value.
77 - (not applicable for integer types)
78 - """
79 -
80 - @property
81 - def size_bits(self):
82 - return self.exponent + self.mantissa + int(self.signed)
83 -
84 - def min(self) -> Union[int, float]:
85 - """
86 - Min representable value for this scalar type.
87 - (accounting for bias if there is one)
88 - """
89 - raise NotImplementedError
90 -
91 - def max(self) -> Union[int, float]:
92 - """
93 - Max representable value for this scalar type.
94 - (accounting for bias if there is one)
95 - """
96 - raise NotImplementedError
97 -
98 - def is_signed(self) -> bool:
99 - """
100 - If the type is signed (i.e. has a sign bit), same as `signed`
101 - added for consistency with:
102 - https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
103 - """
104 - ...
105 -
106 - def is_floating_point(self) -> bool:
107 - "If the type is a floating point type"
108 - return self.exponent != 0
109 -
110 - def is_integer(self) -> bool:
111 - "If the type is an integer type"
112 - return self.exponent == 0
113 -
114 - def has_bias(self) -> bool:
115 - "If the type has a non-zero bias"
116 - return self.bias != 0
117 -
118 - def has_infs(self) -> bool:
119 - "If the type is floating point and supports infinity"
120 - return not self._finite_values_only
121 -
122 - def has_nans(self) -> bool:
123 - return self.nan_repr != NanRepr.NONE.value
124 -
125 - def is_ieee_754(self) -> bool:
126 - """
127 - If the type is a floating point type that follows IEEE 754
128 - conventions
129 - """
130 - return self.nan_repr == NanRepr.IEEE_754.value and \
131 - not self._finite_values_only
132 -
133 - def __str__(self) -> str:
134 - raise NotImplementedError
135 -
136 - def __repr__(self) -> str:
137 - raise NotImplementedError
138 -
139 - # __len__ needs to be defined (and has to throw TypeError) for pytorch's
140 - # opcheck to work.
141 - def __len__(self) -> int:
142 - raise TypeError
143 -
144 - #
145 - # Convenience Constructors
146 - #
147 -
148 - @classmethod
149 - def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
150 - "Create a signed integer scalar type (size_bits includes sign-bit)."
151 - return cls(size_bits - 1, size_bits, bias if bias else 0, True)
152 -
153 - @classmethod
154 - def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
155 - """Create a unsigned integer scalar type."""
156 - return cls(size_bits, size_bits, bias if bias else 0, False)
157 -
158 - @classmethod
159 - def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
160 - """
161 - Create a standard floating point type
162 - (i.e. follows IEEE 754 conventions).
163 - """
164 - return cls(exponent, mantissa, 0, True)
165 -
166 - @classmethod
167 - def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
168 - nan_repr: int) -> 'ScalarType':
169 - """
170 - Create a non-standard floating point type
171 - (i.e. does not follow IEEE 754 conventions).
172 - """
173 - return cls(exponent, mantissa, 0, True, finite_values_only,
174 - nan_repr)
175 -
176 -elif core_C_available:
177 - try:
178 - import vllm._core_C # noqa: F401
179 - except ImportError as e:
180 - logger.warning("Failed to import from vllm._core_C with %r", e)
181 -
182 - ScalarType = torch.classes._core_C.ScalarType
183 -
184 - if (hasattr(torch, "_library")
185 - and hasattr(torch._library, "register_fake_class")):
186 - # Needed for dynamo support of ScalarType.
187 - @torch._library.register_fake_class("_core_C::ScalarType")
188 - class FakeScalarType:
189 -
190 - def __init__(self, scalar_type):
191 - self.ScalarType = scalar_type
192 -
193 - def bias_getter(self) -> int:
194 - return self.ScalarType.bias
195 -
196 - def exponent_getter(self) -> int:
197 - return self.ScalarType.exponent
198 -
199 - def mantissa_getter(self) -> int:
200 - return self.ScalarType.mantissa
201 -
202 - def signed_getter(self) -> bool:
203 - return self.ScalarType.signed
204 -
205 - def size_bits_getter(self) -> int:
206 - return self.ScalarType.size_bits
207 -
208 - @property
209 - def size_bits(self) -> int:
210 - return self.ScalarType.size_bits
211 -
212 - def min(self) -> Union[int, float]:
213 - return self.ScalarType.min()
214 -
215 - def max(self) -> Union[int, float]:
216 - return self.ScalarType.max()
217 -
218 - def is_signed(self) -> bool:
219 - return self.ScalarType.is_signed()
220 -
221 - def is_floating_point(self) -> bool:
222 - return self.ScalarType.is_floating_point()
223 -
224 - def is_integer(self) -> bool:
225 - return self.ScalarType.is_integer()
226 -
227 - def has_bias(self) -> bool:
228 - return self.ScalarType.has_bias()
229 -
230 - def has_infs(self) -> bool:
231 - return self.ScalarType.has_infs()
232 -
233 - def has_nans(self) -> bool:
234 - return self.ScalarType.has_nans()
235 -
236 - def is_ieee_754(self) -> bool:
237 - return self.ScalarType.is_ieee_754()
238 -
239 - def __str__(self) -> str:
240 - return self.ScalarType.__str__()
241 -
242 - def __repr__(self) -> str:
243 - return self.ScalarType.__repr__()
244 -
245 - def __len__(self) -> int:
246 - return self.ScalarType.__len__()
247 -
248 - def __obj_flatten__(self) -> Tuple[Tuple[str, Any], ...]:
249 - return torch.classes._core_C.ScalarType.__obj_flatten__(
250 - self.ScalarType)
251 -
252 - @classmethod
253 - def __obj_unflatten__(
254 - cls, flat_type: Tuple[Tuple[str, Any],
255 - ...]) -> 'ScalarType':
256 - return cls(
257 - torch.classes._core_C.ScalarType.__obj_unflatten__(
258 - flat_type))
259 -
260 - @classmethod
261 - def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
262 - return ScalarType.int_(size_bits, bias)
263 -
264 - @classmethod
265 - def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
266 - return ScalarType.uint(size_bits, bias)
267 -
268 - @classmethod
269 - def float_IEEE754(cls, exponent: int,
270 - mantissa: int) -> 'ScalarType':
271 - return ScalarType.float_IEEE754(exponent, mantissa)
272 -
273 - @classmethod
274 - def float_(cls, exponent: int, mantissa: int,
275 - finite_values_only: bool,
276 - nan_repr: int) -> 'ScalarType':
277 - return ScalarType.float_(exponent, mantissa,
278 - finite_values_only, nan_repr)
vllm/_custom_ops.py
+31 -62
@@ -6,9 +6,9 @@ import torch
6 import torch.library
7
8 import vllm.envs as envs
9 -from vllm._core_ext import ScalarType
9 from vllm.logger import init_logger
10 from vllm.platforms import current_platform
11 +from vllm.scalar_type import ScalarType
12
13 logger = init_logger(__name__)
14
@@ -306,7 +306,7 @@ def gptq_marlin_24_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
306 workspace: torch.Tensor, b_q_type: ScalarType,
307 size_m: int, size_n: int, size_k: int) -> torch.Tensor:
308 return torch.ops._C.gptq_marlin_24_gemm(a, b_q_weight, b_meta, b_scales,
309 - workspace, b_q_type, size_m,
309 + workspace, b_q_type.id, size_m,
310 size_n, size_k)
311
312
@@ -316,8 +316,9 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
316 def _gptq_marlin_24_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
317 b_meta: torch.Tensor, b_scales: torch.Tensor,
318 workspace: torch.Tensor,
319 - b_q_type: ScalarType, size_m: int,
320 - size_n: int, size_k: int) -> torch.Tensor:
319 + b_q_type: ScalarType, size_m: torch.SymInt,
320 + size_n: torch.SymInt,
321 + size_k: torch.SymInt) -> torch.Tensor:
322 return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
323
324 @register_fake("_C::gptq_marlin_gemm")
@@ -329,17 +330,18 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
330 perm: torch.Tensor,
331 workspace: torch.Tensor,
332 b_q_type: ScalarType,
332 - size_m: int,
333 - size_n: int,
334 - size_k: int,
333 + size_m: torch.SymInt,
334 + size_n: torch.SymInt,
335 + size_k: torch.SymInt,
336 is_k_full: bool,
337 has_zp: bool = False,
338 use_fp32_reduce: bool = False) -> torch.Tensor:
339 return torch.empty((size_m, size_n), device=a.device, dtype=a.dtype)
340
341 @register_fake("_C::ggml_dequantize")
341 - def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int, m: int,
342 - n: int) -> torch.Tensor:
342 + def _ggml_dequantize_fake(W: torch.Tensor, quant_type: int,
343 + m: torch.SymInt,
344 + n: torch.SymInt) -> torch.Tensor:
345 return torch.empty((m, n), dtype=torch.float16, device=W.device)
346
347 @register_fake("_C::ggml_mul_mat_vec_a8")
@@ -347,7 +349,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
349 W: torch.Tensor,
350 X: torch.Tensor,
351 quant_type: int,
350 - row: int,
352 + row: torch.SymInt,
353 ) -> torch.Tensor:
354 return torch.empty((1, row), dtype=torch.float16, device=W.device)
355
@@ -356,7 +358,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
358 W: torch.Tensor,
359 X: torch.Tensor,
360 quant_type: int,
359 - row: int,
361 + row: torch.SymInt,
362 ) -> torch.Tensor:
363 batch = X.size(0)
364 return torch.empty((batch, row), dtype=torch.float16, device=W.device)
@@ -365,8 +367,8 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
367 def _marlin_qqq_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
368 s_tok: torch.Tensor, s_ch: torch.Tensor,
369 s_group: torch.Tensor, workspace: torch.Tensor,
368 - size_m: int, size_n: int,
369 - size_k: int) -> torch.Tensor:
370 + size_m: torch.SymInt, size_n: torch.SymInt,
371 + size_k: torch.SymInt) -> torch.Tensor:
372 return torch.empty((size_m, size_n),
373 dtype=torch.float16,
374 device=a.device)
@@ -374,16 +376,16 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
376 @register_fake("_C::marlin_gemm")
377 def _marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
378 b_scales: torch.Tensor, workspace: torch.Tensor,
377 - size_m: int, size_n: int,
378 - size_k: int) -> torch.Tensor:
379 + size_m: torch.SymInt, size_n: torch.SymInt,
380 + size_k: torch.SymInt) -> torch.Tensor:
381 return torch.empty((size_m, size_n),
382 dtype=torch.float16,
383 device=a.device)
384
385 @register_fake("_C::awq_dequantize")
386 def _awq_dequantize_fake(qweight: torch.Tensor, scales: torch.Tensor,
385 - zeros: torch.Tensor, split_k_iters: int, thx: int,
386 - thy: int) -> torch.Tensor:
387 + zeros: torch.Tensor, split_k_iters: torch.SymInt,
388 + thx: int, thy: int) -> torch.Tensor:
389 in_c = qweight.size(0)
390 qout_c = qweight.size(1)
391 out_c = qout_c * 8
@@ -394,7 +396,7 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
396 @register_fake("_C::awq_gemm")
397 def _awq_gemm_fake(input: torch.Tensor, qweight: torch.Tensor,
398 qzeros: torch.Tensor, scales: torch.Tensor,
397 - split_k_iters: int) -> torch.Tensor:
399 + split_k_iters: torch.SymInt) -> torch.Tensor:
400 num_in_feats = input.size(0)
401 return torch.empty((split_k_iters, num_in_feats, qweight.size(1) * 8),
402 dtype=input.dtype,
@@ -429,8 +431,9 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
431 @register_fake("_C::fp8_marlin_gemm")
432 def _fp8_marlin_gemm_fake(a: torch.Tensor, b_q_weight: torch.Tensor,
433 b_scales: torch.Tensor, workspace: torch.Tensor,
432 - num_bits: int, size_m: int, size_n: int,
433 - size_k: int) -> torch.Tensor:
434 + num_bits: int, size_m: torch.SymInt,
435 + size_n: torch.SymInt,
436 + size_k: torch.SymInt) -> torch.Tensor:
437 return torch.empty((size_m, size_n), dtype=a.dtype, device=a.device)
438
439 @register_fake("_C::machete_gemm")
@@ -457,40 +460,6 @@ if hasattr(torch.ops._C, "gptq_marlin_24_gemm"):
460 return torch.empty_like(b_q_weight,
461 memory_format=torch.contiguous_format)
462
460 - @register_fake("_C::causal_conv1d_fwd")
461 - def causal_conv1d_fwd_fake(x: torch.Tensor, weight: torch.Tensor,
462 - bias_: Optional[torch.Tensor],
463 - conv_states: Optional[torch.Tensor],
464 - cu_seq_len: Optional[torch.Tensor],
465 - cache_indices: Optional[torch.Tensor],
466 - has_initial_state: Optional[torch.Tensor],
467 - silu_activation: bool, pad_slot_id: int):
468 - return None
469 -
470 - @register_fake("_C::causal_conv1d_update")
471 - def causal_conv1d_update_fake(x: torch.Tensor, conv_state: torch.Tensor,
472 - weight: torch.Tensor,
473 - bias_: Optional[torch.Tensor],
474 - silu_activation: bool,
475 - cache_seqlens: Optional[torch.Tensor],
476 - conv_state_indices: Optional[torch.Tensor],
477 - pad_slot_id: int) -> None:
478 - return None
479 -
480 - @register_fake("_C::selective_scan_fwd")
481 - def selective_scan_fwd_fake(u: torch.Tensor, delta: torch.Tensor,
482 - A: torch.Tensor, B: torch.Tensor,
483 - C: torch.Tensor, D_: Optional[torch.Tensor],
484 - z_: Optional[torch.Tensor],
485 - delta_bias_: Optional[torch.Tensor],
486 - delta_softplus: bool,
487 - cu_seq_len: Optional[torch.Tensor],
488 - cache_indices: Optional[torch.Tensor],
489 - has_initial_state: Optional[torch.Tensor],
490 - ssm_states: Optional[torch.Tensor],
491 - pad_slot_id: int) -> None:
492 - return None
493 -
463
464 # cutlass
465 def cutlass_scaled_mm_supports_fp8(cuda_device_capability: int) -> bool:
@@ -611,7 +580,7 @@ def gptq_marlin_gemm(a: torch.Tensor,
580 has_zp: bool = False,
581 use_fp32_reduce: bool = False) -> torch.Tensor:
582 return torch.ops._C.gptq_marlin_gemm(a, b_q_weight, b_scales, b_zeros,
614 - g_idx, perm, workspace, b_q_type,
583 + g_idx, perm, workspace, b_q_type.id,
584 size_m, size_n, size_k, is_k_full,
585 has_zp, use_fp32_reduce)
586
@@ -627,7 +596,7 @@ def fp8_marlin_gemm(a: torch.Tensor, b_q_weight: torch.Tensor,
596
597 # machete
598 def machete_supported_schedules(b_type: ScalarType) -> List[str]:
630 - return torch.ops._C.machete_supported_schedules(b_type)
599 + return torch.ops._C.machete_supported_schedules(b_type.id)
600
601
602 def machete_gemm(
@@ -642,13 +611,13 @@ def machete_gemm(
611 beta: Optional[float] = None,
612 schedule: Optional[str] = None,
613 ) -> torch.Tensor:
645 - return torch.ops._C.machete_gemm(a, b_q, b_type, b_scales, b_zeros,
614 + return torch.ops._C.machete_gemm(a, b_q, b_type.id, b_scales, b_zeros,
615 b_group_size, c, alpha, beta, schedule)
616
617
618 def machete_prepack_B(b_q_weight: torch.Tensor,
619 b_type: ScalarType) -> torch.Tensor:
651 - return torch.ops._C.machete_prepack_B(b_q_weight, b_type)
620 + return torch.ops._C.machete_prepack_B(b_q_weight, b_type.id)
621
622
623 if hasattr(torch.ops._C, "permute_cols"):
@@ -862,10 +831,10 @@ if supports_moe_ops and hasattr(torch.ops._moe_C, "marlin_gemm_moe"):
831 topk_ids: torch.Tensor, b_scales: torch.Tensor,
832 b_zero_points: torch.Tensor, g_idx: torch.Tensor,
833 perm: torch.Tensor, workspace: torch.Tensor,
865 - b_q_type: ScalarType, size_m: int, size_n: int,
866 - size_k: int, is_k_full: bool, num_experts: int,
867 - topk: int, moe_block_size: int,
868 - replicate_input: bool,
834 + b_q_type: ScalarType, size_m: torch.SymInt,
835 + size_n: torch.SymInt, size_k: torch.SymInt,
836 + is_k_full: bool, num_experts: int, topk: int,
837 + moe_block_size: int, replicate_input: bool,
838 apply_weights: bool) -> torch.Tensor:
839 return torch.empty((size_m, topk, size_n),
840 dtype=a.dtype,
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+3 -3
@@ -116,7 +116,7 @@ def single_marlin_moe(
116
117 intermediate_cache = torch.ops._moe_C.marlin_gemm_moe(
118 hidden_states, w, sorted_token_ids, topk_weights, topk_ids, scales,
119 - w_zeros, g_idx, sort_indices, workspace, scalar_type, M, N, K,
119 + w_zeros, g_idx, sort_indices, workspace, scalar_type.id, M, N, K,
120 is_k_full, E, topk, block_size_m, True, False)
121
122 return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
@@ -272,7 +272,7 @@ def fused_marlin_moe(
272 g_idx1,
273 sort_indices1,
274 workspace,
275 - scalar_type1,
275 + scalar_type1.id,
276 M,
277 2 * N,
278 K,
@@ -297,7 +297,7 @@ def fused_marlin_moe(
297 g_idx2,
298 sort_indices2,
299 workspace,
300 - scalar_type2,
300 + scalar_type2.id,
301 M,
302 K,
303 N,
vllm/scalar_type.py
+297 -4
@@ -1,4 +1,298 @@
1 -from ._core_ext import NanRepr, ScalarType
1 +import functools
2 +import struct
3 +from dataclasses import dataclass
4 +from enum import Enum
5 +from typing import Optional, Union
6 +
7 +
8 +# Mirrors enum in `core/scalar_type.hpp`
9 +class NanRepr(Enum):
10 + NONE = 0 # nans are not supported
11 + IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s
12 + EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s
13 +
14 +
15 +# This ScalarType class is a parallel implementation of the C++ ScalarType
16 +# class found in csrc/core/scalar_type.hpp. These two classes should be kept
17 +# in sync until the inductor fully supports custom C++ classes.
18 +@dataclass(frozen=True)
19 +class ScalarType:
20 + """
21 + ScalarType can represent a wide range of floating point and integer
22 + types, in particular it can be used to represent sub-byte data types
23 + (something that torch.dtype currently does not support). It is also
24 + capable of representing types with a bias, i.e.:
25 + `stored_value = value + bias`,
26 + this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias
27 + of 8). The implementation for this class can be found in
28 + csrc/core/scalar_type.hpp, these type signatures should be kept in sync
29 + with that file.
30 + """
31 +
32 + exponent: int
33 + """
34 + Number of bits in the exponent if this is a floating point type
35 + (zero if this an integer type)
36 + """
37 +
38 + mantissa: int
39 + """
40 + Number of bits in the mantissa if this is a floating point type,
41 + or the number bits representing an integer excluding the sign bit if
42 + this an integer type.
43 + """
44 +
45 + signed: bool
46 + "If the type is signed (i.e. has a sign bit)"
47 +
48 + bias: int
49 + """
50 + bias used to encode the values in this scalar type
51 + (value = stored_value - bias, default 0) for example if we store the
52 + type as an unsigned integer with a bias of 128 then the value 0 will be
53 + stored as 128 and -1 will be stored as 127 and 1 will be stored as 129.
54 + """
55 +
56 + _finite_values_only: bool = False
57 + """
58 + Private: if infs are supported, used `has_infs()` instead.
59 + """
60 +
61 + nan_repr: NanRepr = NanRepr.IEEE_754
62 + """
63 + How NaNs are represent in this scalar type, returns NanRepr value.
64 + (not applicable for integer types)
65 + """
66 +
67 + def _floating_point_max_int(self) -> int:
68 + assert (
69 + self.mantissa <= 52 and self.exponent <= 11
70 + ), f"Cannot represent max/min as a double for type {self.__str__()}"
71 +
72 + max_mantissa = (1 << self.mantissa) - 1
73 + if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN:
74 + max_mantissa = max_mantissa - 1
75 +
76 + max_exponent = (1 << self.exponent) - 2
77 + if (self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN
78 + or self.nan_repr == NanRepr.NONE):
79 + assert (
80 + self.exponent < 11
81 + ), f"Cannot represent max/min as a double for type {self.__str__()}"
82 + max_exponent = max_exponent + 1
83 +
84 + # adjust the exponent to match that of a double
85 + # for now we assume the exponent bias is the standard 2^(e-1) -1, (where
86 + # e is the exponent bits), there is some precedent for non-standard
87 + # biases, example `float8_e4m3b11fnuz` here:
88 + # https://github.com/jax-ml/ml_dtypes but to avoid premature over
89 + # complication we are just assuming the standard exponent bias until
90 + # there is a need to support non-standard biases
91 + exponent_bias = (1 << (self.exponent - 1)) - 1
92 + exponent_bias_double = (1 << 10) - 1 # double e = 11
93 +
94 + max_exponent_double = (max_exponent - exponent_bias +
95 + exponent_bias_double)
96 +
97 + # shift the mantissa and exponent into the proper positions for an
98 + # IEEE double and bitwise-or them together.
99 + return (max_mantissa <<
100 + (52 - self.mantissa)) | (max_exponent_double << 52)
101 +
102 + def _floating_point_max(self) -> float:
103 + double_raw = self._floating_point_max_int()
104 + return struct.unpack('!d', struct.pack('!Q', double_raw))[0]
105 +
106 + def _raw_max(self) -> Union[int, float]:
107 + if self.is_floating_point():
108 + return self._floating_point_max()
109 + else:
110 + assert (self.size_bits < 64 or self.size_bits == 64
111 + and self.is_signed()), "Cannot represent max as an int"
112 + return (1 << self.mantissa) - 1
113 +
114 + def _raw_min(self) -> Union[int, float]:
115 + if self.is_floating_point():
116 + assert self.is_signed(
117 + ), "We currently assume all floating point types are signed"
118 + sign_bit_double = 1 << 63
119 +
120 + max_raw = self._floating_point_max_int()
121 + min_raw = max_raw | sign_bit_double
122 + return struct.unpack('!d', struct.pack('!Q', min_raw))[0]
123 + else:
124 + assert (not self.is_signed() or
125 + self.size_bits <= 64), "Cannot represent min as a int64_t"
126 +
127 + if self.is_signed():
128 + return -(1 << (self.size_bits - 1))
129 + else:
130 + return 0
131 +
132 + @functools.cached_property
133 + def id(self) -> int:
134 + """
135 + Convert the ScalarType to an int which can be passed to pytorch custom
136 + ops. This layout of the int must be kept in sync with the C++
137 + ScalarType's from_id method.
138 + """
139 + val = 0
140 + offset = 0
141 +
142 + def or_and_advance(member, bit_width):
143 + nonlocal val
144 + nonlocal offset
145 + bit_mask = (1 << bit_width) - 1
146 + val = val | (int(member) & bit_mask) << offset
147 + offset = offset + bit_width
148 +
149 + or_and_advance(self.exponent, 8)
150 + or_and_advance(self.mantissa, 8)
151 + or_and_advance(self.signed, 1)
152 + or_and_advance(self.bias, 32)
153 + or_and_advance(self._finite_values_only, 1)
154 + or_and_advance(self.nan_repr.value, 8)
155 +
156 + assert offset <= 64, \
157 + f"ScalarType fields too big {offset} to fit into an int64"
158 +
159 + return val
160 +
161 + @property
162 + def size_bits(self) -> int:
163 + return self.exponent + self.mantissa + int(self.signed)
164 +
165 + def min(self) -> Union[int, float]:
166 + """
167 + Min representable value for this scalar type.
168 + (accounting for bias if there is one)
169 + """
170 + return self._raw_min() - self.bias
171 +
172 + def max(self) -> Union[int, float]:
173 + """
174 + Max representable value for this scalar type.
175 + (accounting for bias if there is one)
176 + """
177 + return self._raw_max() - self.bias
178 +
179 + def is_signed(self) -> bool:
180 + """
181 + If the type is signed (i.e. has a sign bit), same as `signed`
182 + added for consistency with:
183 + https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html
184 + """
185 + return self.signed
186 +
187 + def is_floating_point(self) -> bool:
188 + "If the type is a floating point type"
189 + return self.exponent != 0
190 +
191 + def is_integer(self) -> bool:
192 + "If the type is an integer type"
193 + return self.exponent == 0
194 +
195 + def has_bias(self) -> bool:
196 + "If the type has a non-zero bias"
197 + return self.bias != 0
198 +
199 + def has_infs(self) -> bool:
200 + "If the type is floating point and supports infinity"
201 + return not self._finite_values_only
202 +
203 + def has_nans(self) -> bool:
204 + return self.nan_repr != NanRepr.NONE.value
205 +
206 + def is_ieee_754(self) -> bool:
207 + """
208 + If the type is a floating point type that follows IEEE 754
209 + conventions
210 + """
211 + return self.nan_repr == NanRepr.IEEE_754.value and \
212 + not self._finite_values_only
213 +
214 + def __str__(self) -> str:
215 + """
216 + naming generally follows: https://github.com/jax-ml/ml_dtypes
217 + for floating point types (leading f) the scheme is:
218 + `float<size_bits>_e<exponent_bits>m<mantissa_bits>[flags]`
219 + flags:
220 + - no-flags: means it follows IEEE 754 conventions
221 + - f: means finite values only (no infinities)
222 + - n: means nans are supported (non-standard encoding)
223 + for integer types the scheme is:
224 + `[u]int<size_bits>[b<bias>]`
225 + - if bias is not present it means its zero
226 + """
227 + if self.is_floating_point():
228 + ret = "float" + str(self.size_bits) + "_e" + str(
229 + self.exponent) + "m" + str(self.mantissa)
230 +
231 + if not self.is_ieee_754():
232 + if self._finite_values_only:
233 + ret = ret + "f"
234 + if self.nan_repr != NanRepr.NONE:
235 + ret = ret + "n"
236 +
237 + return ret
238 + else:
239 + ret = ("int" if self.is_signed() else "uint") + str(self.size_bits)
240 + if self.has_bias():
241 + ret = ret + "b" + str(self.bias)
242 + return ret
243 +
244 + def __repr__(self) -> str:
245 + return "ScalarType." + self.__str__()
246 +
247 + # __len__ needs to be defined (and has to throw TypeError) for pytorch's
248 + # opcheck to work.
249 + def __len__(self) -> int:
250 + raise TypeError
251 +
252 + #
253 + # Convenience Constructors
254 + #
255 +
256 + @classmethod
257 + def int_(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
258 + "Create a signed integer scalar type (size_bits includes sign-bit)."
259 + ret = cls(0, size_bits - 1, True, bias if bias else 0)
260 + ret.id # noqa B018: make sure the id is cached
261 + return ret
262 +
263 + @classmethod
264 + def uint(cls, size_bits: int, bias: Optional[int]) -> 'ScalarType':
265 + """Create a unsigned integer scalar type."""
266 + ret = cls(0, size_bits, False, bias if bias else 0)
267 + ret.id # noqa B018: make sure the id is cached
268 + return ret
269 +
270 + @classmethod
271 + def float_IEEE754(cls, exponent: int, mantissa: int) -> 'ScalarType':
272 + """
273 + Create a standard floating point type
274 + (i.e. follows IEEE 754 conventions).
275 + """
276 + assert (mantissa > 0 and exponent > 0)
277 + ret = cls(exponent, mantissa, True, 0)
278 + ret.id # noqa B018: make sure the id is cached
279 + return ret
280 +
281 + @classmethod
282 + def float_(cls, exponent: int, mantissa: int, finite_values_only: bool,
283 + nan_repr: NanRepr) -> 'ScalarType':
284 + """
285 + Create a non-standard floating point type
286 + (i.e. does not follow IEEE 754 conventions).
287 + """
288 + assert (mantissa > 0 and exponent > 0)
289 + assert (nan_repr != NanRepr.IEEE_754), (
290 + "use `float_IEEE754` constructor for floating point types that "
291 + "follow IEEE 754 conventions")
292 + ret = cls(exponent, mantissa, True, 0, finite_values_only, nan_repr)
293 + ret.id # noqa B018: make sure the id is cached
294 + return ret
295 +
296
297 # naming generally follows: https://github.com/jax-ml/ml_dtypes
298 # for floating point types (leading f) the scheme is:
@@ -17,14 +311,13 @@ class scalar_types:
311 uint4 = ScalarType.uint(4, None)
312 int8 = ScalarType.int_(8, None)
313 uint8 = ScalarType.uint(8, None)
20 - float8_e4m3fn = ScalarType.float_(4, 3, True,
21 - NanRepr.EXTD_RANGE_MAX_MIN.value)
314 + float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN)
315 float8_e5m2 = ScalarType.float_IEEE754(5, 2)
316 float16_e8m7 = ScalarType.float_IEEE754(8, 7)
317 float16_e5m10 = ScalarType.float_IEEE754(5, 10)
318
319 # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main
27 - float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE.value)
320 + float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE)
321
322 # "gptq" types
323 uint2b2 = ScalarType.uint(2, 2)