@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5d55356f9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] fix for deepseek w4a16 (#8906)

Co-authored-by: mgoin <michael@neuralmagic.com>

Lucas Wilkinson committed Sep 27, 2024 at 15:12 UTC c5d55356f9d2b2075ac53cf20453358c1e2b7bde
1 file changed +5 -4
vllm/model_executor/layers/quantization/kernels/marlin.py
+5 -4
@@ -38,10 +38,11 @@ class MarlinLinearKernel(MPLinearKernel):
38 "Marlin, supported group sizes are: "\
39 f"{MARLIN_SUPPORTED_GROUP_SIZES}"
40
41 - return check_marlin_supports_shape(c.partition_weight_shape[0],
42 - c.partition_weight_shape[1],
43 - c.full_weight_shape[1],
44 - c.group_size)
41 + return check_marlin_supports_shape(
42 + c.partition_weight_shape[1], # out_features
43 + c.partition_weight_shape[0], # in_features
44 + c.full_weight_shape[0], # in_features
45 + c.group_size)
46
47 # note assumes that
48 # `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}