A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] fix for deepseek w4a16 (#8906)
Co-authored-by: mgoin <michael@neuralmagic.com>
Lucas Wilkinson committed
Sep 27, 2024 at 15:12 UTC
c5d55356f9d2b2075ac53cf20453358c1e2b7bde
1 file changed
+5
-4
vllm/model_executor/layers/quantization/kernels/marlin.py
+5
-4
@@ -38,10 +38,11 @@ class MarlinLinearKernel(MPLinearKernel):
38
"Marlin, supported group sizes are: "\
39
f"{MARLIN_SUPPORTED_GROUP_SIZES}"
40
41
- return check_marlin_supports_shape(c.partition_weight_shape[0],
42
- c.partition_weight_shape[1],
43
- c.full_weight_shape[1],
44
- c.group_size)
41
+ return check_marlin_supports_shape(
42
+ c.partition_weight_shape[1], # out_features
43
+ c.partition_weight_shape[0], # in_features
44
+ c.full_weight_shape[0], # in_features
45
+ c.group_size)
46
47
# note assumes that
48
# `weight_packed` is: {input_dim = 0, output_dim = 1, packed_dim = 0}