A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Bugfix] gptq_marlin: Ensure g_idx_sort_indices is not a Parameter (#5108)
Alexander Matveev committed
May 29, 2024 at 20:30 UTC
5bf185a1c48fdca524dd76aec4a1424b3a09c9a1
1 file changed
+4
-8
vllm/model_executor/layers/quantization/gptq_marlin.py
+4
-8
@@ -298,14 +298,10 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
298
},
299
)
300
301
- g_idx_sort_indices = Parameter(
302
- torch.empty(
303
- g_idx.shape,
304
- dtype=torch.int32,
305
- ),
306
- requires_grad=False,
301
+ g_idx_sort_indices = torch.empty(
302
+ g_idx.shape,
303
+ dtype=torch.int32,
304
)
308
- set_weight_attrs(g_idx_sort_indices, extra_weight_attrs)
305
306
# Scales
307
scales = Parameter(
@@ -356,9 +352,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
352
353
layer.register_parameter("qweight", qweight)
354
layer.register_parameter("g_idx", g_idx)
359
- layer.register_parameter("g_idx_sort_indices", g_idx_sort_indices)
355
layer.register_parameter("scales", scales)
356
layer.register_parameter("qzeros", qzeros)
357
+ layer.g_idx_sort_indices = g_idx_sort_indices
358
layer.workspace = workspace
359
layer.input_size_per_partition = input_size_per_partition
360
layer.output_size_per_partition = output_size_per_partition