@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5bf185a1c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Bugfix] gptq_marlin: Ensure g_idx_sort_indices is not a Parameter (#5108)

Alexander Matveev committed May 29, 2024 at 20:30 UTC 5bf185a1c48fdca524dd76aec4a1424b3a09c9a1
1 file changed +4 -8
vllm/model_executor/layers/quantization/gptq_marlin.py
+4 -8
@@ -298,14 +298,10 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
298 },
299 )
300
301 - g_idx_sort_indices = Parameter(
302 - torch.empty(
303 - g_idx.shape,
304 - dtype=torch.int32,
305 - ),
306 - requires_grad=False,
301 + g_idx_sort_indices = torch.empty(
302 + g_idx.shape,
303 + dtype=torch.int32,
304 )
308 - set_weight_attrs(g_idx_sort_indices, extra_weight_attrs)
305
306 # Scales
307 scales = Parameter(
@@ -356,9 +352,9 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
352
353 layer.register_parameter("qweight", qweight)
354 layer.register_parameter("g_idx", g_idx)
359 - layer.register_parameter("g_idx_sort_indices", g_idx_sort_indices)
355 layer.register_parameter("scales", scales)
356 layer.register_parameter("qzeros", qzeros)
357 + layer.g_idx_sort_indices = g_idx_sort_indices
358 layer.workspace = workspace
359 layer.input_size_per_partition = input_size_per_partition
360 layer.output_size_per_partition = output_size_per_partition