A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 12:45 UTC
[Chore] Cleanup BNB utilization dead code (#35620)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 1, 2026 at 03:22 UTC
e94b263bd6557dc54582bfc5ba74f0a631bd642d
1 file changed
-39
vllm/model_executor/layers/linear.py
-39
@@ -3,7 +3,6 @@
3
4
import itertools
5
from abc import abstractmethod
6
-from typing import Any
6
7
import torch
8
from torch.nn.parameter import Parameter, UninitializedParameter
@@ -133,44 +132,6 @@ def adjust_scalar_to_fused_array(
132
return param_data[shard_id], loaded_weight
133
134
136
-# TODO(Isotr0py): We might need a more flexible structure to handle
137
-# bitsandbytes shard offsets.
138
-def left_shift_bitsandbytes_4bit_shard(
139
- bnb_weight_attrs: dict[str, Any],
140
-) -> tuple[dict[str, Any], dict[str, Any]]:
141
- """
142
- Separate the BitsAndBytes 4-bit shard.
143
-
144
- For example, given bnb weight attributes as below:
145
- {
146
- 'bnb_shard_offsets': array([0, 4, 8, 16]),
147
- 'bnb_quant_state': {0: ..., 1: ..., 2: ...},
148
- }
149
-
150
- The function will return:
151
- {
152
- 'bnb_shard_offsets': array([0, 4]),
153
- 'bnb_quant_state': {0: ...},
154
- }
155
- and
156
- {
157
- 'bnb_shard_offsets': array([0, 4, 12]),
158
- 'bnb_quant_state': {0: ..., 1: ...},
159
- }
160
- """
161
- shard_offsets = bnb_weight_attrs["bnb_shard_offsets"]
162
- offset_l = shard_offsets[:2]
163
- offset_r = shard_offsets[1:] - shard_offsets[1]
164
- quant_state_l = {0: bnb_weight_attrs["bnb_quant_state"][0]}
165
- quant_state_r = {
166
- i - 1: bnb_weight_attrs["bnb_quant_state"][i]
167
- for i in range(1, len(shard_offsets) - 1)
168
- }
169
- left = dict(bnb_shard_offsets=offset_l, bnb_quant_state=quant_state_l)
170
- right = dict(bnb_shard_offsets=offset_r, bnb_quant_state=quant_state_r)
171
- return left, right
172
-
173
-
135
class LinearMethodBase(QuantizeMethodBase):
136
"""Base class for different (maybe quantized) linear methods."""
137