A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[BUG] fixed fp8 conflict with aqlm (#4307)
Fixes fp8 iterface which broke in AQLM merge.
Robert Shaw committed
Apr 23, 2024 at 21:26 UTC
79a268c4ab2cbf44280eebd998b8efc383bac216
3 files changed
+18
-4
.buildkite/test-pipeline.yaml
+3
@@ -96,6 +96,9 @@ steps:
96
- label: Metrics Test
97
command: pytest -v -s metrics
98
99
+- label: Quantization Test
100
+ command: pytest -v -s quantization
101
+
102
- label: Benchmarks
103
working_dir: "/vllm-workspace/.buildkite"
104
commands:
vllm/model_executor/layers/linear.py
+13
-3
@@ -34,9 +34,19 @@ class LinearMethodBase(ABC):
34
output_partition_sizes: List[int], input_size: int,
35
output_size: int, params_dtype: torch.dtype,
36
**extra_weight_attrs):
37
- """Create weights for a linear layer.
38
-
39
- The weights will be set as attributes of the layer."""
37
+ """Create weights for a linear layer.
38
+ The weights will be set as attributes of the layer.
39
+
40
+ Args:
41
+ layer: The layer that is using the LinearMethodBase factory.
42
+ input_size_per_partition: Size of the weight input dim on rank X.
43
+ output_partition_sizes: Sizes of the output dim of each logical
44
+ weight on rank X. E.g., output_partition_sizes for QKVLinear
45
+ is a list contains the width of Wq, Wk, Wv on rank X.
46
+ input_size: Size of the input dim of the weight across all ranks.
47
+ output_size: Size of the output dim of the weight across all ranks.
48
+ params_dtype: Datatype of the parameters.
49
+ """
50
raise NotImplementedError
51
52
@abstractmethod
vllm/model_executor/layers/quantization/fp8.py
+2
-1
@@ -64,12 +64,13 @@ class Fp8LinearMethod(LinearMethodBase):
64
self,
65
layer: torch.nn.Module,
66
input_size_per_partition: int,
67
- output_size_per_partition: int,
67
+ output_partition_sizes: List[int],
68
input_size: int,
69
output_size: int,
70
params_dtype: torch.dtype,
71
**extra_weight_attrs,
72
):
73
+ output_size_per_partition = sum(output_partition_sizes)
74
weight = Parameter(torch.empty(output_size_per_partition,
75
input_size_per_partition,
76
dtype=params_dtype),