@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 79a268c4ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[BUG] fixed fp8 conflict with aqlm (#4307)

Fixes fp8 iterface which broke in AQLM merge.

Robert Shaw committed Apr 23, 2024 at 21:26 UTC 79a268c4ab2cbf44280eebd998b8efc383bac216
3 files changed +18 -4
.buildkite/test-pipeline.yaml
+3
@@ -96,6 +96,9 @@ steps:
96 - label: Metrics Test
97 command: pytest -v -s metrics
98
99 +- label: Quantization Test
100 + command: pytest -v -s quantization
101 +
102 - label: Benchmarks
103 working_dir: "/vllm-workspace/.buildkite"
104 commands:
vllm/model_executor/layers/linear.py
+13 -3
@@ -34,9 +34,19 @@ class LinearMethodBase(ABC):
34 output_partition_sizes: List[int], input_size: int,
35 output_size: int, params_dtype: torch.dtype,
36 **extra_weight_attrs):
37 - """Create weights for a linear layer.
38 -
39 - The weights will be set as attributes of the layer."""
37 + """Create weights for a linear layer.
38 + The weights will be set as attributes of the layer.
39 +
40 + Args:
41 + layer: The layer that is using the LinearMethodBase factory.
42 + input_size_per_partition: Size of the weight input dim on rank X.
43 + output_partition_sizes: Sizes of the output dim of each logical
44 + weight on rank X. E.g., output_partition_sizes for QKVLinear
45 + is a list contains the width of Wq, Wk, Wv on rank X.
46 + input_size: Size of the input dim of the weight across all ranks.
47 + output_size: Size of the output dim of the weight across all ranks.
48 + params_dtype: Datatype of the parameters.
49 + """
50 raise NotImplementedError
51
52 @abstractmethod
vllm/model_executor/layers/quantization/fp8.py
+2 -1
@@ -64,12 +64,13 @@ class Fp8LinearMethod(LinearMethodBase):
64 self,
65 layer: torch.nn.Module,
66 input_size_per_partition: int,
67 - output_size_per_partition: int,
67 + output_partition_sizes: List[int],
68 input_size: int,
69 output_size: int,
70 params_dtype: torch.dtype,
71 **extra_weight_attrs,
72 ):
73 + output_size_per_partition = sum(output_partition_sizes)
74 weight = Parameter(torch.empty(output_size_per_partition,
75 input_size_per_partition,
76 dtype=params_dtype),