A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Support tensor parallelism for GGUF quantization (#7520)
Isotr0py committed
Aug 20, 2024 at 05:30 UTC
7601cb044ddfe920055f82ae9503729d4dde7259
3 files changed
+39
-15
tests/models/test_gguf.py
+19
-5
@@ -7,6 +7,7 @@ import os
7
8
import pytest
9
from huggingface_hub import hf_hub_download
10
+from transformers import AutoTokenizer
11
12
from tests.quantization.utils import is_quant_method_supported
13
@@ -20,7 +21,7 @@ MAX_MODEL_LEN = 1024
21
MODELS = [
22
("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
23
hf_hub_download("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
23
- filename="tinyllama-1.1b-chat-v1.0.Q4_0.gguf")),
24
+ filename="tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf")),
25
("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
26
hf_hub_download("duyntnet/TinyLlama-1.1B-Chat-v1.0-imatrix-GGUF",
27
filename="TinyLlama-1.1B-Chat-v1.0-IQ4_XS.gguf")),
@@ -39,22 +40,36 @@ MODELS = [
40
@pytest.mark.parametrize("dtype", ["half"])
41
@pytest.mark.parametrize("max_tokens", [32])
42
@pytest.mark.parametrize("num_logprobs", [5])
43
+@pytest.mark.parametrize("tp_size", [1, 2])
44
def test_models(
45
+ num_gpus_available,
46
vllm_runner,
47
example_prompts,
48
model,
49
dtype: str,
50
max_tokens: int,
51
num_logprobs: int,
52
+ tp_size: int,
53
) -> None:
54
+ if num_gpus_available < tp_size:
55
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
56
+
57
original_model, gguf_model = model
58
59
+ tokenizer = AutoTokenizer.from_pretrained(original_model)
60
+ messages = [[{
61
+ 'role': 'user',
62
+ 'content': prompt
63
+ }] for prompt in example_prompts]
64
+ example_prompts = tokenizer.apply_chat_template(messages,
65
+ tokenize=False,
66
+ add_generation_prompt=True)
67
+
68
# Run unquantized model.
69
with vllm_runner(model_name=original_model,
70
dtype=dtype,
71
max_model_len=MAX_MODEL_LEN,
56
- enforce_eager=True,
57
- tensor_parallel_size=1) as original_model:
72
+ tensor_parallel_size=tp_size) as original_model:
73
74
original_outputs = original_model.generate_greedy_logprobs(
75
example_prompts[:-1], max_tokens, num_logprobs)
@@ -63,8 +78,7 @@ def test_models(
78
with vllm_runner(model_name=gguf_model,
79
dtype=dtype,
80
max_model_len=MAX_MODEL_LEN,
66
- enforce_eager=True,
67
- tensor_parallel_size=1) as gguf_model:
81
+ tensor_parallel_size=tp_size) as gguf_model:
82
gguf_outputs = gguf_model.generate_greedy_logprobs(
83
example_prompts[:-1], max_tokens, num_logprobs)
84
vllm/model_executor/layers/linear.py
+20
-6
@@ -507,11 +507,16 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
507
loaded_shard_id
508
509
if is_gguf_weight:
510
- shard_size = loaded_weight.shape[output_dim]
511
- shard_offset = loaded_weight.shape[output_dim] * \
512
- loaded_shard_id
510
+ tp_size = get_tensor_model_parallel_world_size()
511
+ output_dim = getattr(param, "output_dim", None)
512
+ shard_shape = list(loaded_weight.shape)
513
+ shard_shape[output_dim] = shard_shape[output_dim] // tp_size
514
param.shard_id.append(loaded_shard_id)
514
- param.shard_size[loaded_shard_id] = loaded_weight.shape
515
+ param.shard_size[loaded_shard_id] = shard_shape
516
+
517
+ input_dim = getattr(param, "input_dim", None)
518
+ input_size = loaded_weight.shape[input_dim]
519
+ param_data = param_data.narrow(input_dim, 0, input_size)
520
521
param_data = param_data.narrow(output_dim, shard_offset,
522
shard_size)
@@ -863,8 +868,13 @@ class QKVParallelLinear(ColumnParallelLinear):
868
param, orig_qkv_offsets, loaded_shard_id)
869
870
if is_gguf_weight:
871
+ tp_size = get_tensor_model_parallel_world_size()
872
+ output_dim = getattr(param, "output_dim", None)
873
+ shard_shape = list(loaded_weight.shape)
874
+ shard_shape[output_dim] = shard_shape[output_dim] // tp_size
875
param.shard_id.append(loaded_shard_id)
867
- param.shard_size[loaded_shard_id] = loaded_weight.shape
876
+ param.shard_size[loaded_shard_id] = shard_shape
877
+
878
input_dim = getattr(param, "input_dim", None)
879
input_size = loaded_weight.shape[input_dim]
880
param_data = param_data.narrow(input_dim, 0, input_size)
@@ -976,6 +986,7 @@ class RowParallelLinear(LinearBase):
986
987
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
988
tp_rank = get_tensor_model_parallel_rank()
989
+ tp_size = get_tensor_model_parallel_world_size()
990
input_dim = getattr(param, "input_dim", None)
991
992
# Special case for GGUF
@@ -986,7 +997,10 @@ class RowParallelLinear(LinearBase):
997
998
# Materialize GGUF UninitializedParameter
999
if is_gguf_weight and isinstance(param, UninitializedParameter):
989
- param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
1000
+ weight_shape = list(loaded_weight.shape)
1001
+ if input_dim:
1002
+ weight_shape[input_dim] = weight_shape[input_dim] // tp_size
1003
+ param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
1004
1005
param_data = param.data
1006
if input_dim is not None:
vllm/model_executor/layers/quantization/gguf.py
-4
@@ -5,7 +5,6 @@ import torch
5
from torch.nn.parameter import Parameter, UninitializedParameter
6
7
from vllm import _custom_ops as ops
8
-from vllm.distributed import get_tensor_model_parallel_world_size
8
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9
from vllm.model_executor.layers.quantization.base_config import (
10
QuantizationConfig, QuantizeMethodBase)
@@ -39,9 +38,6 @@ class GGUFConfig(QuantizationConfig):
38
39
@classmethod
40
def from_config(cls, config: Dict[str, Any]) -> "GGUFConfig":
42
- if get_tensor_model_parallel_world_size() > 1:
43
- raise ValueError(
44
- "GGUF quantization hasn't supported tensor parallelism yet.")
41
return cls()
42
43
def get_quant_method(self, layer: torch.nn.Module,