@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7601cb044d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Core] Support tensor parallelism for GGUF quantization (#7520)

Isotr0py committed Aug 20, 2024 at 05:30 UTC 7601cb044ddfe920055f82ae9503729d4dde7259
3 files changed +39 -15
tests/models/test_gguf.py
+19 -5
@@ -7,6 +7,7 @@ import os
7
8 import pytest
9 from huggingface_hub import hf_hub_download
10 +from transformers import AutoTokenizer
11
12 from tests.quantization.utils import is_quant_method_supported
13
@@ -20,7 +21,7 @@ MAX_MODEL_LEN = 1024
21 MODELS = [
22 ("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
23 hf_hub_download("TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF",
23 - filename="tinyllama-1.1b-chat-v1.0.Q4_0.gguf")),
24 + filename="tinyllama-1.1b-chat-v1.0.Q4_K_M.gguf")),
25 ("TinyLlama/TinyLlama-1.1B-Chat-v1.0",
26 hf_hub_download("duyntnet/TinyLlama-1.1B-Chat-v1.0-imatrix-GGUF",
27 filename="TinyLlama-1.1B-Chat-v1.0-IQ4_XS.gguf")),
@@ -39,22 +40,36 @@ MODELS = [
40 @pytest.mark.parametrize("dtype", ["half"])
41 @pytest.mark.parametrize("max_tokens", [32])
42 @pytest.mark.parametrize("num_logprobs", [5])
43 +@pytest.mark.parametrize("tp_size", [1, 2])
44 def test_models(
45 + num_gpus_available,
46 vllm_runner,
47 example_prompts,
48 model,
49 dtype: str,
50 max_tokens: int,
51 num_logprobs: int,
52 + tp_size: int,
53 ) -> None:
54 + if num_gpus_available < tp_size:
55 + pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
56 +
57 original_model, gguf_model = model
58
59 + tokenizer = AutoTokenizer.from_pretrained(original_model)
60 + messages = [[{
61 + 'role': 'user',
62 + 'content': prompt
63 + }] for prompt in example_prompts]
64 + example_prompts = tokenizer.apply_chat_template(messages,
65 + tokenize=False,
66 + add_generation_prompt=True)
67 +
68 # Run unquantized model.
69 with vllm_runner(model_name=original_model,
70 dtype=dtype,
71 max_model_len=MAX_MODEL_LEN,
56 - enforce_eager=True,
57 - tensor_parallel_size=1) as original_model:
72 + tensor_parallel_size=tp_size) as original_model:
73
74 original_outputs = original_model.generate_greedy_logprobs(
75 example_prompts[:-1], max_tokens, num_logprobs)
@@ -63,8 +78,7 @@ def test_models(
78 with vllm_runner(model_name=gguf_model,
79 dtype=dtype,
80 max_model_len=MAX_MODEL_LEN,
66 - enforce_eager=True,
67 - tensor_parallel_size=1) as gguf_model:
81 + tensor_parallel_size=tp_size) as gguf_model:
82 gguf_outputs = gguf_model.generate_greedy_logprobs(
83 example_prompts[:-1], max_tokens, num_logprobs)
84
vllm/model_executor/layers/linear.py
+20 -6
@@ -507,11 +507,16 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
507 loaded_shard_id
508
509 if is_gguf_weight:
510 - shard_size = loaded_weight.shape[output_dim]
511 - shard_offset = loaded_weight.shape[output_dim] * \
512 - loaded_shard_id
510 + tp_size = get_tensor_model_parallel_world_size()
511 + output_dim = getattr(param, "output_dim", None)
512 + shard_shape = list(loaded_weight.shape)
513 + shard_shape[output_dim] = shard_shape[output_dim] // tp_size
514 param.shard_id.append(loaded_shard_id)
514 - param.shard_size[loaded_shard_id] = loaded_weight.shape
515 + param.shard_size[loaded_shard_id] = shard_shape
516 +
517 + input_dim = getattr(param, "input_dim", None)
518 + input_size = loaded_weight.shape[input_dim]
519 + param_data = param_data.narrow(input_dim, 0, input_size)
520
521 param_data = param_data.narrow(output_dim, shard_offset,
522 shard_size)
@@ -863,8 +868,13 @@ class QKVParallelLinear(ColumnParallelLinear):
868 param, orig_qkv_offsets, loaded_shard_id)
869
870 if is_gguf_weight:
871 + tp_size = get_tensor_model_parallel_world_size()
872 + output_dim = getattr(param, "output_dim", None)
873 + shard_shape = list(loaded_weight.shape)
874 + shard_shape[output_dim] = shard_shape[output_dim] // tp_size
875 param.shard_id.append(loaded_shard_id)
867 - param.shard_size[loaded_shard_id] = loaded_weight.shape
876 + param.shard_size[loaded_shard_id] = shard_shape
877 +
878 input_dim = getattr(param, "input_dim", None)
879 input_size = loaded_weight.shape[input_dim]
880 param_data = param_data.narrow(input_dim, 0, input_size)
@@ -976,6 +986,7 @@ class RowParallelLinear(LinearBase):
986
987 def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
988 tp_rank = get_tensor_model_parallel_rank()
989 + tp_size = get_tensor_model_parallel_world_size()
990 input_dim = getattr(param, "input_dim", None)
991
992 # Special case for GGUF
@@ -986,7 +997,10 @@ class RowParallelLinear(LinearBase):
997
998 # Materialize GGUF UninitializedParameter
999 if is_gguf_weight and isinstance(param, UninitializedParameter):
989 - param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
1000 + weight_shape = list(loaded_weight.shape)
1001 + if input_dim:
1002 + weight_shape[input_dim] = weight_shape[input_dim] // tp_size
1003 + param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype)
1004
1005 param_data = param.data
1006 if input_dim is not None:
vllm/model_executor/layers/quantization/gguf.py
-4
@@ -5,7 +5,6 @@ import torch
5 from torch.nn.parameter import Parameter, UninitializedParameter
6
7 from vllm import _custom_ops as ops
8 -from vllm.distributed import get_tensor_model_parallel_world_size
8 from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
9 from vllm.model_executor.layers.quantization.base_config import (
10 QuantizationConfig, QuantizeMethodBase)
@@ -39,9 +38,6 @@ class GGUFConfig(QuantizationConfig):
38
39 @classmethod
40 def from_config(cls, config: Dict[str, Any]) -> "GGUFConfig":
42 - if get_tensor_model_parallel_world_size() > 1:
43 - raise ValueError(
44 - "GGUF quantization hasn't supported tensor parallelism yet.")
41 return cls()
42
43 def get_quant_method(self, layer: torch.nn.Module,