A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[BugFix] Fix GGUF tp>1 when vocab_size is not divisible by 64 (#12230)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Jan 21, 2025 at 05:23 UTC
5fe6bf29d657518eb4251981ada9f8c4f34dbbde
2 files changed
+12
-2
tests/models/decoder_only/language/test_gguf.py
+10
@@ -66,12 +66,20 @@ STARCODER_CONFIG = GGUFTestConfig(
66
gguf_filename="starcoder2-3b.Q6_K.gguf",
67
)
68
69
+DOLPHIN_CONFIG = GGUFTestConfig(
70
+ # Test VocabParallelEmbedding sharding issue.
71
+ original_model="cognitivecomputations/TinyDolphin-2.8-1.1b",
72
+ gguf_repo="tsunemoto/TinyDolphin-2.8-1.1b-GGUF",
73
+ gguf_filename="tinydolphin-2.8-1.1b.Q6_K.gguf",
74
+)
75
+
76
MODELS = [
77
LLAMA_CONFIG,
78
QWEN2_CONFIG,
79
PHI3_CONFIG,
80
GPT2_CONFIG,
81
STABLELM_CONFIG,
82
+ DOLPHIN_CONFIG
83
# STARCODER_CONFIG, # broken
84
]
85
@@ -107,6 +115,7 @@ def test_models(
115
116
# Run unquantized model.
117
with vllm_runner(model_name=model.original_model,
118
+ enforce_eager=True, # faster tests
119
dtype=dtype,
120
max_model_len=MAX_MODEL_LEN,
121
tensor_parallel_size=tp_size) as original_model:
@@ -115,6 +124,7 @@ def test_models(
124
125
# Run gguf model.
126
with vllm_runner(model_name=model.gguf_model,
127
+ enforce_eager=True,
128
tokenizer_name=model.original_model,
129
dtype=dtype,
130
max_model_len=MAX_MODEL_LEN,
vllm/model_executor/layers/vocab_parallel_embedding.py
+2
-2
@@ -355,7 +355,7 @@ class VocabParallelEmbedding(torch.nn.Module):
355
elif isinstance(param, UninitializedParameter):
356
shape = list(loaded_weight.shape)
357
if output_dim is not None:
358
- shape[output_dim] = shape[output_dim] // self.tp_size
358
+ shape[output_dim] = self.num_embeddings_per_partition
359
param.materialize(tuple(shape), dtype=loaded_weight.dtype)
360
361
# If parameter does not have output dim, then it should
@@ -381,7 +381,7 @@ class VocabParallelEmbedding(torch.nn.Module):
381
else:
382
assert loaded_weight.shape[output_dim] == self.org_vocab_size
383
384
- # Copy the data.
384
+ # Copy the data. Select chunk corresponding to current shard.
385
loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
386
387
if current_platform.is_hpu():