@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5fe6bf29d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[BugFix] Fix GGUF tp>1 when vocab_size is not divisible by 64 (#12230)

Signed-off-by: NickLucche <nlucches@redhat.com>

Nicolò Lucchesi committed Jan 21, 2025 at 05:23 UTC 5fe6bf29d657518eb4251981ada9f8c4f34dbbde
2 files changed +12 -2
tests/models/decoder_only/language/test_gguf.py
+10
@@ -66,12 +66,20 @@ STARCODER_CONFIG = GGUFTestConfig(
66 gguf_filename="starcoder2-3b.Q6_K.gguf",
67 )
68
69 +DOLPHIN_CONFIG = GGUFTestConfig(
70 + # Test VocabParallelEmbedding sharding issue.
71 + original_model="cognitivecomputations/TinyDolphin-2.8-1.1b",
72 + gguf_repo="tsunemoto/TinyDolphin-2.8-1.1b-GGUF",
73 + gguf_filename="tinydolphin-2.8-1.1b.Q6_K.gguf",
74 +)
75 +
76 MODELS = [
77 LLAMA_CONFIG,
78 QWEN2_CONFIG,
79 PHI3_CONFIG,
80 GPT2_CONFIG,
81 STABLELM_CONFIG,
82 + DOLPHIN_CONFIG
83 # STARCODER_CONFIG, # broken
84 ]
85
@@ -107,6 +115,7 @@ def test_models(
115
116 # Run unquantized model.
117 with vllm_runner(model_name=model.original_model,
118 + enforce_eager=True, # faster tests
119 dtype=dtype,
120 max_model_len=MAX_MODEL_LEN,
121 tensor_parallel_size=tp_size) as original_model:
@@ -115,6 +124,7 @@ def test_models(
124
125 # Run gguf model.
126 with vllm_runner(model_name=model.gguf_model,
127 + enforce_eager=True,
128 tokenizer_name=model.original_model,
129 dtype=dtype,
130 max_model_len=MAX_MODEL_LEN,
vllm/model_executor/layers/vocab_parallel_embedding.py
+2 -2
@@ -355,7 +355,7 @@ class VocabParallelEmbedding(torch.nn.Module):
355 elif isinstance(param, UninitializedParameter):
356 shape = list(loaded_weight.shape)
357 if output_dim is not None:
358 - shape[output_dim] = shape[output_dim] // self.tp_size
358 + shape[output_dim] = self.num_embeddings_per_partition
359 param.materialize(tuple(shape), dtype=loaded_weight.dtype)
360
361 # If parameter does not have output dim, then it should
@@ -381,7 +381,7 @@ class VocabParallelEmbedding(torch.nn.Module):
381 else:
382 assert loaded_weight.shape[output_dim] == self.org_vocab_size
383
384 - # Copy the data.
384 + # Copy the data. Select chunk corresponding to current shard.
385 loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size)
386
387 if current_platform.is_hpu():