@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dbed69058c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Fix the `KeyError` when loading bloom-based models (#441)

Wen Sun committed Jul 14, 2023 at 12:58 UTC dbed69058c88ddf42914e6ab3a9b6ea12e15b12a
1 file changed +10 -3
vllm/model_executor/models/bloom.py
+10 -3
@@ -284,10 +284,17 @@ class BloomForCausalLM(nn.Module):
284 state_dict = self.state_dict()
285 for name, loaded_weight in hf_model_weights_iterator(
286 model_name_or_path, cache_dir, use_np_cache):
287 - if not name.startswith("transformer."):
288 - name = "transformer." + name
287 + if name == "lm_head.weight":
288 + # Since hidden_states are parallelized, we need to
289 + # load lm_head.weight in parallel.
290 + self._column_parallel_weights.append(name)
291 + # If lm_head is provided, use it instead.
292 + param = self.lm_head_weight
293 + else:
294 + if not name.startswith("transformer."):
295 + name = "transformer." + name
296 + param = state_dict[name]
297
290 - param = state_dict[name]
298 if "query_key_value" in name:
299 # NOTE(woosuk): BLOOM's fused QKV has the shape of
300 # [num_heads * 3 * head_size, hidden_size], while the