@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc796b1358
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Convert before transpose (#1073)

Woosuk Kwon committed Sep 18, 2023 at 11:51 UTC cc796b13584419afc741d747c02cb715adb9c019
1 file changed +3 -2
vllm/model_executor/models/llama.py
+3 -2
@@ -43,8 +43,8 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
43 VocabParallelEmbedding)
44 from vllm.model_executor.quantization_utils import QuantizationConfig
45 from vllm.model_executor.weight_utils import (
46 - load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
47 - hf_model_weights_iterator)
46 + convert_pyslice_to_tensor, hf_model_weights_iterator,
47 + load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
48 from vllm.sequence import SamplerOutput
49
50 KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -337,6 +337,7 @@ class LlamaForCausalLM(nn.Module):
337 is_packed = self.quant_config.is_packed(name)
338 is_transposed = self.quant_config.is_transposed(name)
339 if is_transposed:
340 + loaded_weight = convert_pyslice_to_tensor(loaded_weight)
341 loaded_weight = loaded_weight.T
342
343 is_attention_weight = False