A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Convert before transpose (#1073)
Woosuk Kwon committed
Sep 18, 2023 at 11:51 UTC
cc796b13584419afc741d747c02cb715adb9c019
1 file changed
+3
-2
vllm/model_executor/models/llama.py
+3
-2
@@ -43,8 +43,8 @@ from vllm.model_executor.parallel_utils.tensor_parallel import (
43
VocabParallelEmbedding)
44
from vllm.model_executor.quantization_utils import QuantizationConfig
45
from vllm.model_executor.weight_utils import (
46
- load_tensor_parallel_weights, load_padded_tensor_parallel_vocab,
47
- hf_model_weights_iterator)
46
+ convert_pyslice_to_tensor, hf_model_weights_iterator,
47
+ load_tensor_parallel_weights, load_padded_tensor_parallel_vocab)
48
from vllm.sequence import SamplerOutput
49
50
KVCache = Tuple[torch.Tensor, torch.Tensor]
@@ -337,6 +337,7 @@ class LlamaForCausalLM(nn.Module):
337
is_packed = self.quant_config.is_packed(name)
338
is_transposed = self.quant_config.is_transposed(name)
339
if is_transposed:
340
+ loaded_weight = convert_pyslice_to_tensor(loaded_weight)
341
loaded_weight = loaded_weight.T
342
343
is_attention_weight = False