A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Make InternLM follow `rope_scaling` in `config.json` (#1956)
Co-authored-by: lijie8 <lijie8@sensetime.com>
Jie Li committed
Dec 8, 2023 at 00:32 UTC
ebede26ebf9c142465e1bfa8930fc8d2cbf5d953
1 file changed
+4
-1
vllm/model_executor/models/internlm.py
+4
-1
@@ -1,5 +1,5 @@
1
# -*- coding: utf-8 -*-
2
-from typing import List, Optional, Tuple
2
+from typing import Any, Dict, List, Optional, Tuple
3
4
import torch
5
from torch import nn
@@ -67,6 +67,7 @@ class InternLMAttention(nn.Module):
67
rope_theta: float = 10000,
68
max_position_embeddings: int = 8192,
69
linear_method: Optional[LinearMethodBase] = None,
70
+ rope_scaling: Optional[Dict[str, Any]] = None,
71
):
72
super().__init__()
73
self.hidden_size = hidden_size
@@ -99,6 +100,7 @@ class InternLMAttention(nn.Module):
100
rotary_dim=self.head_dim,
101
max_position=self.max_position_embeddings,
102
base=self.rope_theta,
103
+ rope_scaling=rope_scaling,
104
)
105
self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106
@@ -139,6 +141,7 @@ class InternLMDecoderLayer(nn.Module):
141
rope_theta=rope_theta,
142
max_position_embeddings=max_position_embeddings,
143
linear_method=linear_method,
144
+ rope_scaling=getattr(config, "rope_scaling", None),
145
)
146
self.mlp = InternLMMLP(
147
hidden_size=self.hidden_size,