@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ebede26ebf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Make InternLM follow `rope_scaling` in `config.json` (#1956)

Co-authored-by: lijie8 <lijie8@sensetime.com>

Jie Li committed Dec 8, 2023 at 00:32 UTC ebede26ebf9c142465e1bfa8930fc8d2cbf5d953
1 file changed +4 -1
vllm/model_executor/models/internlm.py
+4 -1
@@ -1,5 +1,5 @@
1 # -*- coding: utf-8 -*-
2 -from typing import List, Optional, Tuple
2 +from typing import Any, Dict, List, Optional, Tuple
3
4 import torch
5 from torch import nn
@@ -67,6 +67,7 @@ class InternLMAttention(nn.Module):
67 rope_theta: float = 10000,
68 max_position_embeddings: int = 8192,
69 linear_method: Optional[LinearMethodBase] = None,
70 + rope_scaling: Optional[Dict[str, Any]] = None,
71 ):
72 super().__init__()
73 self.hidden_size = hidden_size
@@ -99,6 +100,7 @@ class InternLMAttention(nn.Module):
100 rotary_dim=self.head_dim,
101 max_position=self.max_position_embeddings,
102 base=self.rope_theta,
103 + rope_scaling=rope_scaling,
104 )
105 self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106
@@ -139,6 +141,7 @@ class InternLMDecoderLayer(nn.Module):
141 rope_theta=rope_theta,
142 max_position_embeddings=max_position_embeddings,
143 linear_method=linear_method,
144 + rope_scaling=getattr(config, "rope_scaling", None),
145 )
146 self.mlp = InternLMMLP(
147 hidden_size=self.hidden_size,