A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model] Add cogagent model support vLLM (#11742)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
sixgod committed
Jan 12, 2025 at 03:05 UTC
4b657d32922cb6d3179f901e312715279ad9c728
1 file changed
+7
-5
vllm/model_executor/models/chatglm.py
+7
-5
@@ -1,6 +1,6 @@
1
# Adapted from
2
-# https://github.com/THUDM/GLM-4
3
-"""Inference-only ChatGLM model compatible with THUDM weights."""
2
+# https://github.com/THUDM/CogAgent
3
+"""Inference-only CogAgent model compatible with THUDM weights."""
4
from argparse import Namespace
5
from array import array
6
from typing import (Dict, Iterable, List, Mapping, Optional, Set, Tuple,
@@ -201,7 +201,6 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
201
202
new_input_ids = []
203
final_processed_position = 0
204
- final_processed_position = 0
204
205
for boi_position, eoi_position in zip(boi_positions, eoi_positions):
206
assert boi_position < eoi_position
@@ -275,12 +274,15 @@ class GLMAttention(nn.Module):
274
# https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
275
rope_ratio = getattr(config, "rope_ratio", 1.0)
276
max_positions = getattr(config, "seq_length", 8192)
277
+ # NOTE: THUDM/cogagent-9b-20241220 uses original_rope=False,
278
+ # which is equivalent to is_neox_style=True
279
+ is_neox_style = not config.original_rope
280
self.rotary_emb = get_rope(
281
self.head_dim,
282
rotary_dim=self.head_dim // 2,
283
max_position=max_positions,
284
base=10000 * rope_ratio,
283
- is_neox_style=False,
285
+ is_neox_style=is_neox_style,
286
)
287
self.attn = Attention(self.num_heads,
288
self.head_dim,
@@ -779,4 +781,4 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
781
return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
782
# Initialize LLM
783
else:
782
- return ChatGLM(vllm_config=vllm_config, prefix=prefix)
784
+ return ChatGLM(vllm_config=vllm_config, prefix=prefix)
\ No newline at end of file