@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b657d3292
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Model] Add cogagent model support vLLM (#11742)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

sixgod committed Jan 12, 2025 at 03:05 UTC 4b657d32922cb6d3179f901e312715279ad9c728
1 file changed +7 -5
vllm/model_executor/models/chatglm.py
+7 -5
@@ -1,6 +1,6 @@
1 # Adapted from
2 -# https://github.com/THUDM/GLM-4
3 -"""Inference-only ChatGLM model compatible with THUDM weights."""
2 +# https://github.com/THUDM/CogAgent
3 +"""Inference-only CogAgent model compatible with THUDM weights."""
4 from argparse import Namespace
5 from array import array
6 from typing import (Dict, Iterable, List, Mapping, Optional, Set, Tuple,
@@ -201,7 +201,6 @@ def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
201
202 new_input_ids = []
203 final_processed_position = 0
204 - final_processed_position = 0
204
205 for boi_position, eoi_position in zip(boi_positions, eoi_positions):
206 assert boi_position < eoi_position
@@ -275,12 +274,15 @@ class GLMAttention(nn.Module):
274 # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
275 rope_ratio = getattr(config, "rope_ratio", 1.0)
276 max_positions = getattr(config, "seq_length", 8192)
277 + # NOTE: THUDM/cogagent-9b-20241220 uses original_rope=False,
278 + # which is equivalent to is_neox_style=True
279 + is_neox_style = not config.original_rope
280 self.rotary_emb = get_rope(
281 self.head_dim,
282 rotary_dim=self.head_dim // 2,
283 max_position=max_positions,
284 base=10000 * rope_ratio,
283 - is_neox_style=False,
285 + is_neox_style=is_neox_style,
286 )
287 self.attn = Attention(self.num_heads,
288 self.head_dim,
@@ -779,4 +781,4 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
781 return ChatGLMV(vllm_config=vllm_config, prefix=prefix)
782 # Initialize LLM
783 else:
782 - return ChatGLM(vllm_config=vllm_config, prefix=prefix)
784 + return ChatGLM(vllm_config=vllm_config, prefix=prefix)
\ No newline at end of file