@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b5bf20988
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] Adding torch compile annotations to some models (#9876)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

Yongzao committed Nov 1, 2024 at 15:25 UTC 2b5bf20988edaab21621b78a9eb589edc93f2763
7 files changed +12 -2
docs/source/models/supported_models.rst
+1 -1
@@ -281,7 +281,7 @@ Text Generation
281 - ✅︎
282 * - :code:`Qwen2ForCausalLM`
283 - Qwen2
284 - - :code:`Qwen/Qwen2-beta-7B`, :code:`Qwen/Qwen2-beta-7B-Chat`, etc.
284 + - :code:`Qwen/Qwen2-7B-Instruct`, :code:`Qwen/Qwen2-7B`, etc.
285 - ✅︎
286 - ✅︎
287 * - :code:`Qwen2MoeForCausalLM`
tests/distributed/test_pipeline_parallel.py
+1 -1
@@ -166,7 +166,7 @@ TEXT_GENERATION_MODELS = {
166 "microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
167 "adept/persimmon-8b-chat": PPTestSettings.fast(),
168 "Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
169 - "Qwen/Qwen2-beta-7B-Chat": PPTestSettings.fast(),
169 + "Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
170 "Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
171 "stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
172 "bigcode/starcoder2-3b": PPTestSettings.fast(),
vllm/model_executor/models/falcon.py
+2
@@ -27,6 +27,7 @@ from torch.nn import LayerNorm
27 from transformers import FalconConfig as HF_FalconConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 +from vllm.compilation.decorators import support_torch_compile
31 from vllm.config import CacheConfig
32 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33 get_tensor_model_parallel_world_size,
@@ -329,6 +330,7 @@ class FalconDecoderLayer(nn.Module):
330 return output
331
332
333 +@support_torch_compile
334 class FalconModel(nn.Module):
335
336 def __init__(
vllm/model_executor/models/phi.py
+2
@@ -42,6 +42,7 @@ from torch import nn
42 from transformers import PhiConfig
43
44 from vllm.attention import Attention, AttentionMetadata
45 +from vllm.compilation.decorators import support_torch_compile
46 from vllm.config import CacheConfig, LoRAConfig
47 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
48 from vllm.model_executor.layers.activation import get_act_fn
@@ -193,6 +194,7 @@ class PhiLayer(nn.Module):
194 return hidden_states
195
196
197 +@support_torch_compile
198 class PhiModel(nn.Module):
199
200 def __init__(self,
vllm/model_executor/models/qwen.py
+2
@@ -20,6 +20,7 @@ from torchvision.transforms import InterpolationMode
20 from transformers import PretrainedConfig
21
22 from vllm.attention import Attention, AttentionMetadata
23 +from vllm.compilation.decorators import support_torch_compile
24 from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
25 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
26 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
@@ -549,6 +550,7 @@ class QWenBlock(nn.Module):
550 return hidden_states, residual
551
552
553 +@support_torch_compile
554 class QWenModel(nn.Module):
555
556 def __init__(
vllm/model_executor/models/qwen2.py
+2
@@ -29,6 +29,7 @@ from torch import nn
29 from transformers import Qwen2Config
30
31 from vllm.attention import Attention, AttentionMetadata
32 +from vllm.compilation.decorators import support_torch_compile
33 from vllm.config import CacheConfig, LoRAConfig
34 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -237,6 +238,7 @@ class Qwen2DecoderLayer(nn.Module):
238 return hidden_states, residual
239
240
241 +@support_torch_compile
242 class Qwen2Model(nn.Module):
243
244 def __init__(
vllm/model_executor/models/qwen2_moe.py
+2
@@ -30,6 +30,7 @@ from torch import nn
30 from transformers import PretrainedConfig
31
32 from vllm.attention import Attention, AttentionMetadata
33 +from vllm.compilation.decorators import support_torch_compile
34 from vllm.config import CacheConfig
35 from vllm.distributed import (get_pp_group,
36 get_tensor_model_parallel_world_size,
@@ -312,6 +313,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
313 return hidden_states, residual
314
315
316 +@support_torch_compile
317 class Qwen2MoeModel(nn.Module):
318
319 def __init__(