A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] Adding torch compile annotations to some models (#9876)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
Yongzao committed
Nov 1, 2024 at 15:25 UTC
2b5bf20988edaab21621b78a9eb589edc93f2763
7 files changed
+12
-2
docs/source/models/supported_models.rst
+1
-1
@@ -281,7 +281,7 @@ Text Generation
281
- ✅︎
282
* - :code:`Qwen2ForCausalLM`
283
- Qwen2
284
- - :code:`Qwen/Qwen2-beta-7B`, :code:`Qwen/Qwen2-beta-7B-Chat`, etc.
284
+ - :code:`Qwen/Qwen2-7B-Instruct`, :code:`Qwen/Qwen2-7B`, etc.
285
- ✅︎
286
- ✅︎
287
* - :code:`Qwen2MoeForCausalLM`
tests/distributed/test_pipeline_parallel.py
+1
-1
@@ -166,7 +166,7 @@ TEXT_GENERATION_MODELS = {
166
"microsoft/Phi-3.5-MoE-instruct": PPTestSettings.fast(trust_remote_code=True), # noqa: E501
167
"adept/persimmon-8b-chat": PPTestSettings.fast(),
168
"Qwen/Qwen-7B-Chat": PPTestSettings.fast(trust_remote_code=True),
169
- "Qwen/Qwen2-beta-7B-Chat": PPTestSettings.fast(),
169
+ "Qwen/Qwen2-7B-Instruct": PPTestSettings.fast(),
170
"Qwen/Qwen1.5-MoE-A2.7B-Chat": PPTestSettings.fast(),
171
"stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
172
"bigcode/starcoder2-3b": PPTestSettings.fast(),
vllm/model_executor/models/falcon.py
+2
@@ -27,6 +27,7 @@ from torch.nn import LayerNorm
27
from transformers import FalconConfig as HF_FalconConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
+from vllm.compilation.decorators import support_torch_compile
31
from vllm.config import CacheConfig
32
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
33
get_tensor_model_parallel_world_size,
@@ -329,6 +330,7 @@ class FalconDecoderLayer(nn.Module):
330
return output
331
332
333
+@support_torch_compile
334
class FalconModel(nn.Module):
335
336
def __init__(
vllm/model_executor/models/phi.py
+2
@@ -42,6 +42,7 @@ from torch import nn
42
from transformers import PhiConfig
43
44
from vllm.attention import Attention, AttentionMetadata
45
+from vllm.compilation.decorators import support_torch_compile
46
from vllm.config import CacheConfig, LoRAConfig
47
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
48
from vllm.model_executor.layers.activation import get_act_fn
@@ -193,6 +194,7 @@ class PhiLayer(nn.Module):
194
return hidden_states
195
196
197
+@support_torch_compile
198
class PhiModel(nn.Module):
199
200
def __init__(self,
vllm/model_executor/models/qwen.py
+2
@@ -20,6 +20,7 @@ from torchvision.transforms import InterpolationMode
20
from transformers import PretrainedConfig
21
22
from vllm.attention import Attention, AttentionMetadata
23
+from vllm.compilation.decorators import support_torch_compile
24
from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
25
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
26
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
@@ -549,6 +550,7 @@ class QWenBlock(nn.Module):
550
return hidden_states, residual
551
552
553
+@support_torch_compile
554
class QWenModel(nn.Module):
555
556
def __init__(
vllm/model_executor/models/qwen2.py
+2
@@ -29,6 +29,7 @@ from torch import nn
29
from transformers import Qwen2Config
30
31
from vllm.attention import Attention, AttentionMetadata
32
+from vllm.compilation.decorators import support_torch_compile
33
from vllm.config import CacheConfig, LoRAConfig
34
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -237,6 +238,7 @@ class Qwen2DecoderLayer(nn.Module):
238
return hidden_states, residual
239
240
241
+@support_torch_compile
242
class Qwen2Model(nn.Module):
243
244
def __init__(
vllm/model_executor/models/qwen2_moe.py
+2
@@ -30,6 +30,7 @@ from torch import nn
30
from transformers import PretrainedConfig
31
32
from vllm.attention import Attention, AttentionMetadata
33
+from vllm.compilation.decorators import support_torch_compile
34
from vllm.config import CacheConfig
35
from vllm.distributed import (get_pp_group,
36
get_tensor_model_parallel_world_size,
@@ -312,6 +313,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
313
return hidden_states, residual
314
315
316
+@support_torch_compile
317
class Qwen2MoeModel(nn.Module):
318
319
def __init__(