A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
Transformers backend already supports V1 (#15463)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Mar 26, 2025 at 03:26 UTC
e42389f9d7a5e04aee3463b3e08bafdc86a9457b
3 files changed
+7
-25
tests/models/test_transformers.py
+5
-17
@@ -3,8 +3,6 @@
3
4
Run `pytest tests/models/test_transformers.py`.
5
"""
6
-from contextlib import nullcontext
7
-
6
import pytest
7
8
from ..conftest import HfRunner, VllmRunner
@@ -42,7 +40,6 @@ def check_implementation(
40
"model,model_impl",
41
[
42
("meta-llama/Llama-3.2-1B-Instruct", "transformers"),
45
- ("openai-community/gpt2", "transformers"),
43
("ArthurZ/Ilama-3.2-1B", "auto"), # CUSTOM CODE
44
]) # trust_remote_code=True by default
45
def test_models(
@@ -52,20 +49,11 @@ def test_models(
49
model: str,
50
model_impl: str,
51
) -> None:
55
-
56
- maybe_raises = nullcontext()
57
- if model == "openai-community/gpt2" and model_impl == "transformers":
58
- # Model is not backend compatible
59
- maybe_raises = pytest.raises(
60
- ValueError,
61
- match="The Transformers implementation.*not compatible with vLLM")
62
-
63
- with maybe_raises:
64
- check_implementation(hf_runner,
65
- vllm_runner,
66
- example_prompts,
67
- model,
68
- model_impl=model_impl)
52
+ check_implementation(hf_runner,
53
+ vllm_runner,
54
+ example_prompts,
55
+ model,
56
+ model_impl=model_impl)
57
58
59
@multi_gpu_test(num_gpus=2)
vllm/engine/arg_utils.py
-8
@@ -1613,14 +1613,6 @@ class EngineArgs:
1613
recommend_to_remove=False)
1614
return False
1615
1616
- # No TransformersModel support so far.
1617
- if (model_config.model_impl == ModelImpl.TRANSFORMERS
1618
- or model_config.model_impl == "transformers"):
1619
- _raise_or_fallback(
1620
- feature_name=f"model_impl={model_config.model_impl}",
1621
- recommend_to_remove=False)
1622
- return False
1623
-
1616
# No Concurrent Partial Prefills so far.
1617
if (self.max_num_partial_prefills
1618
!= EngineArgs.max_num_partial_prefills
vllm/model_executor/models/transformers.py
+2
@@ -24,6 +24,7 @@ from transformers import AutoModel, PretrainedConfig, PreTrainedModel
24
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
25
26
from vllm.attention import Attention
27
+from vllm.compilation.decorators import support_torch_compile
28
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
29
ParallelConfig, VllmConfig)
30
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
@@ -109,6 +110,7 @@ def replace_linear_class(
110
)
111
112
113
+@support_torch_compile
114
class TransformersModel(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
115
embedding_padding_modules = ["lm_head"]
116
embedding_modules = ["embed_tokens"