A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Implement lazy model loader (#2044)
Woosuk Kwon committed
Dec 12, 2023 at 22:21 UTC
518369d78c1ec9ffef308131366e4bda745b5573
3 files changed
+89
-101
vllm/model_executor/model_loader.py
+5
-57
@@ -7,54 +7,9 @@ import torch.nn as nn
7
from transformers import PretrainedConfig
8
9
from vllm.config import ModelConfig
10
-from vllm.model_executor.models import *
10
+from vllm.model_executor.models import ModelRegistry
11
from vllm.model_executor.weight_utils import (get_quant_config,
12
initialize_dummy_weights)
13
-from vllm.utils import is_hip
14
-from vllm.logger import init_logger
15
-
16
-logger = init_logger(__name__)
17
-
18
-# TODO(woosuk): Lazy-load the model classes.
19
-_MODEL_REGISTRY = {
20
- "AquilaModel": AquilaForCausalLM,
21
- "AquilaForCausalLM": AquilaForCausalLM, # AquilaChat2
22
- "BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
23
- "BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
24
- "BloomForCausalLM": BloomForCausalLM,
25
- "ChatGLMModel": ChatGLMForCausalLM,
26
- "ChatGLMForConditionalGeneration": ChatGLMForCausalLM,
27
- "FalconForCausalLM": FalconForCausalLM,
28
- "GPT2LMHeadModel": GPT2LMHeadModel,
29
- "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
30
- "GPTJForCausalLM": GPTJForCausalLM,
31
- "GPTNeoXForCausalLM": GPTNeoXForCausalLM,
32
- "InternLMForCausalLM": InternLMForCausalLM,
33
- "LlamaForCausalLM": LlamaForCausalLM,
34
- "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
35
- "MistralForCausalLM": MistralForCausalLM,
36
- "MixtralForCausalLM": MixtralForCausalLM,
37
- # transformers's mpt class has lower case
38
- "MptForCausalLM": MPTForCausalLM,
39
- "MPTForCausalLM": MPTForCausalLM,
40
- "OPTForCausalLM": OPTForCausalLM,
41
- "PhiForCausalLM": PhiForCausalLM,
42
- "QWenLMHeadModel": QWenLMHeadModel,
43
- "RWForCausalLM": FalconForCausalLM,
44
- "YiForCausalLM": YiForCausalLM,
45
-}
46
-
47
-# Models to be disabled in ROCm
48
-_ROCM_UNSUPPORTED_MODELS = []
49
-if is_hip():
50
- for rocm_model in _ROCM_UNSUPPORTED_MODELS:
51
- del _MODEL_REGISTRY[rocm_model]
52
-
53
-# Models partially supported in ROCm
54
-_ROCM_PARTIALLY_SUPPORTED_MODELS = {
55
- "MistralForCausalLM":
56
- "Sliding window attention is not supported in ROCm's flash attention",
57
-}
13
14
15
@contextlib.contextmanager
@@ -69,19 +24,12 @@ def _set_default_torch_dtype(dtype: torch.dtype):
24
def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
25
architectures = getattr(config, "architectures", [])
26
for arch in architectures:
72
- if arch in _MODEL_REGISTRY:
73
- if is_hip() and arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
74
- logger.warning(
75
- f"{arch} is not fully supported in ROCm. Reason: "
76
- f"{_ROCM_PARTIALLY_SUPPORTED_MODELS[arch]}")
77
- return _MODEL_REGISTRY[arch]
78
- elif arch in _ROCM_UNSUPPORTED_MODELS:
79
- raise ValueError(
80
- f"Model architecture {arch} is not supported by ROCm for now. \n"
81
- f"Supported architectures {list(_MODEL_REGISTRY.keys())}")
27
+ model_cls = ModelRegistry.load_model_cls(arch)
28
+ if model_cls is not None:
29
+ return model_cls
30
raise ValueError(
31
f"Model architectures {architectures} are not supported for now. "
84
- f"Supported architectures: {list(_MODEL_REGISTRY.keys())}")
32
+ f"Supported architectures: {ModelRegistry.get_supported_archs()}")
33
34
35
def get_model(model_config: ModelConfig) -> nn.Module:
vllm/model_executor/models/__init__.py
+77
-38
@@ -1,41 +1,80 @@
1
-from vllm.model_executor.models.aquila import AquilaForCausalLM
2
-from vllm.model_executor.models.baichuan import (BaiChuanForCausalLM,
3
- BaichuanForCausalLM)
4
-from vllm.model_executor.models.bloom import BloomForCausalLM
5
-from vllm.model_executor.models.falcon import FalconForCausalLM
6
-from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
7
-from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
8
-from vllm.model_executor.models.gpt_j import GPTJForCausalLM
9
-from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
10
-from vllm.model_executor.models.internlm import InternLMForCausalLM
11
-from vllm.model_executor.models.llama import LlamaForCausalLM
12
-from vllm.model_executor.models.mistral import MistralForCausalLM
13
-from vllm.model_executor.models.mixtral import MixtralForCausalLM
14
-from vllm.model_executor.models.mpt import MPTForCausalLM
15
-from vllm.model_executor.models.opt import OPTForCausalLM
16
-from vllm.model_executor.models.phi_1_5 import PhiForCausalLM
17
-from vllm.model_executor.models.qwen import QWenLMHeadModel
18
-from vllm.model_executor.models.chatglm import ChatGLMForCausalLM
19
-from vllm.model_executor.models.yi import YiForCausalLM
1
+import importlib
2
+from typing import List, Optional, Type
3
+
4
+import torch.nn as nn
5
+
6
+from vllm.logger import init_logger
7
+from vllm.utils import is_hip
8
+
9
+logger = init_logger(__name__)
10
+
11
+# Architecture -> (module, class).
12
+_MODELS = {
13
+ "AquilaModel": ("aquila", "AquilaForCausalLM"),
14
+ "AquilaForCausalLM": ("aquila", "AquilaForCausalLM"), # AquilaChat2
15
+ "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
16
+ "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
17
+ "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
18
+ "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19
+ "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20
+ "FalconForCausalLM": ("falcon", "FalconForCausalLM"),
21
+ "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
22
+ "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
23
+ "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
24
+ "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
25
+ "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
26
+ "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
27
+ # For decapoda-research/llama-*
28
+ "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
29
+ "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
30
+ "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
31
+ # transformers's mpt class has lower case
32
+ "MptForCausalLM": ("mpt", "MPTForCausalLM"),
33
+ "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
34
+ "OPTForCausalLM": ("opt", "OPTForCausalLM"),
35
+ "PhiForCausalLM": ("phi_1_5", "PhiForCausalLM"),
36
+ "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
37
+ "RWForCausalLM": ("falcon", "FalconForCausalLM"),
38
+ "YiForCausalLM": ("yi", "YiForCausalLM"),
39
+}
40
+
41
+# Models not supported by ROCm.
42
+_ROCM_UNSUPPORTED_MODELS = ["MixtralForCausalLM"]
43
+
44
+# Models partially supported by ROCm.
45
+# Architecture -> Reason.
46
+_ROCM_PARTIALLY_SUPPORTED_MODELS = {
47
+ "MistralForCausalLM":
48
+ "Sliding window attention is not yet supported in ROCm's flash attention",
49
+}
50
+
51
+
52
+class ModelRegistry:
53
+
54
+ @staticmethod
55
+ def load_model_cls(model_arch: str) -> Optional[Type[nn.Module]]:
56
+ if model_arch not in _MODELS:
57
+ return None
58
+ if is_hip():
59
+ if model_arch in _ROCM_UNSUPPORTED_MODELS:
60
+ raise ValueError(
61
+ f"Model architecture {model_arch} is not supported by "
62
+ "ROCm for now.")
63
+ if model_arch in _ROCM_PARTIALLY_SUPPORTED_MODELS:
64
+ logger.warning(
65
+ f"Model architecture {model_arch} is partially supported "
66
+ "by ROCm: " + _ROCM_PARTIALLY_SUPPORTED_MODELS[model_arch])
67
+
68
+ module_name, model_cls_name = _MODELS[model_arch]
69
+ module = importlib.import_module(
70
+ f"vllm.model_executor.models.{module_name}")
71
+ return getattr(module, model_cls_name, None)
72
+
73
+ @staticmethod
74
+ def get_supported_archs() -> List[str]:
75
+ return list(_MODELS.keys())
76
+
77
78
__all__ = [
22
- "AquilaForCausalLM",
23
- "BaiChuanForCausalLM",
24
- "BaichuanForCausalLM",
25
- "BloomForCausalLM",
26
- "ChatGLMForCausalLM",
27
- "FalconForCausalLM",
28
- "GPT2LMHeadModel",
29
- "GPTBigCodeForCausalLM",
30
- "GPTJForCausalLM",
31
- "GPTNeoXForCausalLM",
32
- "InternLMForCausalLM",
33
- "LlamaForCausalLM",
34
- "MPTForCausalLM",
35
- "OPTForCausalLM",
36
- "PhiForCausalLM",
37
- "QWenLMHeadModel",
38
- "MistralForCausalLM",
39
- "MixtralForCausalLM",
40
- "YiForCausalLM",
79
+ "ModelRegistry",
80
]
vllm/model_executor/models/mixtral.py
+7
-6
@@ -33,14 +33,15 @@ from transformers import MixtralConfig
33
34
try:
35
import megablocks.ops as ops
36
-except ImportError:
37
- print(
38
- "MegaBlocks not found. Please install it by `pip install megablocks`.")
36
+except ImportError as e:
37
+ raise ImportError("MegaBlocks not found. "
38
+ "Please install it by `pip install megablocks`.") from e
39
try:
40
import stk
41
-except ImportError:
42
- print(
43
- "STK not found: please see https://github.com/stanford-futuredata/stk")
41
+except ImportError as e:
42
+ raise ImportError(
43
+ "STK not found. "
44
+ "Please install it by `pip install stanford-stk`.") from e
45
46
from vllm.model_executor.input_metadata import InputMetadata
47
from vllm.model_executor.layers.attention import PagedAttention