A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add model_utils
Woosuk Kwon committed
Feb 13, 2023 at 09:36 UTC
fffa2e1f4b7534d5f86e900838d9a24dfba307c9
2 files changed
+17
-2
cacheflow/worker/models/__init__.py
+4
-2
@@ -1,5 +1,7 @@
1
-from cacheflow.worker.models.opt import OPTForCausalLM
1
+from cacheflow.worker.models.model_utils import get_model
2
+
3
4
__all__ = [
4
- 'OPTForCausalLM',
5
+ 'get_model',
6
+
7
]
cacheflow/worker/models/model_utils.py
new
+13
@@ -0,0 +1,13 @@
1
+import torch.nn as nn
2
+
3
+from cacheflow.worker.models.opt import OPTForCausalLM
4
+
5
+MODEL_CLASSES = {
6
+ 'opt': OPTForCausalLM,
7
+}
8
+
9
+
10
+def get_model(model_name: str) -> nn.Module:
11
+ if model_name not in MODEL_CLASSES:
12
+ raise ValueError(f'Invalid model name: {model_name}')
13
+ return MODEL_CLASSES[model_name].from_pretrained(model_name)