@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fffa2e1f4b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add model_utils

Woosuk Kwon committed Feb 13, 2023 at 09:36 UTC fffa2e1f4b7534d5f86e900838d9a24dfba307c9
2 files changed +17 -2
cacheflow/worker/models/__init__.py
+4 -2
@@ -1,5 +1,7 @@
1 -from cacheflow.worker.models.opt import OPTForCausalLM
1 +from cacheflow.worker.models.model_utils import get_model
2 +
3
4 __all__ = [
4 - 'OPTForCausalLM',
5 + 'get_model',
6 +
7 ]
cacheflow/worker/models/model_utils.py new
+13
@@ -0,0 +1,13 @@
1 +import torch.nn as nn
2 +
3 +from cacheflow.worker.models.opt import OPTForCausalLM
4 +
5 +MODEL_CLASSES = {
6 + 'opt': OPTForCausalLM,
7 +}
8 +
9 +
10 +def get_model(model_name: str) -> nn.Module:
11 + if model_name not in MODEL_CLASSES:
12 + raise ValueError(f'Invalid model name: {model_name}')
13 + return MODEL_CLASSES[model_name].from_pretrained(model_name)