A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Set default dtype to half
Woosuk Kwon committed
Feb 23, 2023 at 21:31 UTC
1ce13335732c8b28d4a76118821b391f6b219b7c
3 files changed
+23
-3
cacheflow/models/model_utils.py
+19
-2
@@ -1,3 +1,6 @@
1
+from typing import Union
2
+
3
+import torch
4
import torch.nn as nn
5
6
from cacheflow.models.opt import OPTForCausalLM
@@ -6,9 +9,23 @@ MODEL_CLASSES = {
9
'opt': OPTForCausalLM,
10
}
11
12
+STR_DTYPE_TO_TORCH_DTYPE = {
13
+ 'half': torch.half,
14
+ 'float': torch.float,
15
+ 'float16': torch.float16,
16
+ 'float32': torch.float32,
17
+}
18
+
19
10
-def get_model(model_name: str) -> nn.Module:
20
+def get_model(
21
+ model_name: str,
22
+ dtype: Union[torch.dtype, str],
23
+) -> nn.Module:
24
+ if isinstance(dtype, str):
25
+ torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
26
+ else:
27
+ torch_dtype = dtype
28
for model_class, model in MODEL_CLASSES.items():
29
if model_class in model_name:
13
- return model.from_pretrained(model_name)
30
+ return model.from_pretrained(model_name, torch_dtype=torch_dtype)
31
raise ValueError(f'Invalid model name: {model_name}')
cacheflow/worker/controller.py
+2
@@ -14,6 +14,7 @@ class Controller:
14
block_size: int,
15
num_gpu_blocks: int,
16
num_cpu_blocks: int,
17
+ dtype: str = 'half',
18
) -> None:
19
self.node_id = node_id
20
self.num_workers = num_workers
@@ -35,6 +36,7 @@ class Controller:
36
block_size=block_size,
37
num_gpu_blocks=num_gpu_blocks,
38
num_cpu_blocks=num_cpu_blocks,
39
+ dtype=dtype,
40
)
41
self.workers.append(worker)
42
cacheflow/worker/worker.py
+2
-1
@@ -17,6 +17,7 @@ class Worker:
17
block_size: int,
18
num_gpu_blocks: int,
19
num_cpu_blocks: int,
20
+ dtype: str,
21
) -> None:
22
self.worker_id = worker_id
23
self.gpu_id = gpu_id
@@ -26,7 +27,7 @@ class Worker:
27
28
# Initialize the model.
29
# FIXME(woosuk): This is a hack.
29
- self.model = get_model(model_name).to(device=gpu_id)
30
+ self.model = get_model(model_name, dtype=dtype).to(device=self.device)
31
self.num_layers = self.model.config.num_hidden_layers
32
self.num_heads = self.model.config.num_attention_heads
33
self.head_size = self.model.config.hidden_size // self.num_heads