@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ce1333573
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Set default dtype to half

Woosuk Kwon committed Feb 23, 2023 at 21:31 UTC 1ce13335732c8b28d4a76118821b391f6b219b7c
3 files changed +23 -3
cacheflow/models/model_utils.py
+19 -2
@@ -1,3 +1,6 @@
1 +from typing import Union
2 +
3 +import torch
4 import torch.nn as nn
5
6 from cacheflow.models.opt import OPTForCausalLM
@@ -6,9 +9,23 @@ MODEL_CLASSES = {
9 'opt': OPTForCausalLM,
10 }
11
12 +STR_DTYPE_TO_TORCH_DTYPE = {
13 + 'half': torch.half,
14 + 'float': torch.float,
15 + 'float16': torch.float16,
16 + 'float32': torch.float32,
17 +}
18 +
19
10 -def get_model(model_name: str) -> nn.Module:
20 +def get_model(
21 + model_name: str,
22 + dtype: Union[torch.dtype, str],
23 +) -> nn.Module:
24 + if isinstance(dtype, str):
25 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
26 + else:
27 + torch_dtype = dtype
28 for model_class, model in MODEL_CLASSES.items():
29 if model_class in model_name:
13 - return model.from_pretrained(model_name)
30 + return model.from_pretrained(model_name, torch_dtype=torch_dtype)
31 raise ValueError(f'Invalid model name: {model_name}')
cacheflow/worker/controller.py
+2
@@ -14,6 +14,7 @@ class Controller:
14 block_size: int,
15 num_gpu_blocks: int,
16 num_cpu_blocks: int,
17 + dtype: str = 'half',
18 ) -> None:
19 self.node_id = node_id
20 self.num_workers = num_workers
@@ -35,6 +36,7 @@ class Controller:
36 block_size=block_size,
37 num_gpu_blocks=num_gpu_blocks,
38 num_cpu_blocks=num_cpu_blocks,
39 + dtype=dtype,
40 )
41 self.workers.append(worker)
42
cacheflow/worker/worker.py
+2 -1
@@ -17,6 +17,7 @@ class Worker:
17 block_size: int,
18 num_gpu_blocks: int,
19 num_cpu_blocks: int,
20 + dtype: str,
21 ) -> None:
22 self.worker_id = worker_id
23 self.gpu_id = gpu_id
@@ -26,7 +27,7 @@ class Worker:
27
28 # Initialize the model.
29 # FIXME(woosuk): This is a hack.
29 - self.model = get_model(model_name).to(device=gpu_id)
30 + self.model = get_model(model_name, dtype=dtype).to(device=self.device)
31 self.num_layers = self.model.config.num_hidden_layers
32 self.num_heads = self.model.config.num_attention_heads
33 self.head_size = self.model.config.hidden_size // self.num_heads