A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Introduce LLM class for offline inference (#115)
Woosuk Kwon committed
May 21, 2023 at 17:04 UTC
655a5e48df3937bf793add53aa95ce0c992a24c6
9 files changed
+221
-80
cacheflow/__init__.py
+5
-9
@@ -1,19 +1,15 @@
1
+from cacheflow.entrypoints.llm import LLM
2
from cacheflow.outputs import RequestOutput
3
from cacheflow.sampling_params import SamplingParams
3
-from cacheflow.server.arg_utils import (
4
- add_server_arguments,
5
- create_server_configs_from_args,
6
- initialize_server_from_args,
7
-)
4
+from cacheflow.server.arg_utils import ServerArgs
5
from cacheflow.server.llm_server import LLMServer
6
from cacheflow.server.ray_utils import initialize_cluster
7
8
__all__ = [
12
- "RequestOutput",
9
+ "LLM",
10
"SamplingParams",
11
+ "RequestOutput",
12
"LLMServer",
15
- "add_server_arguments",
16
- "create_server_configs_from_args",
17
- "initialize_server_from_args",
13
+ "ServerArgs",
14
"initialize_cluster",
15
]
cacheflow/config.py
+5
-1
@@ -3,6 +3,8 @@ from typing import Optional
3
import torch
4
from transformers import AutoConfig, PretrainedConfig
5
6
+_GiB = 1 << 30
7
+
8
9
class ModelConfig:
10
@@ -70,7 +72,7 @@ class CacheConfig:
72
) -> None:
73
self.block_size = block_size
74
self.gpu_memory_utilization = gpu_memory_utilization
73
- self.swap_space = swap_space
75
+ self.swap_space_bytes = swap_space * _GiB
76
77
# Will be set after profiling.
78
self.num_gpu_blocks = None
@@ -138,6 +140,8 @@ def _get_and_verify_dtype(
140
else:
141
torch_dtype = config_dtype
142
else:
143
+ if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
144
+ raise ValueError(f"Unknown dtype: {dtype}")
145
torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
146
147
# Verify the dtype.
cacheflow/entrypoints/fastapi_server.py
+3
-4
@@ -12,8 +12,7 @@ import uvicorn
12
13
from cacheflow.outputs import RequestOutput
14
from cacheflow.sampling_params import SamplingParams
15
-from cacheflow.server.arg_utils import (
16
- add_server_arguments, create_server_configs_from_args)
15
+from cacheflow.server.arg_utils import ServerArgs
16
from cacheflow.server.llm_server import LLMServer
17
from cacheflow.server.ray_utils import initialize_cluster
18
@@ -116,10 +115,10 @@ if __name__ == "__main__":
115
parser = argparse.ArgumentParser()
116
parser.add_argument("--host", type=str, default="localhost")
117
parser.add_argument("--port", type=int, default=10002)
119
- parser = add_server_arguments(parser)
118
+ parser = ServerArgs.add_cli_args(parser)
119
args = parser.parse_args()
120
122
- server_configs = create_server_configs_from_args(args)
121
+ server_configs = ServerArgs.from_cli_args(args).create_server_configs()
122
parallel_config = server_configs[2]
123
distributed_init_method, stage_devices = initialize_cluster(parallel_config)
124
cacheflow/entrypoints/llm.py
new
+62
@@ -0,0 +1,62 @@
1
+from typing import List, Optional
2
+
3
+from tqdm import tqdm
4
+
5
+from cacheflow.outputs import RequestOutput
6
+from cacheflow.sampling_params import SamplingParams
7
+from cacheflow.server.arg_utils import ServerArgs
8
+from cacheflow.server.llm_server import LLMServer
9
+from cacheflow.utils import Counter
10
+
11
+
12
+class LLM:
13
+
14
+ def __init__(
15
+ self,
16
+ model: str,
17
+ tensor_parallel_size: int = 1,
18
+ dtype: str = "default",
19
+ seed: int = 0,
20
+ **kwargs,
21
+ ) -> None:
22
+ if "disable_log_stats" not in kwargs:
23
+ kwargs["disable_log_stats"] = True
24
+ server_args = ServerArgs(
25
+ model=model,
26
+ tensor_parallel_size=tensor_parallel_size,
27
+ dtype=dtype,
28
+ seed=seed,
29
+ **kwargs,
30
+ )
31
+ self.llm_server = LLMServer.from_server_args(server_args)
32
+ self.request_counter = Counter()
33
+
34
+ def generate(
35
+ self,
36
+ prompts: List[str],
37
+ sampling_params: Optional[SamplingParams] = None,
38
+ use_tqdm: bool = True,
39
+ ) -> List[RequestOutput]:
40
+ if sampling_params is None:
41
+ sampling_params = SamplingParams()
42
+ # Initialize tqdm.
43
+ if use_tqdm:
44
+ pbar = tqdm(total=len(prompts), desc="Processed prompts")
45
+
46
+ # Add requests to the server.
47
+ for prompt in prompts:
48
+ request_id = str(next(self.request_counter))
49
+ self.llm_server.add_request(request_id, prompt, sampling_params)
50
+
51
+ # Run the server.
52
+ outputs: List[RequestOutput] = []
53
+ while self.llm_server.has_unfinished_requests():
54
+ step_outputs = self.llm_server.step()
55
+ for output in step_outputs:
56
+ if output.done:
57
+ outputs.append(output)
58
+ if use_tqdm:
59
+ pbar.update(1)
60
+ if use_tqdm:
61
+ pbar.close()
62
+ return outputs
cacheflow/outputs.py
+5
-5
@@ -35,7 +35,7 @@ class RequestOutput:
35
prompt: str,
36
prompt_token_ids: List[int],
37
outputs: List[CompletionOutput],
38
- done: bool = False,
38
+ done: bool,
39
) -> None:
40
self.request_id = request_id
41
self.prompt = prompt
@@ -43,8 +43,8 @@ class RequestOutput:
43
self.outputs = outputs
44
self.done = done
45
46
- @staticmethod
47
- def from_seq_group(seq_group: SequenceGroup) -> "RequestOutput":
46
+ @classmethod
47
+ def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
48
# Get the top-n sequences.
49
n = seq_group.sampling_params.n
50
seqs = seq_group.get_seqs()
@@ -70,8 +70,8 @@ class RequestOutput:
70
# Every sequence in the sequence group should have the same prompt.
71
prompt = top_n_seqs[0].prompt
72
prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
73
- return RequestOutput(seq_group.request_id, prompt, prompt_token_ids,
74
- outputs, seq_group.is_finished())
73
+ return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
74
+ seq_group.is_finished())
75
76
def __repr__(self) -> str:
77
return (f"RequestOutput(request_id={self.request_id}, "
cacheflow/server/arg_utils.py
+97
-54
@@ -1,74 +1,117 @@
1
import argparse
2
-from typing import Tuple
2
+import dataclasses
3
+from dataclasses import dataclass
4
+from typing import Optional, Tuple
5
6
from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7
SchedulerConfig)
6
-from cacheflow.server.llm_server import LLMServer
7
-from cacheflow.server.ray_utils import initialize_cluster
8
9
-_GiB = 1 << 30
9
10
+@dataclass
11
+class ServerArgs:
12
+ model: str
13
+ download_dir: Optional[str] = None
14
+ use_np_weights: bool = False
15
+ use_dummy_weights: bool = False
16
+ dtype: str = "default"
17
+ seed: int = 0
18
+ use_ray: bool = False
19
+ pipeline_parallel_size: int = 1
20
+ tensor_parallel_size: int = 1
21
+ block_size: int = 16
22
+ swap_space: int = 4 # GiB
23
+ gpu_memory_utilization: float = 0.95
24
+ max_num_batched_tokens: int = 2560
25
+ max_num_seqs: int = 256
26
+ disable_log_stats: bool = False
27
12
-def add_server_arguments(parser: argparse.ArgumentParser):
13
- """Shared arguments for CacheFlow servers."""
28
+ def __post_init__(self):
29
+ self.max_num_seqs = min(self.max_num_seqs, self.max_num_batched_tokens)
30
+
31
+ @staticmethod
32
+ def add_cli_args(
33
+ parser: argparse.ArgumentParser,
34
+ ) -> argparse.ArgumentParser:
35
+ return _add_server_arguments(parser)
36
+
37
+ @classmethod
38
+ def from_cli_args(cls, args: argparse.Namespace) -> "ServerArgs":
39
+ # Get the list of attributes of this dataclass.
40
+ attrs = [attr.name for attr in dataclasses.fields(cls)]
41
+ # Set the attributes from the parsed arguments.
42
+ server_args = cls(**{attr: getattr(args, attr) for attr in attrs})
43
+ return server_args
44
+
45
+ def create_server_configs(
46
+ self,
47
+ ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
48
+ # Initialize the configs.
49
+ model_config = ModelConfig(
50
+ self.model, self.download_dir, self.use_np_weights,
51
+ self.use_dummy_weights, self.dtype, self.seed)
52
+ cache_config = CacheConfig(self.block_size, self.gpu_memory_utilization,
53
+ self.swap_space)
54
+ parallel_config = ParallelConfig(self.pipeline_parallel_size,
55
+ self.tensor_parallel_size,
56
+ self.use_ray)
57
+ scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
58
+ self.max_num_seqs)
59
+ return model_config, cache_config, parallel_config, scheduler_config
60
+
61
+
62
+def _add_server_arguments(
63
+ parser: argparse.ArgumentParser,
64
+)-> argparse.ArgumentParser:
65
+ """Shared CLI arguments for CacheFlow servers."""
66
# Model arguments
15
- parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
16
- parser.add_argument('--download-dir', type=str, default=None,
67
+ parser.add_argument('--model', type=str, default='facebook/opt-125m',
68
+ help='name or path of the huggingface model to use')
69
+ parser.add_argument('--download-dir', type=str,
70
+ default=ServerArgs.download_dir,
71
help='directory to download and load the weights, '
72
'default to the default cache dir of huggingface')
73
parser.add_argument('--use-np-weights', action='store_true',
20
- help='save a numpy copy of model weights for faster loading')
21
- parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
74
+ help='save a numpy copy of model weights for faster '
75
+ 'loading. This can increase the disk usage by up '
76
+ 'to 2x.')
77
+ parser.add_argument('--use-dummy-weights', action='store_true',
78
+ help='use dummy values for model weights')
79
# TODO(woosuk): Support FP32.
23
- parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
80
+ parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
81
+ choices=['default', 'half', 'bfloat16'],
82
help=('data type for model weights and activations. '
83
'The "default" option will use FP16 precision '
84
'for FP32 and FP16 models, and BF16 precision '
85
'for BF16 models.'))
86
# Parallel arguments
29
- parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
30
- parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
31
- parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
87
+ parser.add_argument('--use-ray', action='store_true',
88
+ help='use Ray for distributed serving, will be '
89
+ 'automatically set when using more than 1 GPU')
90
+ parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
91
+ default=ServerArgs.pipeline_parallel_size,
92
+ help='number of pipeline stages')
93
+ parser.add_argument('--tensor-parallel-size', '-tp', type=int,
94
+ default=ServerArgs.tensor_parallel_size,
95
+ help='number of tensor parallel replicas')
96
# KV cache arguments
33
- parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
97
+ parser.add_argument('--block-size', type=int, default=ServerArgs.block_size,
98
+ choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
99
+ help='token block size')
100
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
35
- parser.add_argument('--seed', type=int, default=0, help='random seed')
36
- parser.add_argument('--swap-space', type=int, default=4, help='CPU swap space size (GiB) per GPU')
37
- parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
38
- parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
39
- parser.add_argument('--max-num-seqs', type=int, default=256, help='maximum number of sequences per iteration')
40
- parser.add_argument('--disable-log-stats', action='store_true', help='disable logging statistics')
101
+ parser.add_argument('--seed', type=int, default=ServerArgs.seed,
102
+ help='random seed')
103
+ parser.add_argument('--swap-space', type=int, default=ServerArgs.swap_space,
104
+ help='CPU swap space size (GiB) per GPU')
105
+ parser.add_argument('--gpu-memory-utilization', type=float,
106
+ default=ServerArgs.gpu_memory_utilization,
107
+ help='the percentage of GPU memory to be used for the '
108
+ 'model executor')
109
+ parser.add_argument('--max-num-batched-tokens', type=int,
110
+ default=ServerArgs.max_num_batched_tokens,
111
+ help='maximum number of batched tokens per iteration')
112
+ parser.add_argument('--max-num-seqs', type=int,
113
+ default=ServerArgs.max_num_seqs,
114
+ help='maximum number of sequences per iteration')
115
+ parser.add_argument('--disable-log-stats', action='store_true',
116
+ help='disable logging statistics')
117
return parser
42
-
43
-
44
-def create_server_configs_from_args(
45
- args: argparse.Namespace,
46
-) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
47
- # Post-process the parsed arguments.
48
- args.swap_space = args.swap_space * _GiB
49
- args.max_num_seqs = min(args.max_num_seqs, args.max_num_batched_tokens)
50
-
51
- # Initialize the configs.
52
- model_config = ModelConfig(
53
- args.model, args.download_dir, args.use_np_weights,
54
- args.use_dummy_weights, args.dtype, args.seed)
55
- cache_config = CacheConfig(args.block_size, args.gpu_memory_utilization,
56
- args.swap_space)
57
- parallel_config = ParallelConfig(args.pipeline_parallel_size,
58
- args.tensor_parallel_size, args.use_ray)
59
- scheduler_config = SchedulerConfig(args.max_num_batched_tokens,
60
- args.max_num_seqs)
61
- return model_config, cache_config, parallel_config, scheduler_config
62
-
63
-
64
-def initialize_server_from_args(args: argparse.Namespace) -> LLMServer:
65
- server_configs = create_server_configs_from_args(args)
66
- parallel_config = server_configs[2]
67
-
68
- # Initialize the cluster.
69
- distributed_init_method, devices = initialize_cluster(parallel_config)
70
-
71
- # Create the LLM server.
72
- server = LLMServer(*server_configs, distributed_init_method, devices,
73
- log_stats=not args.disable_log_stats)
74
- return server
cacheflow/server/llm_server.py
+16
-2
@@ -12,6 +12,8 @@ from cacheflow.core.scheduler import Scheduler
12
from cacheflow.logger import init_logger
13
from cacheflow.outputs import RequestOutput
14
from cacheflow.sampling_params import SamplingParams
15
+from cacheflow.server.arg_utils import ServerArgs
16
+from cacheflow.server.ray_utils import initialize_cluster
17
from cacheflow.server.tokenizer_utils import get_tokenizer
18
from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
19
from cacheflow.utils import Counter
@@ -30,7 +32,7 @@ class LLMServer:
32
scheduler_config: SchedulerConfig,
33
distributed_init_method: str,
34
stage_devices: List[List[Any]],
33
- log_stats: bool = True,
35
+ log_stats: bool,
36
) -> None:
37
logger.info(
38
"Initializing an LLM server with config: "
@@ -90,7 +92,7 @@ class LLMServer:
92
get_all_outputs=True,
93
block_size=self.cache_config.block_size,
94
gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
93
- cpu_swap_space=self.cache_config.swap_space,
95
+ cpu_swap_space=self.cache_config.swap_space_bytes,
96
)
97
98
# Since we use a shared centralized controller, we take the minimum
@@ -107,6 +109,18 @@ class LLMServer:
109
# Initialize the cache.
110
self._run_workers("init_cache_engine", cache_config=self.cache_config)
111
112
+ @classmethod
113
+ def from_server_args(cls, server_args: ServerArgs) -> "LLMServer":
114
+ # Create the server configs.
115
+ server_configs = server_args.create_server_configs()
116
+ parallel_config = server_configs[2]
117
+ # Initialize the cluster.
118
+ distributed_init_method, devices = initialize_cluster(parallel_config)
119
+ # Create the LLM server.
120
+ server = cls(*server_configs, distributed_init_method, devices,
121
+ log_stats=not server_args.disable_log_stats)
122
+ return server
123
+
124
def add_request(
125
self,
126
request_id: str,
examples/offline_inference.py
new
+23
@@ -0,0 +1,23 @@
1
+from cacheflow import LLM, SamplingParams
2
+
3
+
4
+# Sample prompts.
5
+prompts = [
6
+ "Hello, my name is",
7
+ "The president of the United States is",
8
+ "The capital of France is",
9
+ "The future of AI is",
10
+]
11
+# Create a sampling params object.
12
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
13
+
14
+# Create an LLM.
15
+llm = LLM(model="facebook/opt-125m")
16
+# Generate texts from the prompts. The output is a list of RequestOutput objects
17
+# that contain the prompt, generated text, and other information.
18
+outputs = llm.generate(prompts, sampling_params)
19
+# Print the outputs.
20
+for output in outputs:
21
+ prompt = output.prompt
22
+ generated_text = output.outputs[0].text
23
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
examples/simple_server.py
+5
-5
@@ -1,13 +1,13 @@
1
import argparse
2
import uuid
3
4
-from cacheflow import (add_server_arguments, initialize_server_from_args,
5
- SamplingParams)
4
+from cacheflow import ServerArgs, LLMServer, SamplingParams
5
6
7
def main(args: argparse.Namespace):
9
- # Initialize the server.
10
- server = initialize_server_from_args(args)
8
+ # Parse the CLI argument and initialize the server.
9
+ server_args = ServerArgs.from_cli_args(args)
10
+ server = LLMServer.from_server_args(server_args)
11
12
# Test the following prompts.
13
test_prompts = [
@@ -39,6 +39,6 @@ def main(args: argparse.Namespace):
39
40
if __name__ == '__main__':
41
parser = argparse.ArgumentParser(description='Simple CacheFlow server.')
42
- parser = add_server_arguments(parser)
42
+ parser = ServerArgs.add_cli_args(parser)
43
args = parser.parse_args()
44
main(args)