A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Refactor system architecture (#109)
Woosuk Kwon committed
May 20, 2023 at 13:06 UTC
c3442c1f6fabe54adb82d2d676920c5f31b9834e
24 files changed
+1019
-1036
README.md
+8
-5
@@ -10,13 +10,17 @@ pip install -e . # This may take several minutes.
10
## Test simple server
11
12
```bash
13
+# Single-GPU inference.
14
+python examples/simple_server.py # --model <your_model>
15
+
16
+# Multi-GPU inference (e.g., 2 GPUs).
17
ray start --head
14
-python simple_server.py
18
+python examples/simple_server.py -tp 2 # --model <your_model>
19
```
20
21
The detailed arguments for `simple_server.py` can be found by:
22
```bash
19
-python simple_server.py --help
23
+python examples/simple_server.py --help
24
```
25
26
## FastAPI server
@@ -24,12 +28,12 @@ python simple_server.py --help
28
To start the server:
29
```bash
30
ray start --head
27
-python -m cacheflow.http_frontend.fastapi_frontend
31
+python -m cacheflow.entrypoints.fastapi_server # --model <your_model>
32
```
33
34
To test the server:
35
```bash
32
-python -m cacheflow.http_frontend.test_cli_client
36
+python test_cli_client.py
37
```
38
39
## Gradio web server
@@ -55,7 +59,6 @@ Since LLaMA weight is not fully public, we cannot directly download the LLaMA we
59
python src/transformers/models/llama/convert_llama_weights_to_hf.py \
60
--input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path/llama-7b
61
```
58
- Please make sure that `llama` is included in the output directory name.
62
2. For all the commands above, specify the model with `--model /output/path/llama-7b` to load the model. For example:
63
```bash
64
python simple_server.py --model /output/path/llama-7b
cacheflow/__init__.py
new
+19
@@ -0,0 +1,19 @@
1
+from cacheflow.outputs import RequestOutput
2
+from cacheflow.sampling_params import SamplingParams
3
+from cacheflow.server.arg_utils import (
4
+ add_server_arguments,
5
+ create_server_configs_from_args,
6
+ initialize_server_from_args,
7
+)
8
+from cacheflow.server.llm_server import LLMServer
9
+from cacheflow.server.ray_utils import initialize_cluster
10
+
11
+__all__ = [
12
+ "RequestOutput",
13
+ "SamplingParams",
14
+ "LLMServer",
15
+ "add_server_arguments",
16
+ "create_server_configs_from_args",
17
+ "initialize_server_from_args",
18
+ "initialize_cluster",
19
+]
cacheflow/config.py
new
+165
@@ -0,0 +1,165 @@
1
+from typing import Optional
2
+
3
+import torch
4
+from transformers import AutoConfig, PretrainedConfig
5
+
6
+
7
+class ModelConfig:
8
+
9
+ def __init__(
10
+ self,
11
+ model: str,
12
+ download_dir: Optional[str],
13
+ use_np_weights: bool,
14
+ use_dummy_weights: bool,
15
+ dtype: str,
16
+ seed: int,
17
+ ) -> None:
18
+ self.model = model
19
+ self.download_dir = download_dir
20
+ self.use_np_weights = use_np_weights
21
+ self.use_dummy_weights = use_dummy_weights
22
+ self.seed = seed
23
+
24
+ self.hf_config: PretrainedConfig = AutoConfig.from_pretrained(model)
25
+ self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
26
+
27
+ def verify_with_parallel_config(
28
+ self,
29
+ parallel_config: "ParallelConfig",
30
+ ) -> None:
31
+ total_num_attention_heads = self.hf_config.num_attention_heads
32
+ tensor_parallel_size = parallel_config.tensor_parallel_size
33
+ if total_num_attention_heads % tensor_parallel_size != 0:
34
+ raise ValueError(
35
+ f"Total number of attention heads ({total_num_attention_heads})"
36
+ " must be divisible by tensor parallel size "
37
+ f"({tensor_parallel_size}).")
38
+
39
+ total_num_hidden_layers = self.hf_config.num_hidden_layers
40
+ pipeline_parallel_size = parallel_config.pipeline_parallel_size
41
+ if total_num_hidden_layers % pipeline_parallel_size != 0:
42
+ raise ValueError(
43
+ f"Total number of hidden layers ({total_num_hidden_layers}) "
44
+ "must be divisible by pipeline parallel size "
45
+ f"({pipeline_parallel_size}).")
46
+
47
+ def get_hidden_size(self) -> int:
48
+ return self.hf_config.hidden_size
49
+
50
+ def get_head_size(self) -> int:
51
+ # FIXME(woosuk): This may not be true for all models.
52
+ return self.hf_config.hidden_size // self.hf_config.num_attention_heads
53
+
54
+ def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
55
+ total_num_attention_heads = self.hf_config.num_attention_heads
56
+ return total_num_attention_heads // parallel_config.tensor_parallel_size
57
+
58
+ def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
59
+ total_num_hidden_layers = self.hf_config.num_hidden_layers
60
+ return total_num_hidden_layers // parallel_config.pipeline_parallel_size
61
+
62
+
63
+class CacheConfig:
64
+
65
+ def __init__(
66
+ self,
67
+ block_size: int,
68
+ gpu_memory_utilization: float,
69
+ swap_space: int,
70
+ ) -> None:
71
+ self.block_size = block_size
72
+ self.gpu_memory_utilization = gpu_memory_utilization
73
+ self.swap_space = swap_space
74
+
75
+ # Will be set after profiling.
76
+ self.num_gpu_blocks = None
77
+ self.num_cpu_blocks = None
78
+
79
+
80
+class ParallelConfig:
81
+
82
+ def __init__(
83
+ self,
84
+ pipeline_parallel_size: int,
85
+ tensor_parallel_size: int,
86
+ use_ray: bool,
87
+ ) -> None:
88
+ self.pipeline_parallel_size = pipeline_parallel_size
89
+ self.tensor_parallel_size = tensor_parallel_size
90
+ self.use_ray = use_ray
91
+
92
+ self.world_size = pipeline_parallel_size * tensor_parallel_size
93
+ if self.world_size > 1:
94
+ self.use_ray = True
95
+ self._verify_args()
96
+
97
+ def _verify_args(self) -> None:
98
+ if self.pipeline_parallel_size > 1:
99
+ raise NotImplementedError(
100
+ "Pipeline parallelism is not supported yet.")
101
+
102
+
103
+class SchedulerConfig:
104
+
105
+ def __init__(
106
+ self,
107
+ max_num_batched_tokens: int,
108
+ max_num_seqs: int,
109
+ ) -> None:
110
+ self.max_num_batched_tokens = max_num_batched_tokens
111
+ self.max_num_seqs = max_num_seqs
112
+
113
+
114
+_STR_DTYPE_TO_TORCH_DTYPE = {
115
+ "half": torch.float16,
116
+ "float16": torch.float16,
117
+ "float": torch.float32,
118
+ "float32": torch.float32,
119
+ "bfloat16": torch.bfloat16,
120
+}
121
+
122
+
123
+def _get_and_verify_dtype(
124
+ config: PretrainedConfig,
125
+ dtype: str,
126
+) -> torch.dtype:
127
+ # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
128
+ # because config.torch_dtype can be None.
129
+ config_dtype = getattr(config, "torch_dtype", None)
130
+ if config_dtype is None:
131
+ config_dtype = torch.float32
132
+
133
+ dtype = dtype.lower()
134
+ if dtype == "default":
135
+ if config_dtype == torch.float32:
136
+ # Following the common practice, we use float16 for float32 models.
137
+ torch_dtype = torch.float16
138
+ else:
139
+ torch_dtype = config_dtype
140
+ else:
141
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
142
+
143
+ # Verify the dtype.
144
+ if torch_dtype != config_dtype:
145
+ if torch_dtype == torch.float32:
146
+ # Upcasting to float32 is allowed.
147
+ pass
148
+ elif config_dtype == torch.float32:
149
+ # Downcasting from float32 to float16 or bfloat16 is allowed.
150
+ pass
151
+ else:
152
+ # Casting between float16 and bfloat16 is not allowed.
153
+ raise ValueError(
154
+ f"Cannot use {torch_dtype} for {config_dtype} model.")
155
+
156
+ # Check if the GPU supports the dtype.
157
+ if torch_dtype == torch.bfloat16:
158
+ compute_capability = torch.cuda.get_device_capability()
159
+ if compute_capability[0] < 8:
160
+ gpu_name = torch.cuda.get_device_name()
161
+ raise ValueError(
162
+ "Bfloat16 is only supported on GPUs with compute capability "
163
+ f"of at least 8.0. Your {gpu_name} GPU has compute capability "
164
+ f"{compute_capability[0]}.{compute_capability[1]}.")
165
+ return torch_dtype
cacheflow/core/scheduler.py
+80
-84
@@ -2,10 +2,10 @@ import enum
2
import time
3
from typing import Dict, List, Optional, Tuple
4
5
+from cacheflow.config import CacheConfig, SchedulerConfig
6
from cacheflow.core.block_manager import BlockSpaceManager
7
from cacheflow.core.policy import PolicyFactory
8
from cacheflow.logger import init_logger
8
-from cacheflow.sampling_params import SamplingParams
9
from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
10
SequenceGroupMetadata, SequenceOutputs,
11
SequenceStatus)
@@ -28,43 +28,53 @@ class PreemptionMode(enum.Enum):
28
RECOMPUTE = enum.auto()
29
30
31
+class SchedulerOutputs:
32
+
33
+ def __init__(
34
+ self,
35
+ blocks_to_swap_in: Dict[int, int],
36
+ blocks_to_swap_out: Dict[int, int],
37
+ blocks_to_copy: Dict[int, List[int]],
38
+ ) -> None:
39
+ self.blocks_to_swap_in = blocks_to_swap_in
40
+ self.blocks_to_swap_out = blocks_to_swap_out
41
+ self.blocks_to_copy = blocks_to_copy
42
+ # Swap in and swap out should never happen at the same time.
43
+ assert not (blocks_to_swap_in and blocks_to_swap_out)
44
+
45
+ def is_empty(self) -> bool:
46
+ return (not self.blocks_to_swap_in
47
+ and not self.blocks_to_swap_out
48
+ and not self.blocks_to_copy)
49
+
50
+
51
class Scheduler:
52
53
def __init__(
54
self,
35
- controllers: List,
36
- block_size: int,
37
- num_gpu_blocks: int,
38
- num_cpu_blocks: int,
39
- max_num_batched_tokens: int,
40
- max_num_sequences: int,
55
+ scheduler_config: SchedulerConfig,
56
+ cache_config: CacheConfig,
57
log_stats: bool,
58
) -> None:
43
- self.controllers = controllers
44
- self.block_size = block_size
45
- self.num_gpu_blocks = num_gpu_blocks
46
- self.num_cpu_blocks = num_cpu_blocks
47
- self.max_num_batched_tokens = max_num_batched_tokens
48
- self.max_num_sequences = max_num_sequences
59
+ self.scheduler_config = scheduler_config
60
+ self.cache_config = cache_config
61
self.log_stats = log_stats
62
63
# Instantiate the scheduling policy.
64
self.policy = PolicyFactory.get_policy(policy_name='fcfs')
65
# Create the block space manager.
66
self.block_manager = BlockSpaceManager(
55
- block_size=block_size,
56
- num_gpu_blocks=num_gpu_blocks,
57
- num_cpu_blocks=num_cpu_blocks,
67
+ block_size=self.cache_config.block_size,
68
+ num_gpu_blocks=self.cache_config.num_gpu_blocks,
69
+ num_cpu_blocks=self.cache_config.num_cpu_blocks,
70
)
71
72
# Sequence groups in the WAITING state.
73
self.waiting: List[SequenceGroup] = []
74
# Sequence groups in the RUNNING state.
75
self.running: List[SequenceGroup] = []
64
- # Mapping: group_id -> num_steps.
65
- self.num_steps: Dict[int, int] = {}
66
- # Mapping: group_id -> sampling params.
67
- self.sampling_params: Dict[int, SamplingParams] = {}
76
+ # Mapping: request_id -> num_steps.
77
+ self.num_steps: Dict[str, int] = {}
78
# Sequence groups in the SWAPPED state.
79
self.swapped: List[SequenceGroup] = []
80
@@ -72,18 +82,15 @@ class Scheduler:
82
# List[timestamp, num_tokens]
83
self.num_input_tokens: List[Tuple[float, int]] = []
84
75
- def add_sequence_groups(
76
- self,
77
- seq_groups: List[Tuple[SequenceGroup, SamplingParams]],
78
- ) -> None:
85
+ def add_seq_group(self, seq_group: SequenceGroup) -> None:
86
# Add sequence groups to the waiting queue.
80
- for seq_group, sampling_params in seq_groups:
81
- self.waiting.append(seq_group)
82
- self.sampling_params[seq_group.group_id] = sampling_params
87
+ assert seq_group.request_id not in self.num_steps
88
+ self.waiting.append(seq_group)
89
84
- def _schedule(
85
- self,
86
- ) -> Tuple[Dict[int, int], Dict[int, int], Dict[int, List[int]], List[int]]:
90
+ def has_unfinished_seqs(self) -> bool:
91
+ return self.waiting or self.running or self.swapped
92
+
93
+ def _schedule(self) -> Tuple[SchedulerOutputs, List[int]]:
94
# Blocks that need to be swaped or copied before model execution.
95
blocks_to_swap_in: Dict[int, int] = {}
96
blocks_to_swap_out: Dict[int, int] = {}
@@ -136,8 +143,9 @@ class Scheduler:
143
144
# The total number of sequences in the RUNNING state should not
145
# exceed the maximum number of sequences.
139
- num_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
140
- if len(self.running) + num_seqs > self.max_num_sequences:
146
+ num_new_seqs = seq_group.num_seqs(status=SequenceStatus.SWAPPED)
147
+ num_curr_seqs = len(self.running)
148
+ if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
149
break
150
151
seq_group = self.swapped.pop(0)
@@ -151,7 +159,7 @@ class Scheduler:
159
)
160
161
# Join waiting sequences if possible.
154
- prompt_group_ids: List[int] = []
162
+ prompt_group_ids: List[str] = []
163
# NOTE(woosuk): The sequence groups in the SWAPPED state are strictly
164
# prioritized over the sequence groups in the WAITING state.
165
# This is because we want to bound the amount of CPU memory taken by
@@ -172,25 +180,31 @@ class Scheduler:
180
# If the number of batched tokens exceeds the limit, stop.
181
num_prompt_tokens = seq_group.seqs[0].get_len()
182
if (num_batched_tokens + num_prompt_tokens
175
- > self.max_num_batched_tokens):
183
+ > self.scheduler_config.max_num_batched_tokens):
184
break
185
186
# The total number of sequences in the RUNNING state should not
187
# exceed the maximum number of sequences.
180
- num_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
181
- if len(self.running) + num_seqs > self.max_num_sequences:
188
+ num_new_seqs = seq_group.num_seqs(status=SequenceStatus.WAITING)
189
+ num_curr_seqs = len(self.running)
190
+ if num_curr_seqs + num_new_seqs > self.scheduler_config.max_num_seqs:
191
break
192
193
seq_group = self.waiting.pop(0)
194
self._allocate(seq_group)
195
self.running.append(seq_group)
196
num_batched_tokens += num_prompt_tokens
188
- prompt_group_ids.append(seq_group.group_id)
197
+ prompt_group_ids.append(seq_group.request_id)
198
199
+ scheduler_outputs = SchedulerOutputs(
200
+ blocks_to_swap_in=blocks_to_swap_in,
201
+ blocks_to_swap_out=blocks_to_swap_out,
202
+ blocks_to_copy=blocks_to_copy,
203
+ )
204
if not self.log_stats:
191
- return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
192
- prompt_group_ids)
205
+ return scheduler_outputs, prompt_group_ids
206
207
+ # TODO(woosuk): Move the below code to server.
208
now = time.time()
209
if num_batched_tokens > 0:
210
self.num_input_tokens.append((now, num_batched_tokens))
@@ -208,13 +222,16 @@ class Scheduler:
222
else:
223
avg_throughput = 0.0
224
225
+ total_num_gpu_blocks = self.cache_config.num_gpu_blocks
226
num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
212
- num_used_gpu_blocks = self.num_gpu_blocks - num_free_gpu_blocks
213
- gpu_cache_usage = num_used_gpu_blocks / self.num_gpu_blocks
214
- if self.num_cpu_blocks > 0:
227
+ num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
228
+ gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
229
+
230
+ total_num_cpu_blocks = self.cache_config.num_cpu_blocks
231
+ if total_num_cpu_blocks > 0:
232
num_free_cpu_blocks = self.block_manager.get_num_free_cpu_blocks()
216
- num_used_cpu_blocks = self.num_cpu_blocks - num_free_cpu_blocks
217
- cpu_cache_usage = num_used_cpu_blocks / self.num_cpu_blocks
233
+ num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
234
+ cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
235
else:
236
cpu_cache_usage = 0.0
237
@@ -225,27 +242,18 @@ class Scheduler:
242
f"Pending: {len(self.waiting)} reqs, "
243
f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
244
f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
245
+ return scheduler_outputs, prompt_group_ids
246
229
- return (blocks_to_swap_in, blocks_to_swap_out, blocks_to_copy,
230
- prompt_group_ids)
231
-
232
- def step(self) -> List[SequenceGroup]:
247
+ def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
248
# Schedule sequence groups.
249
# This function call changes the internal states of the scheduler
250
# such as self.running, self.swapped, and self.waiting.
236
- scheduler_output = self._schedule()
237
- blocks_to_swap_in = scheduler_output[0]
238
- blocks_to_swap_out = scheduler_output[1]
239
- blocks_to_copy = scheduler_output[2]
240
- prompt_group_ids = scheduler_output[3]
251
+ scheduler_outputs, prompt_group_ids = self._schedule()
252
253
# Create input data structures.
254
seq_group_metadata_list: List[SequenceGroupMetadata] = []
244
- updated_seq_groups: List[SequenceGroup] = self.running.copy()
245
-
255
for seq_group in self.running:
247
- group_id = seq_group.group_id
248
- is_prompt = group_id in prompt_group_ids
256
+ is_prompt = seq_group.request_id in prompt_group_ids
257
258
seq_data: Dict[int, List[SequenceData]] = {}
259
block_tables: Dict[int, List[int]] = {}
@@ -255,36 +263,24 @@ class Scheduler:
263
block_tables[seq_id] = self.block_manager.get_block_table(seq)
264
265
seq_group_metadata = SequenceGroupMetadata(
258
- group_id=group_id,
266
+ request_id=seq_group.request_id,
267
is_prompt=is_prompt,
268
seq_data=seq_data,
261
- sampling_params=self.sampling_params[group_id],
269
+ sampling_params=seq_group.sampling_params,
270
block_tables=block_tables,
271
)
272
seq_group_metadata_list.append(seq_group_metadata)
273
+ return seq_group_metadata_list, scheduler_outputs
274
266
- # Execute the first stage of the pipeline.
267
- if seq_group_metadata_list or blocks_to_swap_in or blocks_to_swap_out:
268
- # Swap in and swap out should never happen at the same time.
269
- assert not (blocks_to_swap_in and blocks_to_swap_out)
270
- self.controllers[0].execute_stage(
271
- seq_group_metadata_list,
272
- blocks_to_swap_in=blocks_to_swap_in,
273
- blocks_to_swap_out=blocks_to_swap_out,
274
- blocks_to_copy=blocks_to_copy,
275
- )
276
-
277
- return updated_seq_groups
278
-
279
- def post_step(
275
+ def update(
276
self,
277
seq_outputs: Dict[int, SequenceOutputs],
282
- ) -> None:
278
+ ) -> List[SequenceGroup]:
279
# Update the running sequences and free blocks.
280
for seq_group in self.running:
285
- group_id = seq_group.group_id
286
- self.num_steps[group_id] += 1
287
- stop_token_ids = self.sampling_params[group_id].stop_token_ids
281
+ request_id = seq_group.request_id
282
+ self.num_steps[request_id] += 1
283
+ stop_token_ids = seq_group.sampling_params.stop_token_ids
284
285
# Process beam search results before processing the next tokens.
286
for seq in seq_group.seqs:
@@ -316,12 +312,13 @@ class Scheduler:
312
continue
313
314
# Check if the sequence has reached the maximum number of steps.
319
- max_num_steps = self.sampling_params[group_id].max_tokens
320
- if self.num_steps[group_id] == max_num_steps:
315
+ max_num_steps = seq_group.sampling_params.max_tokens
316
+ if self.num_steps[request_id] == max_num_steps:
317
self._free_seq(seq)
318
continue
319
320
# Update the running sequences.
321
+ updated = self.running.copy()
322
running: List[SequenceGroup] = []
323
for seq_group in self.running:
324
if seq_group.is_finished():
@@ -329,13 +326,14 @@ class Scheduler:
326
else:
327
running.append(seq_group)
328
self.running = running
329
+ return updated
330
331
def _allocate(self, seq_group: SequenceGroup) -> None:
332
self.block_manager.allocate(seq_group)
333
for seq in seq_group.seqs:
334
seq.status = SequenceStatus.RUNNING
337
- if seq_group.group_id not in self.num_steps:
338
- self.num_steps[seq_group.group_id] = 0
335
+ if seq_group.request_id not in self.num_steps:
336
+ self.num_steps[seq_group.request_id] = 0
337
338
def _append_slot(
339
self,
@@ -410,9 +408,7 @@ class Scheduler:
408
self.block_manager.free(seq)
409
410
def _free_seq_group(self, seq_group: SequenceGroup) -> None:
413
- group_id = seq_group.group_id
414
- del self.num_steps[group_id]
415
- del self.sampling_params[group_id]
411
+ del self.num_steps[seq_group.request_id]
412
413
def _swap_in(
414
self,
cacheflow/core/server.py
deleted
-302
@@ -1,302 +0,0 @@
1
-import argparse
2
-import random
3
-from typing import List, Optional, Tuple
4
-
5
-try:
6
- import ray
7
-except ImportError:
8
- ray = None
9
-import numpy as np
10
-import torch
11
-
12
-from cacheflow.core.scheduler import Scheduler
13
-from cacheflow.frontend.simple_frontend import SimpleFrontend
14
-from cacheflow.logger import init_logger
15
-from cacheflow.sampling_params import SamplingParams
16
-from cacheflow.sequence import SequenceGroup
17
-from cacheflow.worker.controller import Controller, DeviceID
18
-
19
-logger = init_logger(__name__)
20
-
21
-
22
-class Server:
23
-
24
- def __init__(
25
- self,
26
- model: str,
27
- cache_dir: Optional[str],
28
- use_dummy_weights: bool,
29
- use_np_cache: bool,
30
- pipeline_parallel_size: int,
31
- tensor_parallel_size: int,
32
- block_size: int,
33
- dtype: str,
34
- seed: int,
35
- swap_space: int,
36
- gpu_memory_utilization: float,
37
- max_num_batched_tokens: int,
38
- max_num_sequences: int,
39
- num_nodes: int,
40
- num_devices_per_node: int,
41
- distributed_init_method: str,
42
- all_stage_devices: List[List[DeviceID]],
43
- use_ray: bool,
44
- log_stats: bool,
45
- ):
46
- logger.info(
47
- "Initializing a server with config: "
48
- f"model={model!r}, "
49
- f"dtype={dtype}, "
50
- f"use_dummy_weights={use_dummy_weights}, "
51
- f"cache_dir={cache_dir!r}, "
52
- f"use_np_cache={use_np_cache}, "
53
- f"tensor_parallel_size={tensor_parallel_size}, "
54
- f"seed={seed})"
55
- )
56
- self.num_nodes = num_nodes
57
- self.num_devices_per_node = num_devices_per_node
58
- self.world_size = pipeline_parallel_size * tensor_parallel_size
59
-
60
- if not use_ray:
61
- assert self.world_size == 1, (
62
- "Only support single GPU without Ray.")
63
-
64
- # Create a controller for each pipeline stage.
65
- self.controllers: List[Controller] = []
66
- for i in range(pipeline_parallel_size):
67
- controller = Controller(
68
- stage_id=i,
69
- stage_devices=all_stage_devices[i],
70
- world_size=self.world_size,
71
- pipeline_parallel_size=pipeline_parallel_size,
72
- tensor_parallel_size=tensor_parallel_size,
73
- distributed_init_method=distributed_init_method,
74
- model_name=model,
75
- dtype=dtype,
76
- seed=seed,
77
- cache_dir=cache_dir,
78
- use_dummy_weights=use_dummy_weights,
79
- use_np_cache=use_np_cache,
80
- max_num_batched_tokens=max_num_batched_tokens,
81
- max_num_sequences=max_num_sequences,
82
- use_ray=use_ray,
83
- )
84
- self.controllers.append(controller)
85
-
86
- # Initialize cache engine.
87
- all_worker_num_available_blocks = []
88
- for controller in self.controllers:
89
- all_worker_num_available_blocks.extend(
90
- controller.get_num_available_blocks(
91
- block_size, swap_space, gpu_memory_utilization)
92
- )
93
- # Since we use a shared centralized controller, we take the minimum
94
- # number of blocks across all workers to make sure all the memory
95
- # operators can be applied to all workers.
96
- self.num_gpu_blocks = np.min([b[0] for b in all_worker_num_available_blocks])
97
- self.num_cpu_blocks = np.min([b[1] for b in all_worker_num_available_blocks])
98
- logger.info(f'# GPU blocks: {self.num_gpu_blocks}, '
99
- f'# CPU blocks: {self.num_cpu_blocks}')
100
- for controller in self.controllers:
101
- controller.init_cache_engine(block_size, self.num_gpu_blocks,
102
- self.num_cpu_blocks)
103
-
104
- # Create a scheduler.
105
- self.scheduler = Scheduler(
106
- controllers=self.controllers,
107
- block_size=block_size,
108
- num_gpu_blocks=self.num_gpu_blocks,
109
- num_cpu_blocks=self.num_cpu_blocks,
110
- max_num_batched_tokens=max_num_batched_tokens,
111
- max_num_sequences=max_num_sequences,
112
- log_stats=log_stats,
113
- )
114
- # Connect the controllers.
115
- for i in range(len(self.controllers) - 1):
116
- self.controllers[i].set_next(self.controllers[i + 1])
117
- self.controllers[-1].set_next(self.scheduler)
118
-
119
- def add_sequence_groups(
120
- self,
121
- sequence_groups: List[Tuple[SequenceGroup, SamplingParams]]
122
- ):
123
- self.scheduler.add_sequence_groups(sequence_groups)
124
-
125
- def step(self):
126
- return self.scheduler.step()
127
-
128
- def has_unfinished_requests(self):
129
- return (self.scheduler.waiting or self.scheduler.running or
130
- self.scheduler.swapped)
131
-
132
-
133
-def initialize_cluster(
134
- use_ray: bool = False,
135
- address: Optional[str] = None,
136
- pipeline_parallel_size: int = 1,
137
- tensor_parallel_size: int = 1,
138
-) -> Tuple[int, int, str, List[List[DeviceID]]]:
139
- # Initialize cluster locally.
140
- if not use_ray:
141
- assert pipeline_parallel_size * tensor_parallel_size == 1, (
142
- "Only support single GPU without Ray.")
143
- num_nodes = 1
144
- num_devices_per_node = torch.cuda.device_count()
145
- port = random.randint(10000, 20000)
146
- # We need to setup the distributed init method to make sure
147
- # the distributed megatron code (e.g., get world size) works correctly.
148
- distributed_init_method = f"tcp://localhost:{port}"
149
- all_stage_devices = [[(0, None, 0)]]
150
- return (num_nodes, num_devices_per_node, distributed_init_method,
151
- all_stage_devices)
152
-
153
- assert ray is not None, (
154
- "Ray is not installed. Please install Ray to use distributed "
155
- "serving.")
156
-
157
- # Connect to a ray cluster.
158
- ray.init(address=address)
159
-
160
- # Assume we have a uniform cluster that each node has the same number of
161
- # GPUs for now.
162
- valid_node_resources = []
163
- num_devices_per_node = None
164
- for node in ray.nodes():
165
- if (not node['Alive']) or node['Resources']['GPU'] <= 0:
166
- continue
167
- if num_devices_per_node is None:
168
- num_devices_per_node = node['Resources']['GPU']
169
- else:
170
- assert num_devices_per_node == node['Resources']['GPU'], (
171
- "The number of GPUs per node is not uniform.")
172
- for key in node['Resources']:
173
- if key.startswith('node:'):
174
- valid_node_resources.append(key)
175
-
176
- num_nodes = len(valid_node_resources)
177
-
178
- assert (pipeline_parallel_size * tensor_parallel_size
179
- <= num_nodes * num_devices_per_node), (
180
- "The number of required GPUs exceeds the total number of "
181
- "available GPUs.")
182
- if tensor_parallel_size >= num_devices_per_node:
183
- assert tensor_parallel_size % num_devices_per_node == 0, (
184
- "The number of tensor parallelism is not divisible by the "
185
- "number of GPUs per node.")
186
- else:
187
- assert num_devices_per_node % tensor_parallel_size == 0, (
188
- "The number of GPUs per node is not divisible by the number "
189
- "of tensor parallelism.")
190
-
191
- # Assign GPUs to pipeline stages.
192
- rank = 0
193
- current_node_id = 0
194
- current_device_id = 0
195
- distributed_init_method = None
196
- all_stage_devices = []
197
-
198
- for i in range(pipeline_parallel_size):
199
- stage_devices = []
200
- for j in range(tensor_parallel_size):
201
- node_resource = valid_node_resources[current_node_id]
202
- stage_devices.append((rank, node_resource, current_device_id))
203
- if distributed_init_method is None:
204
- ip = node_resource.split("node:")[-1]
205
- port = random.randint(10000, 20000)
206
- distributed_init_method = f"tcp://{ip}:{port}"
207
- rank += 1
208
- current_device_id += 1
209
- if current_device_id >= num_devices_per_node:
210
- current_node_id += 1
211
- current_device_id = 0
212
- all_stage_devices.append(stage_devices)
213
-
214
- return (num_nodes, num_devices_per_node, distributed_init_method,
215
- all_stage_devices)
216
-
217
-
218
-_GiB = 1 << 30
219
-
220
-
221
-def add_server_arguments(parser: argparse.ArgumentParser):
222
- """Shared arguments for CacheFlow servers."""
223
- # Model arguments
224
- parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
225
- parser.add_argument('--cache-dir', type=str, default=None,
226
- help='cache dir to download and load the weights, '
227
- 'default to the default cache dir of huggingface')
228
- parser.add_argument('--use-np-cache', action='store_true',
229
- help='save a numpy copy of model weights for faster loading')
230
- parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
231
- # TODO(woosuk): Support FP32 for debugging.
232
- parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
233
- help=('data type for model weights and activations. '
234
- 'The "default" option will use FP16 precision '
235
- 'for FP32 and FP16 models, and BF16 precision '
236
- 'for BF16 models.'))
237
- # Parallel arguments
238
- parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
239
- parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
240
- parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
241
- # KV cache arguments
242
- parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
243
- # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
244
- parser.add_argument('--seed', type=int, default=0, help='random seed')
245
- parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
246
- parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
247
- parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
248
- parser.add_argument('--max-num-sequences', type=int, default=256, help='maximum number of sequences per iteration')
249
- parser.add_argument('--log-stats', action='store_true', help='log system statistics')
250
- return parser
251
-
252
-
253
-def process_server_arguments(args: argparse.Namespace):
254
- """Post process the parsed arguments."""
255
- if args.pipeline_parallel_size * args.tensor_parallel_size > 1:
256
- args.use_ray = True
257
- args.swap_space = args.swap_space * _GiB
258
- args.max_num_sequences = min(args.max_num_sequences, args.max_num_batched_tokens)
259
- return args
260
-
261
-
262
-def init_local_server_and_frontend_with_arguments(args: argparse.Namespace):
263
- # TODO(zhuohan): Support pipeline parallelism.
264
- assert args.pipeline_parallel_size == 1, (
265
- 'Pipeline parallelism is not supported yet.')
266
-
267
- (num_nodes, num_devices_per_node, distributed_init_method,
268
- all_stage_devices) = (
269
- initialize_cluster(
270
- use_ray=args.use_ray,
271
- pipeline_parallel_size=args.pipeline_parallel_size,
272
- tensor_parallel_size=args.tensor_parallel_size))
273
-
274
- # Create a server.
275
- server = Server(
276
- model=args.model,
277
- cache_dir=args.cache_dir,
278
- use_dummy_weights=args.use_dummy_weights,
279
- use_np_cache=args.use_np_cache,
280
- pipeline_parallel_size=args.pipeline_parallel_size,
281
- tensor_parallel_size=args.tensor_parallel_size,
282
- block_size=args.block_size,
283
- dtype=args.dtype,
284
- seed=args.seed,
285
- swap_space=args.swap_space,
286
- gpu_memory_utilization=args.gpu_memory_utilization,
287
- max_num_batched_tokens=args.max_num_batched_tokens,
288
- max_num_sequences=args.max_num_sequences,
289
- num_nodes=num_nodes,
290
- num_devices_per_node=num_devices_per_node,
291
- distributed_init_method=distributed_init_method,
292
- all_stage_devices=all_stage_devices,
293
- use_ray=args.use_ray,
294
- log_stats=args.log_stats,
295
- )
296
-
297
- # Create a frontend.
298
- frontend = SimpleFrontend(
299
- model_name=args.model,
300
- block_size=args.block_size,
301
- )
302
- return server, frontend
cacheflow/entrypoints/fastapi_server.py
new
+128
@@ -0,0 +1,128 @@
1
+import argparse
2
+import asyncio
3
+import json
4
+import time
5
+from typing import Any, Dict
6
+import uuid
7
+
8
+from fastapi import FastAPI, Request
9
+from fastapi.responses import StreamingResponse
10
+import ray
11
+import uvicorn
12
+
13
+from cacheflow.outputs import RequestOutput
14
+from cacheflow.sampling_params import SamplingParams
15
+from cacheflow.server.arg_utils import (
16
+ add_server_arguments, create_server_configs_from_args)
17
+from cacheflow.server.llm_server import LLMServer
18
+from cacheflow.server.ray_utils import initialize_cluster
19
+
20
+TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
21
+app = FastAPI()
22
+
23
+
24
+class FastAPIServer:
25
+
26
+ def __init__(self, server_use_ray: bool, *args, **kwargs) -> None:
27
+ if server_use_ray:
28
+ remote_server_class = ray.remote(num_cpus=0)(LLMServer)
29
+ else:
30
+ remote_server_class = ray.remote(num_gpus=1)(LLMServer)
31
+ self.server = remote_server_class.remote(*args, **kwargs)
32
+
33
+ # Request id -> request output.
34
+ self.request_outputs: Dict[str, RequestOutput] = {}
35
+ # Request id -> event to notify that there is new output.
36
+ self.request_events: Dict[str, asyncio.Event] = {}
37
+ self.is_server_running = False
38
+
39
+ async def server_step(self):
40
+ self.is_server_running = True
41
+ request_outputs = await self.server.step.remote()
42
+ self.is_server_running = False
43
+ # Notify the waiting coroutines that there are new outputs ready.
44
+ for request_output in request_outputs:
45
+ request_id = request_output.request_id
46
+ self.request_outputs[request_id] = request_output
47
+ self.request_events[request_id].set()
48
+
49
+ async def generate(self, request_dict: Dict[str, Any]):
50
+ # Preprocess the request.
51
+ arrival_time = time.time()
52
+ prompt = request_dict.pop("prompt")
53
+ sampling_params = SamplingParams(**request_dict)
54
+
55
+ # Create an event to notify us that there is new output from the
56
+ # cacheflow server.
57
+ request_id = str(uuid.uuid4().hex[:8])
58
+ request_event = asyncio.Event()
59
+ self.request_events[request_id] = request_event
60
+
61
+ # Add the request into the cacheflow server's waiting queue.
62
+ await self.server.add_request.remote(
63
+ request_id, prompt, sampling_params, arrival_time=arrival_time)
64
+
65
+ # The cacheflow server does not have a background loop that keeps
66
+ # processing incoming requests. Therefore, we need to keep kicking
67
+ # the server to process the requests.
68
+ while True:
69
+ # Kick the server if the server is not running.
70
+ if not self.is_server_running:
71
+ await self.server_step()
72
+
73
+ # Wait for new output. The group_event will be set in server_step
74
+ # when there is new output available for the sequence group.
75
+ # Added a timeout to prevent deadlock.
76
+ try:
77
+ await asyncio.wait_for(request_event.wait(),
78
+ timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
79
+ except asyncio.TimeoutError:
80
+ continue
81
+ # Reset the event to wait for the next output.
82
+ request_event.clear()
83
+
84
+ # Decode and return new outputs.
85
+ request_output = self.request_outputs[request_id]
86
+ prompt = request_output.prompt
87
+ text_outputs = [
88
+ prompt + output.text
89
+ for output in request_output.outputs
90
+ ]
91
+ ret = {
92
+ "text": text_outputs,
93
+ "error": 0,
94
+ }
95
+ yield (json.dumps(ret) + "\0").encode("utf-8")
96
+
97
+ # Once finished, release the resources of the sequence group.
98
+ if request_output.done:
99
+ del self.request_outputs[request_id]
100
+ del self.request_events[request_id]
101
+ # Kick the server if the server is not running. This is to
102
+ # prevent that there are still requests in server's waiting
103
+ # queue to be executed.
104
+ if not self.is_server_running:
105
+ await self.server_step()
106
+ break
107
+
108
+
109
+@app.post("/generate")
110
+async def generate_stream(request: Request):
111
+ request_dict = await request.json()
112
+ return StreamingResponse(server.generate(request_dict))
113
+
114
+
115
+if __name__ == "__main__":
116
+ parser = argparse.ArgumentParser()
117
+ parser.add_argument("--host", type=str, default="localhost")
118
+ parser.add_argument("--port", type=int, default=10002)
119
+ parser = add_server_arguments(parser)
120
+ args = parser.parse_args()
121
+
122
+ server_configs = create_server_configs_from_args(args)
123
+ parallel_config = server_configs[2]
124
+ distributed_init_method, stage_devices = initialize_cluster(parallel_config)
125
+
126
+ server = FastAPIServer(
127
+ args.use_ray, *server_configs, distributed_init_method, stage_devices)
128
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/frontend/fastapi_frontend.py
deleted
-201
@@ -1,201 +0,0 @@
1
-import argparse
2
-import asyncio
3
-import json
4
-import time
5
-from typing import List, Dict, Optional
6
-
7
-from fastapi import FastAPI, Request
8
-from fastapi.responses import StreamingResponse
9
-import ray
10
-import uvicorn
11
-
12
-from cacheflow.core.server import (Server, add_server_arguments,
13
- process_server_arguments,
14
- initialize_cluster)
15
-from cacheflow.frontend.utils import get_tokenizer
16
-from cacheflow.sampling_params import SamplingParams
17
-from cacheflow.sequence import Sequence, SequenceGroup
18
-from cacheflow.utils import Counter
19
-from cacheflow.worker.controller import DeviceID
20
-
21
-TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
22
-app = FastAPI()
23
-
24
-
25
-class FastAPIServer:
26
- def __init__(
27
- self,
28
- model: str,
29
- cache_dir: Optional[str],
30
- use_np_cache: bool,
31
- pipeline_parallel_size: int,
32
- tensor_parallel_size: int,
33
- block_size: int,
34
- dtype: str,
35
- seed: int,
36
- swap_space: int,
37
- gpu_memory_utilization: float,
38
- max_num_batched_tokens: int,
39
- max_num_sequences: int,
40
- num_nodes: int,
41
- num_devices_per_node: int,
42
- distributed_init_method: str,
43
- all_stage_devices: List[List[DeviceID]],
44
- server_use_ray: bool,
45
- log_stats: bool,
46
- ):
47
- self.block_size = block_size
48
-
49
- self.tokenizer = get_tokenizer(model)
50
- self.seq_group_counter = Counter()
51
- self.seq_counter = Counter()
52
- if server_use_ray:
53
- remote_server_class = ray.remote(num_cpus=0)(Server)
54
- else:
55
- remote_server_class = ray.remote(num_gpus=1)(Server)
56
- self.server = remote_server_class.remote(
57
- model=model,
58
- cache_dir=cache_dir,
59
- use_dummy_weights=False,
60
- use_np_cache=use_np_cache,
61
- pipeline_parallel_size=pipeline_parallel_size,
62
- tensor_parallel_size=tensor_parallel_size,
63
- block_size=block_size,
64
- dtype=dtype,
65
- seed=seed,
66
- swap_space=swap_space,
67
- gpu_memory_utilization=gpu_memory_utilization,
68
- max_num_batched_tokens=max_num_batched_tokens,
69
- max_num_sequences=max_num_sequences,
70
- num_nodes=num_nodes,
71
- num_devices_per_node=num_devices_per_node,
72
- distributed_init_method=distributed_init_method,
73
- all_stage_devices=all_stage_devices,
74
- use_ray=server_use_ray,
75
- log_stats=log_stats,
76
- )
77
-
78
- self.running_seq_groups: Dict[int, SequenceGroup] = {}
79
- self.sequence_group_events: Dict[int, asyncio.Event] = {}
80
- self.is_server_running = False
81
-
82
- async def server_step(self):
83
- self.is_server_running = True
84
- updated_seq_groups = await self.server.step.remote()
85
- self.is_server_running = False
86
- # Notify the waiting coroutines that there are new outputs ready.
87
- for seq_group in updated_seq_groups:
88
- group_id = seq_group.group_id
89
- self.running_seq_groups[group_id] = seq_group
90
- self.sequence_group_events[group_id].set()
91
-
92
- async def generate(self, request_dict: Dict):
93
- # Preprocess the request.
94
- prompt = request_dict.pop("prompt")
95
- sampling_params = SamplingParams(**request_dict)
96
- sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
97
- token_ids = self.tokenizer.encode(prompt)
98
- seqs: List[Sequence] = []
99
- for _ in range(sampling_params.n):
100
- seq_id = next(self.seq_counter)
101
- seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
102
- seqs.append(seq)
103
-
104
- arrival_time = time.time()
105
- group_id = next(self.seq_group_counter)
106
- seq_group = SequenceGroup(group_id, seqs, arrival_time)
107
- # Create an event to notify us that there is new output from the
108
- # cacheflow server.
109
- group_event = asyncio.Event()
110
- self.running_seq_groups[group_id] = seq_group
111
- self.sequence_group_events[group_id] = group_event
112
- # Add the request into the cacheflow server's waiting queue.
113
- await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
114
- # The cacheflow server does not have a background loop that keeps
115
- # processing incoming requests. Therefore, we need to keep kicking
116
- # the server to process the requests.
117
- while True:
118
- # Kick the server if the server is not running.
119
- if not self.is_server_running:
120
- await self.server_step()
121
- # Wait for new output. The group_event will be set in server_step
122
- # when there is new output available for the sequence group.
123
- # Added a timeout to prevent deadlock.
124
- try:
125
- await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
126
- except asyncio.TimeoutError:
127
- continue
128
- # Reset the event to wait for the next output.
129
- group_event.clear()
130
- # Decode and return new outputs
131
- seq_group = self.running_seq_groups[group_id]
132
- all_outputs = []
133
- for seq in seq_group.seqs:
134
- token_ids = seq.get_token_ids()
135
- output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
136
- all_outputs.append(output)
137
- ret = {
138
- "text": all_outputs,
139
- "error": 0,
140
- }
141
- yield (json.dumps(ret) + "\0").encode("utf-8")
142
-
143
- # Once finished, release the resources of the sequence group.
144
- if seq_group.is_finished():
145
- del self.running_seq_groups[group_id]
146
- del self.sequence_group_events[group_id]
147
- # Kick the server if the server is not running. This is to
148
- # prevent that there are still requests in server's waiting
149
- # queue to be executed.
150
- if not self.is_server_running:
151
- await self.server_step()
152
- break
153
-
154
-
155
-@app.post("/generate")
156
-async def generate_stream(request: Request):
157
- request_dict = await request.json()
158
- return StreamingResponse(server.generate(request_dict))
159
-
160
-
161
-if __name__ == "__main__":
162
- parser = argparse.ArgumentParser()
163
- parser.add_argument("--host", type=str, default="localhost")
164
- parser.add_argument("--port", type=int, default=10002)
165
- parser = add_server_arguments(parser)
166
- args = parser.parse_args()
167
- args = process_server_arguments(args)
168
-
169
- # TODO(zhuohan): Support pipeline parallelism.
170
- assert args.pipeline_parallel_size == 1, (
171
- 'Pipeline parallelism is not supported yet.')
172
-
173
- (num_nodes, num_devices_per_node, distributed_init_method,
174
- all_stage_devices) = (
175
- initialize_cluster(
176
- use_ray=True,
177
- pipeline_parallel_size=args.pipeline_parallel_size,
178
- tensor_parallel_size=args.tensor_parallel_size))
179
-
180
- server = FastAPIServer(
181
- model=args.model,
182
- cache_dir=args.cache_dir,
183
- use_np_cache=args.use_np_cache,
184
- pipeline_parallel_size=args.pipeline_parallel_size,
185
- tensor_parallel_size=args.tensor_parallel_size,
186
- block_size=args.block_size,
187
- dtype=args.dtype,
188
- seed=args.seed,
189
- swap_space=args.swap_space,
190
- gpu_memory_utilization=args.gpu_memory_utilization,
191
- max_num_batched_tokens=args.max_num_batched_tokens,
192
- max_num_sequences=args.max_num_sequences,
193
- num_nodes=num_nodes,
194
- num_devices_per_node=num_devices_per_node,
195
- distributed_init_method=distributed_init_method,
196
- all_stage_devices=all_stage_devices,
197
- server_use_ray=args.use_ray,
198
- log_stats=args.log_stats,
199
- )
200
-
201
- uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/frontend/simple_frontend.py
deleted
-72
@@ -1,72 +0,0 @@
1
-import time
2
-from typing import List, Optional, Tuple
3
-
4
-from cacheflow.frontend.utils import get_tokenizer
5
-from cacheflow.logger import init_logger
6
-from cacheflow.sampling_params import SamplingParams
7
-from cacheflow.sequence import Sequence, SequenceGroup
8
-from cacheflow.utils import Counter
9
-
10
-logger = init_logger(__name__)
11
-
12
-
13
-class SimpleFrontend:
14
-
15
- def __init__(
16
- self,
17
- model_name: str,
18
- block_size: int,
19
- ) -> None:
20
- self.block_size = block_size
21
-
22
- self.tokenizer = get_tokenizer(model_name)
23
- self.seq_group_counter = Counter()
24
- self.seq_counter = Counter()
25
- self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
26
-
27
- def add_eos_token(self, sampling_params: SamplingParams) -> SamplingParams:
28
- # Stop generation when we see an EOS token.
29
- sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
30
- return sampling_params
31
-
32
- def query(
33
- self,
34
- prompt: str,
35
- sampling_params: SamplingParams,
36
- ) -> None:
37
- token_ids = self.tokenizer.encode(prompt)
38
- self._add_query(prompt, token_ids, sampling_params)
39
-
40
- def _add_query(
41
- self,
42
- prompt: str,
43
- token_ids: List[int],
44
- sampling_params: SamplingParams,
45
- arrival_time: Optional[float] = None,
46
- ) -> None:
47
- if arrival_time is None:
48
- arrival_time = time.time()
49
- seqs: List[Sequence] = []
50
- for _ in range(sampling_params.n):
51
- seq_id = next(self.seq_counter)
52
- seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
53
- seqs.append(seq)
54
-
55
- group_id = next(self.seq_group_counter)
56
- seq_group = SequenceGroup(group_id, seqs, arrival_time)
57
- self.inputs.append((seq_group, sampling_params))
58
-
59
- def get_inputs(self) -> List[Tuple[SequenceGroup, SamplingParams]]:
60
- inputs = self.inputs
61
- self.inputs = []
62
- return inputs
63
-
64
- def print_response(
65
- self,
66
- seq_group: SequenceGroup,
67
- ) -> None:
68
- for seq in seq_group.seqs:
69
- token_ids = seq.get_token_ids()
70
- output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
71
- output = output.strip()
72
- logger.info(f"Seq {seq.seq_id}: {output!r}")
cacheflow/model_executor/__init__.py
+1
-3
@@ -1,12 +1,10 @@
1
from cacheflow.model_executor.input_metadata import InputMetadata
2
from cacheflow.model_executor.model_loader import get_model
3
-from cacheflow.model_executor.utils import (set_random_seed,
4
- get_cache_block_size)
3
+from cacheflow.model_executor.utils import set_random_seed
4
5
6
__all__ = [
7
"InputMetadata",
9
- "get_cache_block_size",
8
"get_model",
9
"set_random_seed",
10
]
cacheflow/model_executor/layers/attention.py
+1
-1
@@ -10,9 +10,9 @@ from cacheflow import cache_ops
10
from cacheflow import pos_encoding_ops
11
from cacheflow.model_executor.input_metadata import InputMetadata
12
13
-
13
_SUPPORTED_HEAD_SIZES = [32, 64, 80, 96, 128, 160, 192, 256]
14
15
+
16
class GPTCacheFlowAttention(nn.Module):
17
"""GPT-style multi-head attention.
18
cacheflow/model_executor/model_loader.py
+12
-43
@@ -1,16 +1,13 @@
1
"""Utilities for selecting and loading models."""
2
-from typing import Optional
3
-
2
import torch
3
import torch.nn as nn
6
-from transformers import AutoConfig, PretrainedConfig
4
+from transformers import PretrainedConfig
5
6
+from cacheflow.config import ModelConfig
7
from cacheflow.model_executor.models import (
8
GPT2LMHeadModel, GPTNeoXForCausalLM, LlamaForCausalLM, OPTForCausalLM)
10
-from cacheflow.model_executor.utils import get_torch_dtype
9
from cacheflow.model_executor.weight_utils import initialize_dummy_weights
10
13
-
11
# TODO(woosuk): Lazy-load the model classes.
12
_MODEL_REGISTRY = {
13
"GPT2LMHeadModel": GPT2LMHeadModel,
@@ -19,6 +16,7 @@ _MODEL_REGISTRY = {
16
"OPTForCausalLM": OPTForCausalLM,
17
}
18
19
+
20
def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
21
architectures = getattr(config, "architectures", [])
22
for arch in architectures:
@@ -30,51 +28,22 @@ def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
28
)
29
30
33
-def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
34
- # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
35
- # because config.torch_dtype can be None.
36
- config_dtype = getattr(config, "torch_dtype", None)
37
- if config_dtype is None:
38
- config_dtype = torch.float32
39
- if dtype == "default":
40
- if config_dtype == torch.float32:
41
- # Following the common practice, we use float16 for float32 models.
42
- torch_dtype = torch.float16
43
- else:
44
- torch_dtype = config_dtype
45
- else:
46
- torch_dtype = get_torch_dtype(dtype)
47
- if torch_dtype != config_dtype and config_dtype != torch.float32:
48
- # TODO(woosuk): Allow using float16 for bfloat16 models and
49
- # vice versa. Print a warning message and continue.
50
- raise ValueError(
51
- f"Cannot use {torch_dtype} for {config_dtype} model.")
52
- return torch_dtype
53
-
54
-
55
-def get_model(
56
- model_name: str,
57
- dtype: str,
58
- cache_dir: Optional[str],
59
- use_dummy_weights: bool,
60
- use_np_cache: bool,
61
-) -> nn.Module:
62
- config = AutoConfig.from_pretrained(model_name)
63
- torch_dtype = _get_dtype(config, dtype)
64
- torch.set_default_dtype(torch_dtype)
65
- model_class = _get_model_architecture(config)
31
+def get_model(model_config: ModelConfig) -> nn.Module:
32
+ model_class = _get_model_architecture(model_config.hf_config)
33
+ torch.set_default_dtype(model_config.dtype)
34
35
# Create a model instance.
36
# The weights will be initialized as empty tensors.
69
- model = model_class(config)
70
- if use_dummy_weights:
37
+ model = model_class(model_config.hf_config)
38
+ if model_config.use_dummy_weights:
39
model = model.cuda()
40
# NOTE(woosuk): For accurate performance evaluation, we assign
41
# random values to the weights.
42
initialize_dummy_weights(model)
43
else:
44
# Load the weights from the cached or downloaded files.
77
- model.load_weights(model_name, cache_dir, use_np_cache)
45
+ model.load_weights(
46
+ model_config.model, model_config.download_dir,
47
+ model_config.use_np_weights)
48
model = model.cuda()
79
- return model.eval(), torch_dtype
80
-
49
+ return model.eval()
cacheflow/model_executor/utils.py
-35
@@ -1,6 +1,5 @@
1
"""Utils for model executor."""
2
import random
3
-from typing import Union
3
4
import numpy as np
5
import torch
@@ -9,28 +8,6 @@ from cacheflow.model_executor.parallel_utils.parallel_state import model_paralle
8
from cacheflow.model_executor.parallel_utils.tensor_parallel import model_parallel_cuda_manual_seed
9
10
12
-_STR_DTYPE_TO_TORCH_DTYPE = {
13
- "half": torch.half,
14
- "float": torch.float,
15
- "float16": torch.float16,
16
- "float32": torch.float32,
17
- "bfloat16": torch.bfloat16,
18
-}
19
-
20
-
21
-def get_torch_dtype(dtype: Union[torch.dtype, str]) -> torch.dtype:
22
- if isinstance(dtype, str):
23
- torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype.lower()]
24
- else:
25
- torch_dtype = dtype
26
- return torch_dtype
27
-
28
-
29
-def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
30
- torch_dtype = get_torch_dtype(dtype)
31
- return torch.tensor([], dtype=torch_dtype).element_size()
32
-
33
-
11
def set_random_seed(seed: int) -> None:
12
random.seed(seed)
13
np.random.seed(seed)
@@ -40,15 +17,3 @@ def set_random_seed(seed: int) -> None:
17
18
if model_parallel_is_initialized():
19
model_parallel_cuda_manual_seed(seed)
43
-
44
-
45
-def get_cache_block_size(block_size: int,
46
- num_heads: int,
47
- head_size: int,
48
- num_layers: int,
49
- dtype: str) -> int:
50
- key_cache_block = block_size * num_heads * head_size
51
- value_cache_block = key_cache_block
52
- total = num_layers * (key_cache_block + value_cache_block)
53
- dtype_size = get_dtype_size(dtype)
54
- return dtype_size * total
cacheflow/outputs.py
new
+79
@@ -0,0 +1,79 @@
1
+from typing import Dict, List, Union
2
+
3
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
4
+
5
+from cacheflow.sequence import SequenceGroup
6
+
7
+
8
+class CompletionOutput:
9
+
10
+ def __init__(
11
+ self,
12
+ text: str,
13
+ token_ids: List[int],
14
+ cumulative_logprobs: float,
15
+ logprobs: List[Dict[int, float]],
16
+ ) -> None:
17
+ self.text = text
18
+ self.token_ids = token_ids
19
+ self.cumulative_logprobs = cumulative_logprobs
20
+ self.logprobs = logprobs
21
+
22
+ def __repr__(self) -> str:
23
+ return (f"CompletionOutput(output={self.text!r}, "
24
+ f"token_ids={self.token_ids}, "
25
+ f"cumulative_logprobs={self.cumulative_logprobs}, "
26
+ f"logprobs={self.logprobs})")
27
+
28
+
29
+class RequestOutput:
30
+
31
+ def __init__(
32
+ self,
33
+ request_id: int,
34
+ prompt: str,
35
+ prompt_token_ids: List[int],
36
+ outputs: List[CompletionOutput],
37
+ done: bool = False,
38
+ ) -> None:
39
+ self.request_id = request_id
40
+ self.prompt = prompt
41
+ self.prompt_token_ids = prompt_token_ids
42
+ self.outputs = outputs
43
+ self.done = done
44
+
45
+ @staticmethod
46
+ def from_seq_group(
47
+ seq_group: SequenceGroup,
48
+ tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
49
+ ) -> "RequestOutput":
50
+ outputs: List[CompletionOutput] = []
51
+ seqs = seq_group.get_seqs()
52
+ for seq in seqs:
53
+ output_token_ids = seq.data.output_token_ids
54
+ output_str = tokenizer.decode(output_token_ids,
55
+ skip_special_tokens=True)
56
+ seq_logprobs = seq.data.cumulative_logprobs
57
+
58
+ logprobs = seq.output_logprobs
59
+ if seq_group.sampling_params.logprobs == 0:
60
+ # NOTE: We need to take care of this case because the sequence
61
+ # always has the logprobs of the sampled tokens even if the
62
+ # logprobs are not requested.
63
+ logprobs = {}
64
+ output = CompletionOutput(output_str, output_token_ids,
65
+ seq_logprobs, logprobs)
66
+ outputs.append(output)
67
+
68
+ # Every sequence in the sequence group should have the same prompt.
69
+ prompt = seqs[0].prompt
70
+ prompt_token_ids = seqs[0].data.prompt_token_ids
71
+ return RequestOutput(seq_group.request_id, prompt, prompt_token_ids,
72
+ outputs, seq_group.is_finished())
73
+
74
+ def __repr__(self) -> str:
75
+ return (f"RequestOutput(request_id={self.request_id}, "
76
+ f"prompt={self.prompt!r}, "
77
+ f"prompt_token_ids={self.prompt_token_ids}, "
78
+ f"outputs={self.outputs}, "
79
+ f"done={self.done})")
cacheflow/sampling_params.py
+1
-1
@@ -116,4 +116,4 @@ class SamplingParams:
116
f"use_beam_search={self.use_beam_search}, "
117
f"stop_token_ids={self.stop_token_ids}, "
118
f"max_tokens={self.max_tokens}, "
119
- f"logprobs={self.logprobs}")
119
+ f"logprobs={self.logprobs})")
cacheflow/sequence.py
+9
-6
@@ -115,12 +115,14 @@ class SequenceGroup:
115
116
def __init__(
117
self,
118
- group_id: int,
118
+ request_id: str,
119
seqs: List[Sequence],
120
+ sampling_params: SamplingParams,
121
arrival_time: float,
122
) -> None:
122
- self.group_id = group_id
123
+ self.request_id = request_id
124
self.seqs = seqs
125
+ self.sampling_params = sampling_params
126
self.arrival_time = arrival_time
127
128
def get_seqs(
@@ -145,21 +147,22 @@ class SequenceGroup:
147
return all(seq.status == SequenceStatus.FINISHED for seq in self.seqs)
148
149
def __repr__(self) -> str:
148
- return (f'SequenceGroup(group_id={self.group_id}, '
149
- f'num_seqs={len(self.seqs)})')
150
+ return (f"SequenceGroup(request_id={self.request_id}, "
151
+ f"sampling_params={self.sampling_params}, "
152
+ f"num_seqs={len(self.seqs)})")
153
154
155
class SequenceGroupMetadata:
156
157
def __init__(
158
self,
156
- group_id: int,
159
+ request_id: str,
160
is_prompt: bool,
161
seq_data: Dict[int, SequenceData], # Seq id -> sequence data.
162
sampling_params: SamplingParams,
163
block_tables: Dict[int, List[int]], # Seq id -> list of physical block numbers.
164
) -> None:
162
- self.group_id = group_id
165
+ self.request_id = request_id
166
self.is_prompt = is_prompt
167
self.seq_data = seq_data
168
self.sampling_params = sampling_params
cacheflow/server/arg_utils.py
new
+74
@@ -0,0 +1,74 @@
1
+import argparse
2
+from typing import Tuple
3
+
4
+from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5
+ SchedulerConfig)
6
+from cacheflow.server.llm_server import LLMServer
7
+from cacheflow.server.ray_utils import initialize_cluster
8
+
9
+_GiB = 1 << 30
10
+
11
+
12
+def add_server_arguments(parser: argparse.ArgumentParser):
13
+ """Shared arguments for CacheFlow servers."""
14
+ # Model arguments
15
+ parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
16
+ parser.add_argument('--download-dir', type=str, default=None,
17
+ help='directory to download and load the weights, '
18
+ 'default to the default cache dir of huggingface')
19
+ parser.add_argument('--use-np-weights', action='store_true',
20
+ help='save a numpy copy of model weights for faster loading')
21
+ parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
22
+ # TODO(woosuk): Support FP32.
23
+ parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
24
+ help=('data type for model weights and activations. '
25
+ 'The "default" option will use FP16 precision '
26
+ 'for FP32 and FP16 models, and BF16 precision '
27
+ 'for BF16 models.'))
28
+ # Parallel arguments
29
+ parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
30
+ parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
31
+ parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
32
+ # KV cache arguments
33
+ parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
34
+ # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
35
+ parser.add_argument('--seed', type=int, default=0, help='random seed')
36
+ parser.add_argument('--swap-space', type=int, default=4, help='CPU swap space size (GiB) per GPU')
37
+ parser.add_argument('--gpu-memory-utilization', type=float, default=0.95, help='the percentage of GPU memory to be used for the model executor')
38
+ parser.add_argument('--max-num-batched-tokens', type=int, default=2560, help='maximum number of batched tokens per iteration')
39
+ parser.add_argument('--max-num-seqs', type=int, default=256, help='maximum number of sequences per iteration')
40
+ parser.add_argument('--disable-log-stats', action='store_true', help='disable logging statistics')
41
+ return parser
42
+
43
+
44
+def create_server_configs_from_args(
45
+ args: argparse.Namespace,
46
+) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig]:
47
+ # Post-process the parsed arguments.
48
+ args.swap_space = args.swap_space * _GiB
49
+ args.max_num_seqs = min(args.max_num_seqs, args.max_num_batched_tokens)
50
+
51
+ # Initialize the configs.
52
+ model_config = ModelConfig(
53
+ args.model, args.download_dir, args.use_np_weights,
54
+ args.use_dummy_weights, args.dtype, args.seed)
55
+ cache_config = CacheConfig(args.block_size, args.gpu_memory_utilization,
56
+ args.swap_space)
57
+ parallel_config = ParallelConfig(args.pipeline_parallel_size,
58
+ args.tensor_parallel_size, args.use_ray)
59
+ scheduler_config = SchedulerConfig(args.max_num_batched_tokens,
60
+ args.max_num_seqs)
61
+ return model_config, cache_config, parallel_config, scheduler_config
62
+
63
+
64
+def initialize_server_from_args(args: argparse.Namespace) -> LLMServer:
65
+ server_configs = create_server_configs_from_args(args)
66
+ parallel_config = server_configs[2]
67
+
68
+ # Initialize the cluster.
69
+ distributed_init_method, devices = initialize_cluster(parallel_config)
70
+
71
+ # Create the LLM server.
72
+ server = LLMServer(*server_configs, distributed_init_method, devices,
73
+ log_stats=not args.disable_log_stats)
74
+ return server
cacheflow/server/llm_server.py
new
+198
@@ -0,0 +1,198 @@
1
+import time
2
+from typing import Any, List, Optional
3
+
4
+try:
5
+ import ray
6
+except ImportError:
7
+ ray = None
8
+
9
+from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
10
+ SchedulerConfig)
11
+from cacheflow.core.scheduler import Scheduler
12
+from cacheflow.logger import init_logger
13
+from cacheflow.outputs import RequestOutput
14
+from cacheflow.sampling_params import SamplingParams
15
+from cacheflow.server.tokenizer_utils import get_tokenizer
16
+from cacheflow.sequence import Sequence, SequenceGroup
17
+from cacheflow.utils import Counter
18
+from cacheflow.worker.worker import Worker
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+class LLMServer:
24
+
25
+ def __init__(
26
+ self,
27
+ model_config: ModelConfig,
28
+ cache_config: CacheConfig,
29
+ parallel_config: ParallelConfig,
30
+ scheduler_config: SchedulerConfig,
31
+ distributed_init_method: str,
32
+ stage_devices: List[List[Any]],
33
+ log_stats: bool = True,
34
+ ) -> None:
35
+ logger.info(
36
+ "Initializing an LLM server with config: "
37
+ f"model={model_config.model!r}, "
38
+ f"dtype={model_config.dtype}, "
39
+ f"use_dummy_weights={model_config.use_dummy_weights}, "
40
+ f"download_dir={model_config.download_dir!r}, "
41
+ f"use_np_weights={model_config.use_np_weights}, "
42
+ f"tensor_parallel_size={parallel_config.tensor_parallel_size}, "
43
+ f"seed={model_config.seed})"
44
+ )
45
+ # TODO(woosuk): Print more configs in debug mode.
46
+
47
+ self.model_config = model_config
48
+ self.cache_config = cache_config
49
+ self.parallel_config = parallel_config
50
+ self.scheduler_config = scheduler_config
51
+ self.log_stats = log_stats
52
+
53
+ self._verify_args()
54
+
55
+ self.tokenizer = get_tokenizer(model_config.model)
56
+ self.seq_counter = Counter()
57
+
58
+ # Create the parallel GPU workers.
59
+ self.workers: List[Worker] = []
60
+ assert len(stage_devices) == 1, "Only support one stage for now."
61
+ for rank, node_resource, _ in stage_devices[0]:
62
+ worker_cls = Worker
63
+ if self.parallel_config.use_ray:
64
+ worker_cls = ray.remote(
65
+ num_cpus=0,
66
+ num_gpus=1,
67
+ resources={node_resource: 1e-5},
68
+ )(worker_cls).remote
69
+
70
+ worker = worker_cls(
71
+ model_config,
72
+ parallel_config,
73
+ scheduler_config,
74
+ rank,
75
+ distributed_init_method,
76
+ )
77
+ self.workers.append(worker)
78
+ # Profile the memory usage and initialize the cache.
79
+ self._init_cache()
80
+
81
+ # Create the scheduler.
82
+ self.scheduler = Scheduler(scheduler_config, cache_config, log_stats)
83
+
84
+ def _verify_args(self) -> None:
85
+ self.model_config.verify_with_parallel_config(self.parallel_config)
86
+
87
+ def _init_cache(self) -> None:
88
+ # Get the maximum number of blocks that can be allocated on GPU and CPU.
89
+ num_blocks = self._run_workers(
90
+ "profile_num_available_blocks",
91
+ get_all_outputs=True,
92
+ block_size=self.cache_config.block_size,
93
+ gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
94
+ cpu_swap_space=self.cache_config.swap_space,
95
+ )
96
+
97
+ # Since we use a shared centralized controller, we take the minimum
98
+ # number of blocks across all workers to make sure all the memory
99
+ # operators can be applied to all workers.
100
+ num_gpu_blocks = min(b[0] for b in num_blocks)
101
+ num_cpu_blocks = min(b[1] for b in num_blocks)
102
+ # FIXME(woosuk): Change to debug log.
103
+ logger.info(f'# GPU blocks: {num_gpu_blocks}, '
104
+ f'# CPU blocks: {num_cpu_blocks}')
105
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
106
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
107
+
108
+ # Initialize the cache.
109
+ self._run_workers("init_cache_engine", cache_config=self.cache_config)
110
+
111
+ def add_request(
112
+ self,
113
+ request_id: str,
114
+ prompt: str,
115
+ sampling_params: SamplingParams,
116
+ prompt_token_ids: Optional[List[int]] = None,
117
+ arrival_time: Optional[float] = None,
118
+ ) -> None:
119
+ if arrival_time is None:
120
+ arrival_time = time.time()
121
+ if prompt_token_ids is None:
122
+ prompt_token_ids = self.tokenizer.encode(prompt)
123
+
124
+ # Create the sequences.
125
+ block_size = self.cache_config.block_size
126
+ seqs: List[Sequence] = []
127
+ for _ in range(sampling_params.n):
128
+ seq_id = next(self.seq_counter)
129
+ seq = Sequence(seq_id, prompt, prompt_token_ids, block_size)
130
+ seqs.append(seq)
131
+
132
+ # FIXME(woosuk)
133
+ # Add the EOS token to the stop token list.
134
+ sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
135
+
136
+ # Create the sequence group.
137
+ seq_group = SequenceGroup(request_id, seqs, sampling_params,
138
+ arrival_time)
139
+
140
+ # Add the sequence group to the scheduler.
141
+ self.scheduler.add_seq_group(seq_group)
142
+
143
+ def has_unfinished_requests(self) -> bool:
144
+ return self.scheduler.has_unfinished_seqs()
145
+
146
+ def step(self) -> List[RequestOutput]:
147
+ seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
148
+ if (not seq_group_metadata_list) and scheduler_outputs.is_empty():
149
+ # Nothing to do.
150
+ return []
151
+
152
+ # Execute the model.
153
+ output = self._run_workers(
154
+ "execute_model",
155
+ seq_group_metadata_list=seq_group_metadata_list,
156
+ blocks_to_swap_in=scheduler_outputs.blocks_to_swap_in,
157
+ blocks_to_swap_out=scheduler_outputs.blocks_to_swap_out,
158
+ blocks_to_copy=scheduler_outputs.blocks_to_copy,
159
+ )
160
+ # Update the scheduler.
161
+ updated_seq_groups = self.scheduler.update(output)
162
+
163
+ # Create the outputs.
164
+ request_outputs: List[RequestOutput] = []
165
+ for seq_group in updated_seq_groups:
166
+ # TODO(woosuk): Batch-decode the outputs for speedup.
167
+ request_output = RequestOutput.from_seq_group(seq_group,
168
+ self.tokenizer)
169
+ request_outputs.append(request_output)
170
+ return request_outputs
171
+
172
+ def _run_workers(
173
+ self,
174
+ method: str,
175
+ get_all_outputs: bool = False,
176
+ *args,
177
+ **kwargs,
178
+ ) -> Any:
179
+ all_outputs = []
180
+ for worker in self.workers:
181
+ executor = getattr(worker, method)
182
+ if self.parallel_config.use_ray:
183
+ executor = executor.remote
184
+
185
+ output = executor(*args, **kwargs)
186
+ all_outputs.append(output)
187
+
188
+ if self.parallel_config.use_ray:
189
+ all_outputs = ray.get(all_outputs)
190
+
191
+ if get_all_outputs:
192
+ return all_outputs
193
+
194
+ # Make sure all workers have the same results.
195
+ output = all_outputs[0]
196
+ for other_output in all_outputs[1:]:
197
+ assert output == other_output
198
+ return output
cacheflow/server/ray_utils.py
new
+90
@@ -0,0 +1,90 @@
1
+import random
2
+from typing import List, Optional, Tuple
3
+
4
+try:
5
+ import ray
6
+except ImportError:
7
+ ray = None
8
+
9
+from cacheflow.config import ParallelConfig
10
+
11
+DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
12
+
13
+
14
+def initialize_cluster(
15
+ parallel_config: ParallelConfig,
16
+ address: Optional[str] = None,
17
+) -> Tuple[str, List[List[DeviceID]]]:
18
+ if not parallel_config.use_ray:
19
+ # Initialize cluster locally.
20
+ port = random.randint(10000, 20000)
21
+ # We need to setup the distributed init method to make sure
22
+ # the distributed megatron code (e.g., get world size) works correctly.
23
+ distributed_init_method = f"tcp://localhost:{port}"
24
+ all_stage_devices = [[(0, None, 0)]]
25
+ return distributed_init_method, all_stage_devices
26
+
27
+ if ray is None:
28
+ raise ImportError(
29
+ "Ray is not installed. Please install Ray to use distributed "
30
+ "serving.")
31
+ # Connect to a ray cluster.
32
+ ray.init(address=address)
33
+
34
+ # Assume we have a uniform cluster that each node has the same number of
35
+ # GPUs for now.
36
+ valid_node_resources = []
37
+ num_devices_per_node = None
38
+ for node in ray.nodes():
39
+ if (not node['Alive']) or node['Resources']['GPU'] <= 0:
40
+ continue
41
+ if num_devices_per_node is None:
42
+ num_devices_per_node = node['Resources']['GPU']
43
+ else:
44
+ assert num_devices_per_node == node['Resources']['GPU'], (
45
+ "The number of GPUs per node is not uniform.")
46
+ for key in node['Resources']:
47
+ if key.startswith('node:'):
48
+ valid_node_resources.append(key)
49
+
50
+ # Verify the parallel config.
51
+ num_nodes = len(valid_node_resources)
52
+ if parallel_config.world_size > num_nodes * num_devices_per_node:
53
+ raise ValueError(
54
+ "The number of required GPUs exceeds the total number of "
55
+ "available GPUs.")
56
+ if parallel_config.tensor_parallel_size >= num_devices_per_node:
57
+ if parallel_config.tensor_parallel_size % num_devices_per_node != 0:
58
+ raise ValueError(
59
+ "The number of tensor parallelism is not divisible by the "
60
+ "number of GPUs per node.")
61
+ else:
62
+ if num_devices_per_node % parallel_config.tensor_parallel_size != 0:
63
+ raise ValueError(
64
+ "The number of GPUs per node is not divisible by the number "
65
+ "of tensor parallelism.")
66
+
67
+ # Assign GPUs to pipeline stages.
68
+ rank = 0
69
+ current_node_id = 0
70
+ current_device_id = 0
71
+ distributed_init_method = None
72
+ all_stage_devices = []
73
+
74
+ for _ in range(parallel_config.pipeline_parallel_size):
75
+ stage_devices = []
76
+ for _ in range(parallel_config.tensor_parallel_size):
77
+ node_resource = valid_node_resources[current_node_id]
78
+ stage_devices.append((rank, node_resource, current_device_id))
79
+ if distributed_init_method is None:
80
+ ip = node_resource.split("node:")[-1]
81
+ port = random.randint(10000, 20000)
82
+ distributed_init_method = f"tcp://{ip}:{port}"
83
+ rank += 1
84
+ current_device_id += 1
85
+ if current_device_id >= num_devices_per_node:
86
+ current_node_id += 1
87
+ current_device_id = 0
88
+ all_stage_devices.append(stage_devices)
89
+
90
+ return distributed_init_method, all_stage_devices
cacheflow/server/tokenizer_utils.py
renamed
-1
@@ -3,7 +3,6 @@ from typing import Union
3
from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer,
4
PreTrainedTokenizerFast)
5
6
-
6
_MODEL_TYPES_WITH_SLOW_TOKENIZER = [
7
# LLaMA fast tokenizer has a bug related to protobuf.
8
# See https://github.com/WoosukKwon/cacheflow/issues/80#issue-1698550554
cacheflow/worker/cache_engine.py
+37
-21
@@ -4,6 +4,7 @@ from typing import Dict, List, Tuple
4
import torch
5
6
from cacheflow import cache_ops
7
+from cacheflow.config import CacheConfig, ModelConfig, ParallelConfig
8
9
KVCache = Tuple[torch.Tensor, torch.Tensor]
10
@@ -18,27 +19,22 @@ class CacheEngine:
19
20
def __init__(
21
self,
21
- worker_id: int,
22
- num_layers: int,
23
- num_heads: int,
24
- head_size: int,
25
- block_size: int,
26
- num_gpu_blocks: int,
27
- num_cpu_blocks: int,
28
- dtype: torch.dtype,
22
+ cache_config: CacheConfig,
23
+ model_config: ModelConfig,
24
+ parallel_config: ParallelConfig,
25
) -> None:
30
- if head_size % 16 != 0:
31
- raise ValueError(
32
- f'head_size ({head_size}) must be a multiple of 16.')
33
-
34
- self.worker_id = worker_id
35
- self.num_layers = num_layers
36
- self.num_heads = num_heads
37
- self.head_size = head_size
38
- self.block_size = block_size
39
- self.num_gpu_blocks = num_gpu_blocks
40
- self.num_cpu_blocks = num_cpu_blocks
41
- self.dtype = dtype
26
+ self.cache_config = cache_config
27
+ self.model_config = model_config
28
+ self.parallel_config = parallel_config
29
+
30
+ self.head_size = model_config.get_head_size()
31
+ self.num_layers = model_config.get_num_layers(parallel_config)
32
+ self.num_heads = model_config.get_num_heads(parallel_config)
33
+ self.dtype = model_config.dtype
34
+
35
+ self.block_size = cache_config.block_size
36
+ self.num_gpu_blocks = cache_config.num_gpu_blocks
37
+ self.num_cpu_blocks = cache_config.num_cpu_blocks
38
39
# Initialize the cache.
40
self.gpu_cache = self.allocate_gpu_cache()
@@ -48,7 +44,7 @@ class CacheEngine:
44
self.cache_stream = torch.cuda.Stream()
45
assert self.cache_stream != torch.cuda.current_stream()
46
# Initialize the events for stream synchronization.
51
- self.events = [torch.cuda.Event() for _ in range(num_layers)]
47
+ self.events = [torch.cuda.Event() for _ in range(self.num_layers)]
48
49
def get_key_block_shape(self) -> Tuple[int, int, int, int]:
50
element_size = torch.tensor([], dtype=self.dtype).element_size()
@@ -133,3 +129,23 @@ class CacheEngine:
129
value_caches = [value_cache for _, value_cache in self.gpu_cache]
130
# NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
131
cache_ops.copy_blocks(key_caches, value_caches, src_to_dsts)
132
+
133
+ @staticmethod
134
+ def get_cache_block_size(
135
+ block_size: int,
136
+ model_config: ModelConfig,
137
+ parallel_config: ParallelConfig,
138
+ ) -> int:
139
+ head_size = model_config.get_head_size()
140
+ num_heads = model_config.get_num_heads(parallel_config)
141
+ num_layers = model_config.get_num_layers(parallel_config)
142
+
143
+ key_cache_block = block_size * num_heads * head_size
144
+ value_cache_block = key_cache_block
145
+ total = num_layers * (key_cache_block + value_cache_block)
146
+ dtype_size = _get_dtype_size(model_config.dtype)
147
+ return dtype_size * total
148
+
149
+
150
+def _get_dtype_size(dtype: torch.dtype) -> int:
151
+ return torch.tensor([], dtype=dtype).element_size()
cacheflow/worker/controller.py
deleted
-130
@@ -1,130 +0,0 @@
1
-from typing import List, Optional, Tuple, Union
2
-
3
-try:
4
- import ray
5
-except ImportError:
6
- ray = None
7
-
8
-from cacheflow.core.scheduler import Scheduler
9
-from cacheflow.worker.worker import Worker
10
-
11
-
12
-DeviceID = Tuple[int, str, int] # rank, node resource (node IP), device id
13
-
14
-
15
-class Controller:
16
-
17
- def __init__(
18
- self,
19
- stage_id: int,
20
- stage_devices: List[DeviceID],
21
- world_size: int,
22
- tensor_parallel_size: int,
23
- pipeline_parallel_size: int,
24
- distributed_init_method: str,
25
- model_name: str,
26
- dtype: str,
27
- seed: int,
28
- cache_dir: Optional[str],
29
- use_dummy_weights: bool,
30
- use_np_cache: bool,
31
- max_num_batched_tokens: int,
32
- max_num_sequences: int,
33
- use_ray: bool,
34
- ) -> None:
35
- self.stage_id = stage_id
36
- self.stage_devices = stage_devices
37
- self.model_name = model_name
38
- self.use_ray = use_ray
39
-
40
- # Which pipeline stage is this node assigned to?
41
- self.is_first_stage = stage_id == 0
42
- self.is_last_stage = False
43
-
44
- self.workers: List[Worker] = []
45
- for rank, node_resource, device_id in stage_devices:
46
- if self.use_ray:
47
- worker_cls = ray.remote(num_cpus=0,
48
- num_gpus=1,
49
- resources={node_resource: 1e-5})(Worker).remote
50
- else:
51
- worker_cls = Worker
52
- worker = worker_cls(
53
- model_name=model_name,
54
- dtype=dtype,
55
- seed=seed,
56
- distributed_init_method=distributed_init_method,
57
- rank=rank,
58
- world_size=world_size,
59
- tensor_parallel_size=tensor_parallel_size,
60
- pipeline_parallel_size=pipeline_parallel_size,
61
- cache_dir=cache_dir,
62
- use_dummy_weights=use_dummy_weights,
63
- use_np_cache=use_np_cache,
64
- max_num_batched_tokens=max_num_batched_tokens,
65
- max_num_sequences=max_num_sequences,
66
- )
67
- self.workers.append(worker)
68
-
69
- def get_num_available_blocks(self, block_size: int, cpu_swap_space: int,
70
- gpu_memory_utilization: float) -> List[Tuple[int, int]]:
71
- all_worker_results = []
72
- for worker in self.workers:
73
- executor = worker.get_num_available_blocks
74
- if self.use_ray:
75
- executor = executor.remote
76
-
77
- result = executor(
78
- block_size,
79
- cpu_swap_space,
80
- gpu_memory_utilization,
81
- )
82
- all_worker_results.append(result)
83
- if self.use_ray:
84
- all_worker_results = ray.get(all_worker_results)
85
- return all_worker_results
86
-
87
- def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
88
- num_cpu_blocks: int):
89
- all_worker_futures = []
90
- for worker in self.workers:
91
- executor = worker.init_cache_engine
92
- if self.use_ray:
93
- executor = executor.remote
94
- future = executor(
95
- block_size,
96
- num_gpu_blocks,
97
- num_cpu_blocks,
98
- )
99
- all_worker_futures.append(future)
100
- if self.use_ray:
101
- ray.get(all_worker_futures)
102
-
103
- def set_next(
104
- self,
105
- next_node: Union['Controller', 'Scheduler'],
106
- ) -> None:
107
- self.next_node = next_node
108
- self.is_last_stage = isinstance(next_node, Scheduler)
109
-
110
- def execute_stage(self, *args, **kwargs) -> None:
111
- all_outputs = []
112
- for worker in self.workers:
113
- executor = (worker.execute_stage.remote
114
- if self.use_ray else worker.execute_stage)
115
- output = executor(*args, **kwargs)
116
- all_outputs.append(output)
117
-
118
- if self.use_ray:
119
- all_outputs = ray.get(all_outputs)
120
-
121
- # Make sure all workers have the same results.
122
- output = all_outputs[0]
123
- for other_output in all_outputs[1:]:
124
- assert output == other_output
125
-
126
- if self.is_last_stage:
127
- self.next_node.post_step(output)
128
- else:
129
- # TODO: Support pipeline parallelism.
130
- assert False
cacheflow/worker/worker.py
+73
-93
@@ -1,14 +1,13 @@
1
"""A GPU worker class."""
2
-from typing import Dict, List, Optional, Tuple
2
+from typing import Dict, List, Tuple
3
4
import torch
5
6
-from cacheflow.model_executor import (get_model, get_cache_block_size,
7
- InputMetadata, set_random_seed)
6
+from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
7
+ SchedulerConfig)
8
+from cacheflow.model_executor import get_model, InputMetadata, set_random_seed
9
from cacheflow.model_executor.parallel_utils.parallel_state import (
9
- initialize_model_parallel,
10
- initialize_all_reduce_launcher,
11
- get_tensor_model_parallel_world_size)
10
+ initialize_model_parallel, initialize_all_reduce_launcher)
11
from cacheflow.sampling_params import SamplingParams
12
from cacheflow.sequence import (SequenceData, SequenceGroupMetadata,
13
SequenceOutputs)
@@ -26,59 +25,46 @@ class Worker:
25
26
def __init__(
27
self,
29
- model_name: str,
30
- dtype: str,
31
- seed: int,
32
- distributed_init_method: str,
28
+ model_config: ModelConfig,
29
+ parallel_config: ParallelConfig,
30
+ scheduler_config: SchedulerConfig,
31
rank: int,
34
- world_size: int,
35
- cache_dir: Optional[str],
36
- use_dummy_weights: bool,
37
- use_np_cache: bool,
38
- max_num_batched_tokens: int,
39
- max_num_sequences: int,
40
- tensor_parallel_size: int = 1,
41
- pipeline_parallel_size: int = 1,
32
+ distributed_init_method: str,
33
) -> None:
43
- self.init_distributed_environment(distributed_init_method,
44
- rank,
45
- world_size,
46
- tensor_parallel_size,
47
- pipeline_parallel_size)
48
- self.worker_id = rank
49
- self.seed = seed
50
- set_random_seed(self.seed)
34
+ self.model_config = model_config
35
+ self.parallel_config = parallel_config
36
+ self.scheduler_config = scheduler_config
37
+ self.rank = rank
38
+ self.distributed_init_method = distributed_init_method
39
+
40
+ # Initialize the distributed environment.
41
+ _init_distributed_environment(parallel_config, rank,
42
+ distributed_init_method)
43
44
# Initialize the model.
53
- self.model, self.dtype = get_model(
54
- model_name, dtype=dtype, cache_dir=cache_dir,
55
- use_dummy_weights=use_dummy_weights, use_np_cache=use_np_cache)
56
- tensor_model_parallel_world_size = (
57
- get_tensor_model_parallel_world_size())
58
- self.max_num_batched_tokens = max_num_batched_tokens
45
+ set_random_seed(self.model_config.seed)
46
+ self.model = get_model(model_config)
47
initialize_all_reduce_launcher(
60
- self.max_num_batched_tokens, self.model.config.hidden_size, self.dtype)
61
- self.max_num_sequences = max_num_sequences
62
- self.num_layers = self.model.config.num_hidden_layers
63
- assert self.model.config.num_attention_heads % tensor_model_parallel_world_size == 0
64
- self.num_heads = self.model.config.num_attention_heads // tensor_model_parallel_world_size
65
- self.head_size = self.model.config.hidden_size // (self.num_heads * tensor_model_parallel_world_size)
66
-
67
- # We reset the seed after initializing the model to ensure that
68
- # the random state is not affected by the model initialization.
69
- set_random_seed(seed)
70
-
71
- # Uninitialized cache engine. Will be initialized with
48
+ self.scheduler_config.max_num_batched_tokens,
49
+ self.model_config.get_hidden_size(),
50
+ self.model_config.dtype,
51
+ )
52
+
53
+ # Uninitialized cache engine. Will be initialized by
54
# self.init_cache_engine().
55
+ self.cache_config = None
56
self.block_size = None
57
self.cache_engine = None
58
self.cache_events = None
59
self.gpu_cache = None
60
61
@torch.inference_mode()
79
- def get_num_available_blocks(
80
- self, block_size: int, cpu_swap_space: int,
81
- gpu_memory_utilization: float) -> Tuple[int, int]:
62
+ def profile_num_available_blocks(
63
+ self,
64
+ block_size: int,
65
+ gpu_memory_utilization: float,
66
+ cpu_swap_space: int,
67
+ ) -> Tuple[int, int]:
68
# Profile the memory usage of the model and get the maximum number of
69
# cache blocks that can be allocated with the remaining free memory.
70
torch.cuda.empty_cache()
@@ -90,14 +76,15 @@ class Worker:
76
# Enable top-k sampling to reflect the accurate memory usage.
77
sampling_params = SamplingParams(top_p=0.99,
78
top_k=self.model.config.vocab_size - 1)
79
+ max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
80
+ max_num_seqs = self.scheduler_config.max_num_seqs
81
seqs = []
94
- for group_id in range(self.max_num_sequences):
95
- seq_len = (self.max_num_batched_tokens // self.max_num_sequences +
96
- (group_id < self.max_num_batched_tokens %
97
- self.max_num_sequences))
82
+ for group_id in range(max_num_seqs):
83
+ seq_len = (max_num_batched_tokens // max_num_seqs +
84
+ (group_id < max_num_batched_tokens % max_num_seqs))
85
seq_data = SequenceData([0] * seq_len)
86
seq = SequenceGroupMetadata(
100
- group_id=group_id,
87
+ request_id=str(group_id),
88
is_prompt=True,
89
seq_data={group_id: seq_data},
90
sampling_params=sampling_params,
@@ -105,13 +92,14 @@ class Worker:
92
)
93
seqs.append(seq)
94
108
- input_tokens, input_positions, input_metadata = self.prepare_inputs(seqs)
95
+ input_tokens, input_positions, input_metadata = self._prepare_inputs(seqs)
96
97
# Execute the model.
98
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
99
self.model(
100
input_ids=input_tokens,
101
positions=input_positions,
114
- kv_caches=[(None, None)] * self.num_layers,
102
+ kv_caches=[(None, None)] * num_layers,
103
input_metadata=input_metadata,
104
cache_events=None,
105
)
@@ -121,53 +109,27 @@ class Worker:
109
torch.cuda.synchronize()
110
peak_memory = torch.cuda.max_memory_allocated()
111
total_gpu_memory = get_gpu_memory()
124
- cache_block_size = get_cache_block_size(block_size, self.num_heads,
125
- self.head_size, self.num_layers,
126
- self.dtype)
112
+ cache_block_size = CacheEngine.get_cache_block_size(
113
+ block_size, self.model_config, self.parallel_config)
114
num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
115
- peak_memory) // cache_block_size)
116
num_cpu_blocks = int(cpu_swap_space // cache_block_size)
117
torch.cuda.empty_cache()
131
- # Reset the seed to ensure that the model output is not affected by
132
- # the profiling.
133
- set_random_seed(self.seed)
118
+
119
+ # Reset the seed to ensure that the random state is not affected by
120
+ # the model initialization and profiling.
121
+ set_random_seed(self.model_config.seed)
122
return num_gpu_blocks, num_cpu_blocks
123
136
- def init_cache_engine(self, block_size: int, num_gpu_blocks: int,
137
- num_cpu_blocks: int):
138
- self.block_size = block_size
124
+ def init_cache_engine(self, cache_config: CacheConfig) -> None:
125
+ self.cache_config = cache_config
126
+ self.block_size = cache_config.block_size
127
self.cache_engine = CacheEngine(
140
- worker_id=self.worker_id,
141
- num_layers=self.num_layers,
142
- num_heads=self.num_heads,
143
- head_size=self.head_size,
144
- block_size=self.block_size,
145
- num_gpu_blocks=num_gpu_blocks,
146
- num_cpu_blocks=num_cpu_blocks,
147
- dtype=self.dtype,
148
- )
128
+ self.cache_config, self.model_config, self.parallel_config)
129
self.cache_events = self.cache_engine.events
130
self.gpu_cache = self.cache_engine.gpu_cache
131
152
- def init_distributed_environment(self,
153
- distributed_init_method: str,
154
- rank: int,
155
- world_size: int,
156
- tensor_parallel_size: int = 1,
157
- pipeline_parallel_size: int = 1) -> None:
158
- """Initialize the distributed environment."""
159
- torch.distributed.init_process_group(
160
- backend='nccl',
161
- init_method=distributed_init_method,
162
- world_size=world_size,
163
- rank=rank,
164
- )
165
- # A small all_reduce for warmup.
166
- torch.distributed.all_reduce(torch.zeros(1).cuda())
167
- initialize_model_parallel(tensor_parallel_size,
168
- pipeline_parallel_size)
169
-
170
- def prepare_inputs(
132
+ def _prepare_inputs(
133
self,
134
seq_group_metadata_list: List[SequenceGroupMetadata],
135
) -> Tuple[torch.LongTensor, torch.LongTensor, InputMetadata]:
@@ -284,7 +246,7 @@ class Worker:
246
return tokens_tensor, positions_tensor, input_metadata
247
248
@torch.inference_mode()
287
- def execute_stage(
249
+ def execute_model(
250
self,
251
seq_group_metadata_list: List[SequenceGroupMetadata],
252
blocks_to_swap_in: Dict[int, int],
@@ -316,7 +278,7 @@ class Worker:
278
return {}
279
280
# Prepare input tensors.
319
- input_tokens, input_positions, input_metadata = self.prepare_inputs(
281
+ input_tokens, input_positions, input_metadata = self._prepare_inputs(
282
seq_group_metadata_list)
283
284
# Execute the model.
@@ -330,6 +292,24 @@ class Worker:
292
return output
293
294
295
+def _init_distributed_environment(
296
+ parallel_config: ParallelConfig,
297
+ rank: int,
298
+ distributed_init_method: str,
299
+) -> None:
300
+ """Initialize the distributed environment."""
301
+ torch.distributed.init_process_group(
302
+ backend="nccl",
303
+ world_size=parallel_config.world_size,
304
+ rank=rank,
305
+ init_method=distributed_init_method,
306
+ )
307
+ # A small all_reduce for warmup.
308
+ torch.distributed.all_reduce(torch.zeros(1).cuda())
309
+ initialize_model_parallel(parallel_config.tensor_parallel_size,
310
+ parallel_config.pipeline_parallel_size)
311
+
312
+
313
def _pad_to_alignment(x: List[int], multiple_of: int) -> List[int]:
314
return x + [0] * ((-len(x)) % multiple_of)
315
examples/simple_server.py
new
+44
@@ -0,0 +1,44 @@
1
+import argparse
2
+import uuid
3
+
4
+from cacheflow import (add_server_arguments, initialize_server_from_args,
5
+ SamplingParams)
6
+
7
+
8
+def main(args: argparse.Namespace):
9
+ # Initialize the server.
10
+ server = initialize_server_from_args(args)
11
+
12
+ # Test the following prompts.
13
+ test_prompts = [
14
+ ("A robot may not injure a human being", SamplingParams()),
15
+ ("To be or not to be,",
16
+ SamplingParams(temperature=0.8, top_k=5, presence_penalty=0.2)),
17
+ ("What is the meaning of life?",
18
+ SamplingParams(n=2, temperature=0.8, top_p=0.95, frequency_penalty=0.1)),
19
+ ("It is only with the heart that one can see rightly",
20
+ SamplingParams(n=3, use_beam_search=True, temperature=0.0)),
21
+ ]
22
+
23
+ # Run the server.
24
+ while True:
25
+ # To test iteration-level scheduling, we add one request at each step.
26
+ if test_prompts:
27
+ prompt, sampling_params = test_prompts.pop(0)
28
+ request_id = str(uuid.uuid4().hex[:8])
29
+ server.add_request(request_id, prompt, sampling_params)
30
+
31
+ request_outputs = server.step()
32
+ for request_output in request_outputs:
33
+ if request_output.done:
34
+ print(request_output)
35
+
36
+ if not (server.has_unfinished_requests() or test_prompts):
37
+ break
38
+
39
+
40
+if __name__ == '__main__':
41
+ parser = argparse.ArgumentParser(description='Simple CacheFlow server.')
42
+ parser = add_server_arguments(parser)
43
+ args = parser.parse_args()
44
+ main(args)
simple_server.py
deleted
-38
@@ -1,38 +0,0 @@
1
-import argparse
2
-
3
-from cacheflow.core.server import (
4
- add_server_arguments, process_server_arguments,
5
- init_local_server_and_frontend_with_arguments)
6
-from cacheflow.sampling_params import SamplingParams
7
-
8
-
9
-def main(args: argparse.Namespace):
10
- server, frontend = init_local_server_and_frontend_with_arguments(args)
11
- # Test the following inputs.
12
- test_inputs = [
13
- ("A robot may not injure a human being", {}), # Use default parameters.
14
- ("To be or not to be,", {"temperature": 0.8, "top_k": 5, "presence_penalty": 0.2}),
15
- ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95, "frequency_penalty": 0.1}),
16
- ("It is only with the heart that one can see rightly", {"n": 3, "use_beam_search": True, "temperature": 0.0}),
17
- ]
18
- while True:
19
- if test_inputs:
20
- text, sampling_params_dict = test_inputs.pop(0)
21
- sampling_params = SamplingParams(**sampling_params_dict)
22
- sampling_params = frontend.add_eos_token(sampling_params)
23
- frontend.query(text, sampling_params)
24
- server.add_sequence_groups(frontend.get_inputs())
25
- updated_seq_groups = server.step()
26
- for seq_group in updated_seq_groups:
27
- if seq_group.is_finished():
28
- frontend.print_response(seq_group)
29
- if not (server.has_unfinished_requests() or test_inputs):
30
- break
31
-
32
-
33
-if __name__ == '__main__':
34
- parser = argparse.ArgumentParser(description='CacheFlow simple server.')
35
- parser = add_server_arguments(parser)
36
- args = parser.parse_args()
37
- args = process_server_arguments(args)
38
- main(args)