A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
FastAPI-based working frontend (#10)
Zhuohan Li committed
Mar 29, 2023 at 14:48 UTC
721fa3df155e5649bbe2188517594f24f4e63523
15 files changed
+536
-146
README.md
+39
-2
@@ -8,9 +8,46 @@ pip install flash-attn # This may take up to 10 mins.
8
pip install -e .
9
```
10
11
-## Run
11
+## Test simple server
12
13
```bash
14
ray start --head
15
-python server.py [--tensor-parallel-size <N>]
15
+python simple_server.py
16
+```
17
+
18
+The detailed arguments for `simple_server.py` can be found by:
19
+```bash
20
+python simple_server.py --help
21
+```
22
+
23
+## FastAPI server
24
+
25
+Install the following additional dependencies:
26
+```bash
27
+pip install fastapi uvicorn
28
+```
29
+
30
+To start the server:
31
+```bash
32
+ray start --head
33
+python -m cacheflow.http_frontend.fastapi_frontend
34
+```
35
+
36
+To test the server:
37
+```bash
38
+python -m cacheflow.http_frontend.test_cli_client
39
+```
40
+
41
+## Gradio web server
42
+
43
+Install the following additional dependencies:
44
+```bash
45
+pip install gradio
46
+```
47
+
48
+Start the server:
49
+```bash
50
+python -m cacheflow.http_frontend.fastapi_frontend
51
+# At another terminal
52
+python -m cacheflow.http_frontend.gradio_webserver
53
```
cacheflow/http_frontend/fastapi_frontend.py
new
+152
@@ -0,0 +1,152 @@
1
+import argparse
2
+import asyncio
3
+import time
4
+from typing import List, Dict
5
+import json
6
+
7
+import ray
8
+from transformers import AutoTokenizer
9
+from fastapi import FastAPI, Request
10
+from fastapi.responses import StreamingResponse
11
+import uvicorn
12
+
13
+from cacheflow.sampling_params import SamplingParams
14
+from cacheflow.sequence import Sequence, SequenceGroup
15
+from cacheflow.master.server import (Server, add_server_arguments,
16
+ initialize_ray_cluster)
17
+from cacheflow.worker.controller import DeviceID
18
+from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
19
+
20
+app = FastAPI()
21
+
22
+class FastAPIFrontend:
23
+ def __init__(
24
+ self,
25
+ model: str,
26
+ model_path: str,
27
+ pipeline_parallel_size: int,
28
+ tensor_parallel_size: int,
29
+ block_size: int,
30
+ dtype: str,
31
+ seed: int,
32
+ swap_space: int,
33
+ max_batch_size: int,
34
+ num_nodes: int,
35
+ num_devices_per_node: int,
36
+ distributed_init_method: str,
37
+ all_stage_devices: List[List[DeviceID]],
38
+ ):
39
+ self.block_size = block_size
40
+
41
+ self.tokenizer = AutoTokenizer.from_pretrained(model)
42
+ self.seq_group_counter = Counter()
43
+ self.seq_counter = Counter()
44
+ remote_server_class = ray.remote(num_cpus=0)(Server)
45
+ self.server = remote_server_class.remote(
46
+ model=model,
47
+ model_path=model_path,
48
+ pipeline_parallel_size=pipeline_parallel_size,
49
+ tensor_parallel_size=tensor_parallel_size,
50
+ block_size=block_size,
51
+ dtype=dtype,
52
+ seed=seed,
53
+ swap_space=swap_space,
54
+ max_batch_size=max_batch_size,
55
+ num_nodes=num_nodes,
56
+ num_devices_per_node=num_devices_per_node,
57
+ distributed_init_method=distributed_init_method,
58
+ all_stage_devices=all_stage_devices,
59
+ gpu_memory=get_gpu_memory(),
60
+ cpu_memory=get_cpu_memory(),
61
+ )
62
+
63
+ self.running_seq_groups: Dict[int, SequenceGroup] = {}
64
+ self.sequence_group_events: Dict[int, asyncio.Event] = {}
65
+ self.is_server_running = False
66
+
67
+ async def server_step(self):
68
+ self.is_server_running = True
69
+ updated_seq_groups = await self.server.step.remote()
70
+ self.is_server_running = False
71
+ for seq_group in updated_seq_groups:
72
+ group_id = seq_group.group_id
73
+ self.running_seq_groups[group_id] = seq_group
74
+ self.sequence_group_events[group_id].set()
75
+
76
+ async def generate(self, request_dict: Dict):
77
+ prompt = request_dict["prompt"]
78
+ sampling_params = SamplingParams.from_dict(request_dict)
79
+ sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
80
+ token_ids = self.tokenizer.encode(prompt)
81
+ seqs: List[Sequence] = []
82
+ for _ in range(sampling_params.n):
83
+ seq_id = next(self.seq_counter)
84
+ seq = Sequence(seq_id, token_ids, block_size=self.block_size)
85
+ seqs.append(seq)
86
+
87
+ group_id = next(self.seq_group_counter)
88
+ seq_group = SequenceGroup(group_id, seqs)
89
+ group_event = asyncio.Event()
90
+ self.sequence_group_events[group_id] = group_event
91
+ await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
92
+ while True:
93
+ if not self.is_server_running:
94
+ await self.server_step()
95
+ # Wait for new output. Add a 1s timeout to prevent dead lock.
96
+ await asyncio.wait_for(group_event.wait(), timeout=1)
97
+ group_event.clear()
98
+ seq_group = self.running_seq_groups[group_id]
99
+ all_outputs = []
100
+ for seq in seq_group.seqs:
101
+ token_ids = seq.get_token_ids()
102
+ output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
103
+ all_outputs.append(output)
104
+ ret = {
105
+ "text": all_outputs,
106
+ "error": 0,
107
+ }
108
+ yield (json.dumps(ret) + "\0").encode("utf-8")
109
+ if seq_group.is_finished():
110
+ break
111
+
112
+
113
+@app.post("/generate")
114
+async def generate_stream(request: Request):
115
+ request_dict = await request.json()
116
+ return StreamingResponse(frontend.generate(request_dict))
117
+
118
+
119
+if __name__ == "__main__":
120
+ parser = argparse.ArgumentParser()
121
+ parser.add_argument("--host", type=str, default="localhost")
122
+ parser.add_argument("--port", type=int, default=10002)
123
+ parser = add_server_arguments(parser)
124
+ args = parser.parse_args()
125
+
126
+ # TODO(zhuohan): Support pipeline parallelism.
127
+ assert args.pipeline_parallel_size == 1, (
128
+ 'Pipeline parallelism is not supported yet.')
129
+
130
+ (num_nodes, num_devices_per_node, distributed_init_method,
131
+ all_stage_devices) = (
132
+ initialize_ray_cluster(
133
+ pipeline_parallel_size=args.pipeline_parallel_size,
134
+ tensor_parallel_size=args.tensor_parallel_size))
135
+
136
+ frontend = FastAPIFrontend(
137
+ model=args.model,
138
+ model_path=args.model_path,
139
+ pipeline_parallel_size=args.pipeline_parallel_size,
140
+ tensor_parallel_size=args.tensor_parallel_size,
141
+ block_size=args.block_size,
142
+ dtype=args.dtype,
143
+ seed=args.seed,
144
+ swap_space=args.swap_space,
145
+ max_batch_size=args.max_batch_size,
146
+ num_nodes=num_nodes,
147
+ num_devices_per_node=num_devices_per_node,
148
+ distributed_init_method=distributed_init_method,
149
+ all_stage_devices=all_stage_devices,
150
+ )
151
+
152
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/http_frontend/gradio_webserver.py
new
+43
@@ -0,0 +1,43 @@
1
+import argparse
2
+import json
3
+import time
4
+
5
+import gradio as gr
6
+import requests
7
+
8
+
9
+def http_bot(prompt):
10
+ headers = {"User-Agent": "Cacheflow Client"}
11
+ pload = {
12
+ "prompt": prompt,
13
+ "max_num_steps": 128,
14
+ }
15
+ response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
16
+
17
+ for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
18
+ if chunk:
19
+ data = json.loads(chunk.decode("utf-8"))
20
+ output = data["text"][0]
21
+ yield output
22
+
23
+
24
+def build_demo():
25
+ with gr.Blocks() as demo:
26
+ gr.Markdown(
27
+ "# Cacheflow demo\n"
28
+ )
29
+ inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")# .style(container=False)
30
+ outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
31
+ inputbox.submit(http_bot, [inputbox], [outputbox])
32
+ return demo
33
+
34
+
35
+if __name__ == "__main__":
36
+ parser = argparse.ArgumentParser()
37
+ parser.add_argument("--host", type=str, default="localhost")
38
+ parser.add_argument("--port", type=int, default=10003)
39
+ parser.add_argument("--model-url", type=str, default="http://localhost:10002/generate")
40
+ args = parser.parse_args()
41
+
42
+ demo = build_demo()
43
+ demo.queue(concurrency_count=100).launch(server_name=args.host, server_port=args.port)
\ No newline at end of file
cacheflow/http_frontend/test_cli_client.py
new
+23
@@ -0,0 +1,23 @@
1
+import requests
2
+import json
3
+
4
+def http_request():
5
+ prompt = "Ion Stoica is a"
6
+
7
+ headers = {"User-Agent": "Test Client"}
8
+ pload = {
9
+ "prompt": prompt,
10
+ "n": 4,
11
+ "use_beam_search": True,
12
+ "temperature": 0.0,
13
+ }
14
+ response = requests.post("http://localhost:10002/generate", headers=headers, json=pload, stream=True)
15
+
16
+ for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
17
+ if chunk:
18
+ data = json.loads(chunk.decode("utf-8"))
19
+ output = data["text"]
20
+ yield output
21
+
22
+for h in http_request():
23
+ print(h, flush=True)
cacheflow/master/scheduler.py
+14
-12
@@ -1,7 +1,6 @@
1
-from typing import Dict, List
1
+from typing import Dict, List, Tuple
2
3
from cacheflow.master.block_manager import BlockSpaceManager
4
-from cacheflow.master.frontend import Frontend
4
from cacheflow.sampling_params import SamplingParams
5
from cacheflow.sequence import Sequence
6
from cacheflow.sequence import SequenceGroup
@@ -14,14 +13,12 @@ class Scheduler:
13
14
def __init__(
15
self,
17
- frontend: Frontend,
16
controllers: List,
17
block_size: int,
18
num_gpu_blocks: int,
19
num_cpu_blocks: int,
20
max_num_batched_tokens: int,
21
) -> None:
24
- self.frontend = frontend
22
self.controllers = controllers
23
self.block_size = block_size
24
self.num_gpu_blocks = num_gpu_blocks
@@ -47,9 +44,12 @@ class Scheduler:
44
# Pending sequence groups (FIFO).
45
self.pending: List[SequenceGroup] = []
46
50
- def _fetch_inputs(self) -> None:
51
- inputs = self.frontend.get_inputs()
52
- for seq_group, sampling_params in inputs:
47
+ def add_sequence_groups(
48
+ self,
49
+ sequence_groups: List[Tuple[SequenceGroup, SamplingParams]],
50
+ ) -> None:
51
+ # Add sequence groups to the pending queue.
52
+ for seq_group, sampling_params in sequence_groups:
53
self.pending.append(seq_group)
54
self.sampling_params[seq_group.group_id] = sampling_params
55
@@ -104,7 +104,7 @@ class Scheduler:
104
seq.status = SequenceStatus.SWAPPED
105
self.swapped.append(seq_group)
106
107
- def step(self) -> None:
107
+ def step(self) -> List[SequenceGroup]:
108
# Blocks that need to be swaped or copied before model execution.
109
blocks_to_swap_in: Dict[int, int] = {}
110
blocks_to_swap_out: Dict[int, int] = {}
@@ -158,7 +158,6 @@ class Scheduler:
158
# 3. Join new sequences if possible.
159
# NOTE: Here we implicitly assume FCFS scheduling.
160
# TODO(woosuk): Add a batching policy to control the batch size.
161
- self._fetch_inputs()
161
if not self.swapped:
162
for i, seq_group in enumerate(self.pending):
163
num_prompt_tokens = seq_group.seqs[0].get_len()
@@ -176,6 +175,8 @@ class Scheduler:
175
176
# 4. Create input data structures.
177
input_seq_groups: List[SequenceGroupInputs] = []
178
+ updated_seq_groups: List[SequenceGroup] = self.running.copy()
179
+
180
for seq_group in self.running:
181
group_id = seq_group.group_id
182
num_steps = self.num_steps[group_id]
@@ -219,6 +220,8 @@ class Scheduler:
220
blocks_to_copy,
221
)
222
223
+ return updated_seq_groups
224
+
225
def post_step(
226
self,
227
seq_outputs: Dict[int, SequenceOutputs],
@@ -268,13 +271,12 @@ class Scheduler:
271
running: List[SequenceGroup] = []
272
for seq_group in self.running:
273
if seq_group.is_finished():
271
- self._return(seq_group)
274
+ self._free_seq_group(seq_group)
275
else:
276
running.append(seq_group)
277
self.running = running
278
276
- def _return(self, seq_group: SequenceGroup) -> None:
279
+ def _free_seq_group(self, seq_group: SequenceGroup) -> None:
280
group_id = seq_group.group_id
281
del self.num_steps[group_id]
282
del self.sampling_params[group_id]
280
- self.frontend.print_response(seq_group)
cacheflow/master/server.py
renamed
+89
-87
@@ -1,13 +1,98 @@
1
import argparse
2
+from typing import List, Tuple
3
import random
3
-from typing import List, Tuple, Dict
4
5
import ray
6
7
-from cacheflow.master.frontend import Frontend
7
from cacheflow.master.scheduler import Scheduler
8
from cacheflow.models import get_memory_analyzer
9
from cacheflow.worker.controller import Controller, DeviceID
10
+from cacheflow.sequence import SequenceGroup
11
+from cacheflow.sampling_params import SamplingParams
12
+
13
+class Server:
14
+ def __init__(
15
+ self,
16
+ model: str,
17
+ model_path: str,
18
+ pipeline_parallel_size: int,
19
+ tensor_parallel_size: int,
20
+ block_size: int,
21
+ dtype: str,
22
+ seed: int,
23
+ swap_space: int,
24
+ max_batch_size: int,
25
+ num_nodes: int,
26
+ num_devices_per_node: int,
27
+ distributed_init_method: str,
28
+ all_stage_devices: List[List[DeviceID]],
29
+ gpu_memory: int,
30
+ cpu_memory: int,
31
+ ):
32
+ self.num_nodes = num_nodes
33
+ self.num_devices_per_node = num_devices_per_node
34
+ self.world_size = pipeline_parallel_size * tensor_parallel_size
35
+
36
+ self.memory_analyzer = get_memory_analyzer(
37
+ model_name=model,
38
+ block_size=block_size,
39
+ dtype=dtype,
40
+ gpu_memory=gpu_memory,
41
+ cpu_memory=cpu_memory,
42
+ tensor_parallel_size=tensor_parallel_size,
43
+ )
44
+ self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
45
+ max_num_batched_tokens=max_batch_size)
46
+ self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
47
+ swap_space=swap_space)
48
+ print(f'# GPU blocks: {self.num_gpu_blocks}, '
49
+ f'# CPU blocks: {self.num_cpu_blocks}')
50
+
51
+ # Create a controller for each pipeline stage.
52
+ self.controllers: List[Controller] = []
53
+ for i in range(pipeline_parallel_size):
54
+ controller = Controller(
55
+ stage_id=i,
56
+ stage_devices=all_stage_devices[i],
57
+ world_size=self.world_size,
58
+ pipeline_parallel_size=pipeline_parallel_size,
59
+ tensor_parallel_size=tensor_parallel_size,
60
+ distributed_init_method=distributed_init_method,
61
+ model_name=model,
62
+ block_size=block_size,
63
+ num_gpu_blocks=self.num_gpu_blocks,
64
+ num_cpu_blocks=self.num_cpu_blocks,
65
+ dtype=dtype,
66
+ seed=seed,
67
+ model_path=model_path,
68
+ )
69
+ self.controllers.append(controller)
70
+
71
+ # Create a scheduler.
72
+ self.scheduler = Scheduler(
73
+ controllers=self.controllers,
74
+ block_size=block_size,
75
+ num_gpu_blocks=self.num_gpu_blocks,
76
+ num_cpu_blocks=self.num_cpu_blocks,
77
+ max_num_batched_tokens=max_batch_size,
78
+ )
79
+ # Connect the controllers.
80
+ for i in range(len(self.controllers) - 1):
81
+ self.controllers[i].set_next(self.controllers[i + 1])
82
+ self.controllers[-1].set_next(self.scheduler)
83
+
84
+ def add_sequence_groups(
85
+ self,
86
+ sequence_groups: List[Tuple[SequenceGroup, SamplingParams]]
87
+ ):
88
+ self.scheduler.add_sequence_groups(sequence_groups)
89
+
90
+ def step(self):
91
+ return self.scheduler.step()
92
+
93
+ def has_unfinished_requests(self):
94
+ return (self.scheduler.pending or self.scheduler.running or
95
+ self.scheduler.swapped)
96
97
98
def initialize_ray_cluster(
@@ -76,88 +161,7 @@ def initialize_ray_cluster(
161
all_stage_devices)
162
163
79
-def main(args: argparse.Namespace):
80
- # TODO(zhuohan): Support pipeline parallelism.
81
- assert args.pipeline_parallel_size == 1, (
82
- 'Pipeline parallelism is not supported yet.')
83
-
84
- (num_nodes, num_devices_per_node, distributed_init_method,
85
- all_stage_devices) = (
86
- initialize_ray_cluster(
87
- pipeline_parallel_size=args.pipeline_parallel_size,
88
- tensor_parallel_size=args.tensor_parallel_size))
89
-
90
- world_size = args.pipeline_parallel_size * args.tensor_parallel_size
91
-
92
- memory_analyzer = get_memory_analyzer(
93
- model_name=args.model,
94
- block_size=args.block_size,
95
- dtype=args.dtype,
96
- tensor_parallel_size=args.tensor_parallel_size,
97
- )
98
- num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
99
- max_num_batched_tokens=args.max_batch_size)
100
- num_cpu_blocks = memory_analyzer.get_max_num_cpu_blocks(
101
- swap_space=args.swap_space)
102
- print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
103
-
104
- # Create a controller for each pipeline stage.
105
- controllers: List[Controller] = []
106
- for i in range(args.pipeline_parallel_size):
107
- controller = Controller(
108
- stage_id=i,
109
- stage_devices=all_stage_devices[i],
110
- world_size=world_size,
111
- pipeline_parallel_size=args.pipeline_parallel_size,
112
- tensor_parallel_size=args.tensor_parallel_size,
113
- distributed_init_method=distributed_init_method,
114
- model_name=args.model,
115
- block_size=args.block_size,
116
- num_gpu_blocks=num_gpu_blocks,
117
- num_cpu_blocks=num_cpu_blocks,
118
- dtype=args.dtype,
119
- seed=args.seed,
120
- model_path=args.model_path,
121
- )
122
- controllers.append(controller)
123
-
124
- # Create a frontend.
125
- frontend = Frontend(
126
- model_name=args.model,
127
- block_size=args.block_size,
128
- )
129
-
130
- # Create a scheduler.
131
- scheduler = Scheduler(
132
- frontend=frontend,
133
- controllers=controllers,
134
- block_size=args.block_size,
135
- num_gpu_blocks=num_gpu_blocks,
136
- num_cpu_blocks=num_cpu_blocks,
137
- max_num_batched_tokens=args.max_batch_size,
138
- )
139
- # Connect the controllers.
140
- for i in range(len(controllers) - 1):
141
- controllers[i].set_next(controllers[i + 1])
142
- controllers[-1].set_next(scheduler)
143
-
144
- # Test the following inputs.
145
- test_inputs = [
146
- ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
147
- ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
148
- ('The future of cloud computing is', {}), # Use default parameters.
149
- ]
150
- while True:
151
- if test_inputs:
152
- text, sampling_params = test_inputs.pop(0)
153
- frontend.query(text, **sampling_params)
154
- scheduler.step()
155
- if not (scheduler.pending or scheduler.running or test_inputs):
156
- break
157
-
158
-
159
-if __name__ == '__main__':
160
- parser = argparse.ArgumentParser(description='CacheFlow server')
164
+def add_server_arguments(parser: argparse.ArgumentParser):
165
# Model arguments
166
parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
167
parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
@@ -173,6 +177,4 @@ if __name__ == '__main__':
177
parser.add_argument('--seed', type=int, default=0, help='random seed')
178
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
179
parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
176
- args = parser.parse_args()
177
-
178
- main(args)
180
+ return parser
cacheflow/master/simple_frontend.py
renamed
+8
-23
@@ -3,12 +3,11 @@ from typing import List, Optional, Set, Tuple
3
from transformers import AutoTokenizer
4
5
from cacheflow.sampling_params import SamplingParams
6
-from cacheflow.sequence import Sequence
7
-from cacheflow.sequence import SequenceGroup
6
+from cacheflow.sequence import Sequence, SequenceGroup
7
from cacheflow.utils import Counter
8
9
11
-class Frontend:
10
+class SimpleFrontend:
11
12
def __init__(
13
self,
@@ -22,30 +21,16 @@ class Frontend:
21
self.seq_counter = Counter()
22
self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
23
24
+ def add_eos_token(self, sampling_params: SamplingParams) -> SamplingParams:
25
+ # Stop generation when we see an EOS token.
26
+ sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
27
+ return sampling_params
28
+
29
def query(
30
self,
31
prompt: str,
28
- n: int = 1,
29
- temperature: float = 1.0,
30
- top_p: float = 1.0,
31
- use_beam_search: bool = False,
32
- stop_token_ids: Set[int] = set(),
33
- max_num_steps: int = 16, # From OpenAI API.
34
- num_logprobs: int = 0,
35
- context_window_size: Optional[int] = None,
32
+ sampling_params: SamplingParams,
33
) -> None:
37
- # Stop when we see an EOS token.
38
- stop_token_ids.add(self.tokenizer.eos_token_id)
39
- sampling_params = SamplingParams(
40
- n=n,
41
- temperature=temperature,
42
- top_p=top_p,
43
- use_beam_search=use_beam_search,
44
- stop_token_ids=stop_token_ids,
45
- max_num_steps=max_num_steps,
46
- num_logprobs=num_logprobs,
47
- context_window_size=context_window_size,
48
- )
34
token_ids = self.tokenizer.encode(prompt)
35
self._add_query(token_ids, sampling_params)
36
cacheflow/models/memory_analyzer.py
+9
-9
@@ -1,9 +1,7 @@
1
import torch
2
from transformers import AutoConfig
3
4
-from cacheflow.models.utils import get_cpu_memory
4
from cacheflow.models.utils import get_dtype_size
6
-from cacheflow.models.utils import get_gpu_memory
5
6
_GiB = 1 << 30
7
@@ -31,11 +29,15 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
29
model_name: str,
30
block_size: int,
31
dtype: torch.dtype,
32
+ gpu_memory: int,
33
+ cpu_memory: int,
34
tensor_parallel_size: int,
35
) -> None:
36
self.model_name = model_name
37
self.block_size = block_size
38
self.dtype = dtype
39
+ self.gpu_memory = gpu_memory
40
+ self.cpu_memory = cpu_memory
41
self.tensor_parallel_size = tensor_parallel_size
42
43
config = AutoConfig.from_pretrained(model_name)
@@ -106,8 +108,7 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
108
memory_utilization: float = 0.95,
109
) -> int:
110
# NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
109
- gpu_memory = get_gpu_memory()
110
- usable_memory = int(memory_utilization * gpu_memory)
111
+ usable_memory = int(memory_utilization * self.gpu_memory)
112
113
param_size = self._get_param_size()
114
act_size = self._get_max_act_size(max_num_batched_tokens)
@@ -122,16 +123,15 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
123
swap_space: int,
124
) -> int:
125
swap_space = swap_space * _GiB
125
- cpu_memory = get_cpu_memory()
126
- if swap_space > 0.8 * cpu_memory:
126
+ if swap_space > 0.8 * self.cpu_memory:
127
raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
128
'takes more than 80% of the available memory '
129
- f'({cpu_memory / _GiB:.2f} GiB).'
129
+ f'({self.cpu_memory / _GiB:.2f} GiB).'
130
'Please check the swap space size.')
131
- if swap_space > 0.5 * cpu_memory:
131
+ if swap_space > 0.5 * self.cpu_memory:
132
print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
133
'takes more than 50% of the available memory '
134
- f'({cpu_memory / _GiB:.2f} GiB).'
134
+ f'({self.cpu_memory / _GiB:.2f} GiB).'
135
'This may slow the system performance.')
136
max_num_blocks = swap_space // self._get_cache_block_size()
137
return max_num_blocks
cacheflow/models/model_utils.py
+4
-1
@@ -44,11 +44,14 @@ def get_memory_analyzer(
44
model_name: str,
45
block_size: int,
46
dtype: Union[torch.dtype, str],
47
+ gpu_memory: int,
48
+ cpu_memory: int,
49
tensor_parallel_size: int = 1,
50
) -> CacheFlowMemoryAnalyzer:
51
torch_dtype = get_torch_dtype(dtype)
52
for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
53
if model_class in model_name:
54
return memory_analyzer(
53
- model_name, block_size, torch_dtype, tensor_parallel_size)
55
+ model_name, block_size, torch_dtype, gpu_memory, cpu_memory,
56
+ tensor_parallel_size)
57
raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/utils.py
-11
@@ -1,9 +1,5 @@
1
from typing import Union
2
3
-import random
4
-
5
-import numpy as np
6
-import psutil
3
import torch
4
5
_STR_DTYPE_TO_TORCH_DTYPE = {
@@ -26,10 +22,3 @@ def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
22
torch_dtype = get_torch_dtype(dtype)
23
return torch.tensor([], dtype=torch_dtype).element_size()
24
29
-
30
-def get_gpu_memory(gpu: int = 0) -> int:
31
- return torch.cuda.get_device_properties(gpu).total_memory
32
-
33
-
34
-def get_cpu_memory() -> int:
35
- return psutil.virtual_memory().total
cacheflow/sampling_params.py
+14
-1
@@ -1,4 +1,4 @@
1
-from typing import Optional, Set
1
+from typing import Optional, Set, Dict
2
3
4
class SamplingParams:
@@ -69,3 +69,16 @@ class SamplingParams:
69
f'max_num_steps={self.max_num_steps}, '
70
f'num_logprobs={self.num_logprobs}, '
71
f'context_window_size={self.context_window_size})')
72
+
73
+ @classmethod
74
+ def from_dict(cls, d: Dict) -> 'SamplingParams':
75
+ return cls(
76
+ n=d.get('n', 1),
77
+ temperature=d.get('temperature', 1.0),
78
+ top_p=d.get('top_p', 1.0),
79
+ use_beam_search=d.get('use_beam_search', False),
80
+ stop_token_ids=set(d.get('stop_token_ids', set())),
81
+ max_num_steps=d.get('max_num_steps', 16),
82
+ num_logprobs=d.get('num_logprobs', 0),
83
+ context_window_size=d.get('context_window_size', None),
84
+ )
cacheflow/utils.py
+10
@@ -1,5 +1,6 @@
1
import enum
2
import random
3
+import psutil
4
5
import numpy as np
6
import torch
@@ -26,6 +27,7 @@ class Counter:
27
def reset(self) -> None:
28
self.counter = 0
29
30
+
31
def set_random_seed(seed: int):
32
random.seed(seed)
33
np.random.seed(seed)
@@ -35,3 +37,11 @@ def set_random_seed(seed: int):
37
38
if model_parallel_is_initialized():
39
model_parallel_cuda_manual_seed(seed)
40
+
41
+
42
+def get_gpu_memory(gpu: int = 0) -> int:
43
+ return torch.cuda.get_device_properties(gpu).total_memory
44
+
45
+
46
+def get_cpu_memory() -> int:
47
+ return psutil.virtual_memory().total
playground/http_client.py
new
+20
@@ -0,0 +1,20 @@
1
+import requests
2
+import json
3
+
4
+def http_bot():
5
+ prompt = "How are you? I'm fine."
6
+
7
+ headers = {"User-Agent": "Test Client"}
8
+ pload = {
9
+ "prompt": prompt,
10
+ }
11
+ response = requests.post("http://localhost:10002", headers=headers, json=pload, stream=True)
12
+
13
+ for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
14
+ if chunk:
15
+ data = json.loads(chunk.decode("utf-8"))
16
+ output = data["text"]
17
+ yield output
18
+
19
+for h in http_bot():
20
+ print(h, end="", flush=True)
\ No newline at end of file
playground/streaming_fastapi_worker.py
new
+40
@@ -0,0 +1,40 @@
1
+import argparse
2
+import asyncio
3
+import time
4
+from typing import Union
5
+import json
6
+
7
+from fastapi import FastAPI, Request
8
+from fastapi.responses import StreamingResponse
9
+import uvicorn
10
+
11
+
12
+app = FastAPI()
13
+
14
+
15
+async def text_streamer(args):
16
+ context = args["prompt"]
17
+ words = context.split(" ")
18
+ for word in words:
19
+ await asyncio.sleep(1)
20
+ print("word:", word)
21
+ ret = {
22
+ "text": word + " ",
23
+ "error": 0,
24
+ }
25
+ yield (json.dumps(ret) + "\0").encode("utf-8")
26
+
27
+
28
+@app.post("/")
29
+async def read_root(request: Request):
30
+ args = await request.json()
31
+ return StreamingResponse(text_streamer(args))
32
+
33
+
34
+if __name__ == "__main__":
35
+ parser = argparse.ArgumentParser()
36
+ parser.add_argument("--host", type=str, default="localhost")
37
+ parser.add_argument("--port", type=int, default=10002)
38
+ args = parser.parse_args()
39
+
40
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info")
simple_server.py
new
+71
@@ -0,0 +1,71 @@
1
+import argparse
2
+from typing import List
3
+
4
+from cacheflow.master.simple_frontend import SimpleFrontend
5
+from cacheflow.master.server import (Server, add_server_arguments,
6
+ initialize_ray_cluster)
7
+from cacheflow.sampling_params import SamplingParams
8
+from cacheflow.utils import get_gpu_memory, get_cpu_memory
9
+
10
+def main(args: argparse.Namespace):
11
+ # TODO(zhuohan): Support pipeline parallelism.
12
+ assert args.pipeline_parallel_size == 1, (
13
+ 'Pipeline parallelism is not supported yet.')
14
+
15
+ (num_nodes, num_devices_per_node, distributed_init_method,
16
+ all_stage_devices) = (
17
+ initialize_ray_cluster(
18
+ pipeline_parallel_size=args.pipeline_parallel_size,
19
+ tensor_parallel_size=args.tensor_parallel_size))
20
+
21
+ # Create a server.
22
+ server = Server(
23
+ model=args.model,
24
+ model_path=args.model_path,
25
+ pipeline_parallel_size=args.pipeline_parallel_size,
26
+ tensor_parallel_size=args.tensor_parallel_size,
27
+ block_size=args.block_size,
28
+ dtype=args.dtype,
29
+ seed=args.seed,
30
+ swap_space=args.swap_space,
31
+ max_batch_size=args.max_batch_size,
32
+ num_nodes=num_nodes,
33
+ num_devices_per_node=num_devices_per_node,
34
+ distributed_init_method=distributed_init_method,
35
+ all_stage_devices=all_stage_devices,
36
+ gpu_memory=get_gpu_memory(),
37
+ cpu_memory=get_cpu_memory(),
38
+ )
39
+
40
+ # Create a frontend.
41
+ frontend = SimpleFrontend(
42
+ model_name=args.model,
43
+ block_size=args.block_size,
44
+ )
45
+
46
+ # Test the following inputs.
47
+ test_inputs = [
48
+ ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
49
+ ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
50
+ ('The future of cloud computing is', {}), # Use default parameters.
51
+ ]
52
+ while True:
53
+ if test_inputs:
54
+ text, sampling_params_dict = test_inputs.pop(0)
55
+ sampling_params = SamplingParams.from_dict(sampling_params_dict)
56
+ sampling_params = frontend.add_eos_token(sampling_params)
57
+ frontend.query(text, sampling_params)
58
+ server.add_sequence_groups(frontend.get_inputs())
59
+ updated_seq_groups = server.step()
60
+ for seq_group in updated_seq_groups:
61
+ if seq_group.is_finished():
62
+ frontend.print_response(seq_group)
63
+ if not (server.has_unfinished_requests() or test_inputs):
64
+ break
65
+
66
+
67
+if __name__ == '__main__':
68
+ parser = argparse.ArgumentParser(description='CacheFlow simple server.')
69
+ parser = add_server_arguments(parser)
70
+ args = parser.parse_args()
71
+ main(args)