@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 721fa3df15
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

FastAPI-based working frontend (#10)

Zhuohan Li committed Mar 29, 2023 at 14:48 UTC 721fa3df155e5649bbe2188517594f24f4e63523
15 files changed +536 -146
README.md
+39 -2
@@ -8,9 +8,46 @@ pip install flash-attn # This may take up to 10 mins.
8 pip install -e .
9 ```
10
11 -## Run
11 +## Test simple server
12
13 ```bash
14 ray start --head
15 -python server.py [--tensor-parallel-size <N>]
15 +python simple_server.py
16 +```
17 +
18 +The detailed arguments for `simple_server.py` can be found by:
19 +```bash
20 +python simple_server.py --help
21 +```
22 +
23 +## FastAPI server
24 +
25 +Install the following additional dependencies:
26 +```bash
27 +pip install fastapi uvicorn
28 +```
29 +
30 +To start the server:
31 +```bash
32 +ray start --head
33 +python -m cacheflow.http_frontend.fastapi_frontend
34 +```
35 +
36 +To test the server:
37 +```bash
38 +python -m cacheflow.http_frontend.test_cli_client
39 +```
40 +
41 +## Gradio web server
42 +
43 +Install the following additional dependencies:
44 +```bash
45 +pip install gradio
46 +```
47 +
48 +Start the server:
49 +```bash
50 +python -m cacheflow.http_frontend.fastapi_frontend
51 +# At another terminal
52 +python -m cacheflow.http_frontend.gradio_webserver
53 ```
cacheflow/http_frontend/fastapi_frontend.py new
+152
@@ -0,0 +1,152 @@
1 +import argparse
2 +import asyncio
3 +import time
4 +from typing import List, Dict
5 +import json
6 +
7 +import ray
8 +from transformers import AutoTokenizer
9 +from fastapi import FastAPI, Request
10 +from fastapi.responses import StreamingResponse
11 +import uvicorn
12 +
13 +from cacheflow.sampling_params import SamplingParams
14 +from cacheflow.sequence import Sequence, SequenceGroup
15 +from cacheflow.master.server import (Server, add_server_arguments,
16 + initialize_ray_cluster)
17 +from cacheflow.worker.controller import DeviceID
18 +from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
19 +
20 +app = FastAPI()
21 +
22 +class FastAPIFrontend:
23 + def __init__(
24 + self,
25 + model: str,
26 + model_path: str,
27 + pipeline_parallel_size: int,
28 + tensor_parallel_size: int,
29 + block_size: int,
30 + dtype: str,
31 + seed: int,
32 + swap_space: int,
33 + max_batch_size: int,
34 + num_nodes: int,
35 + num_devices_per_node: int,
36 + distributed_init_method: str,
37 + all_stage_devices: List[List[DeviceID]],
38 + ):
39 + self.block_size = block_size
40 +
41 + self.tokenizer = AutoTokenizer.from_pretrained(model)
42 + self.seq_group_counter = Counter()
43 + self.seq_counter = Counter()
44 + remote_server_class = ray.remote(num_cpus=0)(Server)
45 + self.server = remote_server_class.remote(
46 + model=model,
47 + model_path=model_path,
48 + pipeline_parallel_size=pipeline_parallel_size,
49 + tensor_parallel_size=tensor_parallel_size,
50 + block_size=block_size,
51 + dtype=dtype,
52 + seed=seed,
53 + swap_space=swap_space,
54 + max_batch_size=max_batch_size,
55 + num_nodes=num_nodes,
56 + num_devices_per_node=num_devices_per_node,
57 + distributed_init_method=distributed_init_method,
58 + all_stage_devices=all_stage_devices,
59 + gpu_memory=get_gpu_memory(),
60 + cpu_memory=get_cpu_memory(),
61 + )
62 +
63 + self.running_seq_groups: Dict[int, SequenceGroup] = {}
64 + self.sequence_group_events: Dict[int, asyncio.Event] = {}
65 + self.is_server_running = False
66 +
67 + async def server_step(self):
68 + self.is_server_running = True
69 + updated_seq_groups = await self.server.step.remote()
70 + self.is_server_running = False
71 + for seq_group in updated_seq_groups:
72 + group_id = seq_group.group_id
73 + self.running_seq_groups[group_id] = seq_group
74 + self.sequence_group_events[group_id].set()
75 +
76 + async def generate(self, request_dict: Dict):
77 + prompt = request_dict["prompt"]
78 + sampling_params = SamplingParams.from_dict(request_dict)
79 + sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
80 + token_ids = self.tokenizer.encode(prompt)
81 + seqs: List[Sequence] = []
82 + for _ in range(sampling_params.n):
83 + seq_id = next(self.seq_counter)
84 + seq = Sequence(seq_id, token_ids, block_size=self.block_size)
85 + seqs.append(seq)
86 +
87 + group_id = next(self.seq_group_counter)
88 + seq_group = SequenceGroup(group_id, seqs)
89 + group_event = asyncio.Event()
90 + self.sequence_group_events[group_id] = group_event
91 + await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
92 + while True:
93 + if not self.is_server_running:
94 + await self.server_step()
95 + # Wait for new output. Add a 1s timeout to prevent dead lock.
96 + await asyncio.wait_for(group_event.wait(), timeout=1)
97 + group_event.clear()
98 + seq_group = self.running_seq_groups[group_id]
99 + all_outputs = []
100 + for seq in seq_group.seqs:
101 + token_ids = seq.get_token_ids()
102 + output = self.tokenizer.decode(token_ids, skip_special_tokens=True)
103 + all_outputs.append(output)
104 + ret = {
105 + "text": all_outputs,
106 + "error": 0,
107 + }
108 + yield (json.dumps(ret) + "\0").encode("utf-8")
109 + if seq_group.is_finished():
110 + break
111 +
112 +
113 +@app.post("/generate")
114 +async def generate_stream(request: Request):
115 + request_dict = await request.json()
116 + return StreamingResponse(frontend.generate(request_dict))
117 +
118 +
119 +if __name__ == "__main__":
120 + parser = argparse.ArgumentParser()
121 + parser.add_argument("--host", type=str, default="localhost")
122 + parser.add_argument("--port", type=int, default=10002)
123 + parser = add_server_arguments(parser)
124 + args = parser.parse_args()
125 +
126 + # TODO(zhuohan): Support pipeline parallelism.
127 + assert args.pipeline_parallel_size == 1, (
128 + 'Pipeline parallelism is not supported yet.')
129 +
130 + (num_nodes, num_devices_per_node, distributed_init_method,
131 + all_stage_devices) = (
132 + initialize_ray_cluster(
133 + pipeline_parallel_size=args.pipeline_parallel_size,
134 + tensor_parallel_size=args.tensor_parallel_size))
135 +
136 + frontend = FastAPIFrontend(
137 + model=args.model,
138 + model_path=args.model_path,
139 + pipeline_parallel_size=args.pipeline_parallel_size,
140 + tensor_parallel_size=args.tensor_parallel_size,
141 + block_size=args.block_size,
142 + dtype=args.dtype,
143 + seed=args.seed,
144 + swap_space=args.swap_space,
145 + max_batch_size=args.max_batch_size,
146 + num_nodes=num_nodes,
147 + num_devices_per_node=num_devices_per_node,
148 + distributed_init_method=distributed_init_method,
149 + all_stage_devices=all_stage_devices,
150 + )
151 +
152 + uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/http_frontend/gradio_webserver.py new
+43
@@ -0,0 +1,43 @@
1 +import argparse
2 +import json
3 +import time
4 +
5 +import gradio as gr
6 +import requests
7 +
8 +
9 +def http_bot(prompt):
10 + headers = {"User-Agent": "Cacheflow Client"}
11 + pload = {
12 + "prompt": prompt,
13 + "max_num_steps": 128,
14 + }
15 + response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
16 +
17 + for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
18 + if chunk:
19 + data = json.loads(chunk.decode("utf-8"))
20 + output = data["text"][0]
21 + yield output
22 +
23 +
24 +def build_demo():
25 + with gr.Blocks() as demo:
26 + gr.Markdown(
27 + "# Cacheflow demo\n"
28 + )
29 + inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")# .style(container=False)
30 + outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
31 + inputbox.submit(http_bot, [inputbox], [outputbox])
32 + return demo
33 +
34 +
35 +if __name__ == "__main__":
36 + parser = argparse.ArgumentParser()
37 + parser.add_argument("--host", type=str, default="localhost")
38 + parser.add_argument("--port", type=int, default=10003)
39 + parser.add_argument("--model-url", type=str, default="http://localhost:10002/generate")
40 + args = parser.parse_args()
41 +
42 + demo = build_demo()
43 + demo.queue(concurrency_count=100).launch(server_name=args.host, server_port=args.port)
\ No newline at end of file
cacheflow/http_frontend/test_cli_client.py new
+23
@@ -0,0 +1,23 @@
1 +import requests
2 +import json
3 +
4 +def http_request():
5 + prompt = "Ion Stoica is a"
6 +
7 + headers = {"User-Agent": "Test Client"}
8 + pload = {
9 + "prompt": prompt,
10 + "n": 4,
11 + "use_beam_search": True,
12 + "temperature": 0.0,
13 + }
14 + response = requests.post("http://localhost:10002/generate", headers=headers, json=pload, stream=True)
15 +
16 + for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
17 + if chunk:
18 + data = json.loads(chunk.decode("utf-8"))
19 + output = data["text"]
20 + yield output
21 +
22 +for h in http_request():
23 + print(h, flush=True)
cacheflow/master/scheduler.py
+14 -12
@@ -1,7 +1,6 @@
1 -from typing import Dict, List
1 +from typing import Dict, List, Tuple
2
3 from cacheflow.master.block_manager import BlockSpaceManager
4 -from cacheflow.master.frontend import Frontend
4 from cacheflow.sampling_params import SamplingParams
5 from cacheflow.sequence import Sequence
6 from cacheflow.sequence import SequenceGroup
@@ -14,14 +13,12 @@ class Scheduler:
13
14 def __init__(
15 self,
17 - frontend: Frontend,
16 controllers: List,
17 block_size: int,
18 num_gpu_blocks: int,
19 num_cpu_blocks: int,
20 max_num_batched_tokens: int,
21 ) -> None:
24 - self.frontend = frontend
22 self.controllers = controllers
23 self.block_size = block_size
24 self.num_gpu_blocks = num_gpu_blocks
@@ -47,9 +44,12 @@ class Scheduler:
44 # Pending sequence groups (FIFO).
45 self.pending: List[SequenceGroup] = []
46
50 - def _fetch_inputs(self) -> None:
51 - inputs = self.frontend.get_inputs()
52 - for seq_group, sampling_params in inputs:
47 + def add_sequence_groups(
48 + self,
49 + sequence_groups: List[Tuple[SequenceGroup, SamplingParams]],
50 + ) -> None:
51 + # Add sequence groups to the pending queue.
52 + for seq_group, sampling_params in sequence_groups:
53 self.pending.append(seq_group)
54 self.sampling_params[seq_group.group_id] = sampling_params
55
@@ -104,7 +104,7 @@ class Scheduler:
104 seq.status = SequenceStatus.SWAPPED
105 self.swapped.append(seq_group)
106
107 - def step(self) -> None:
107 + def step(self) -> List[SequenceGroup]:
108 # Blocks that need to be swaped or copied before model execution.
109 blocks_to_swap_in: Dict[int, int] = {}
110 blocks_to_swap_out: Dict[int, int] = {}
@@ -158,7 +158,6 @@ class Scheduler:
158 # 3. Join new sequences if possible.
159 # NOTE: Here we implicitly assume FCFS scheduling.
160 # TODO(woosuk): Add a batching policy to control the batch size.
161 - self._fetch_inputs()
161 if not self.swapped:
162 for i, seq_group in enumerate(self.pending):
163 num_prompt_tokens = seq_group.seqs[0].get_len()
@@ -176,6 +175,8 @@ class Scheduler:
175
176 # 4. Create input data structures.
177 input_seq_groups: List[SequenceGroupInputs] = []
178 + updated_seq_groups: List[SequenceGroup] = self.running.copy()
179 +
180 for seq_group in self.running:
181 group_id = seq_group.group_id
182 num_steps = self.num_steps[group_id]
@@ -219,6 +220,8 @@ class Scheduler:
220 blocks_to_copy,
221 )
222
223 + return updated_seq_groups
224 +
225 def post_step(
226 self,
227 seq_outputs: Dict[int, SequenceOutputs],
@@ -268,13 +271,12 @@ class Scheduler:
271 running: List[SequenceGroup] = []
272 for seq_group in self.running:
273 if seq_group.is_finished():
271 - self._return(seq_group)
274 + self._free_seq_group(seq_group)
275 else:
276 running.append(seq_group)
277 self.running = running
278
276 - def _return(self, seq_group: SequenceGroup) -> None:
279 + def _free_seq_group(self, seq_group: SequenceGroup) -> None:
280 group_id = seq_group.group_id
281 del self.num_steps[group_id]
282 del self.sampling_params[group_id]
280 - self.frontend.print_response(seq_group)
cacheflow/master/server.py renamed
+89 -87
@@ -1,13 +1,98 @@
1 import argparse
2 +from typing import List, Tuple
3 import random
3 -from typing import List, Tuple, Dict
4
5 import ray
6
7 -from cacheflow.master.frontend import Frontend
7 from cacheflow.master.scheduler import Scheduler
8 from cacheflow.models import get_memory_analyzer
9 from cacheflow.worker.controller import Controller, DeviceID
10 +from cacheflow.sequence import SequenceGroup
11 +from cacheflow.sampling_params import SamplingParams
12 +
13 +class Server:
14 + def __init__(
15 + self,
16 + model: str,
17 + model_path: str,
18 + pipeline_parallel_size: int,
19 + tensor_parallel_size: int,
20 + block_size: int,
21 + dtype: str,
22 + seed: int,
23 + swap_space: int,
24 + max_batch_size: int,
25 + num_nodes: int,
26 + num_devices_per_node: int,
27 + distributed_init_method: str,
28 + all_stage_devices: List[List[DeviceID]],
29 + gpu_memory: int,
30 + cpu_memory: int,
31 + ):
32 + self.num_nodes = num_nodes
33 + self.num_devices_per_node = num_devices_per_node
34 + self.world_size = pipeline_parallel_size * tensor_parallel_size
35 +
36 + self.memory_analyzer = get_memory_analyzer(
37 + model_name=model,
38 + block_size=block_size,
39 + dtype=dtype,
40 + gpu_memory=gpu_memory,
41 + cpu_memory=cpu_memory,
42 + tensor_parallel_size=tensor_parallel_size,
43 + )
44 + self.num_gpu_blocks = self.memory_analyzer.get_max_num_gpu_blocks(
45 + max_num_batched_tokens=max_batch_size)
46 + self.num_cpu_blocks = self.memory_analyzer.get_max_num_cpu_blocks(
47 + swap_space=swap_space)
48 + print(f'# GPU blocks: {self.num_gpu_blocks}, '
49 + f'# CPU blocks: {self.num_cpu_blocks}')
50 +
51 + # Create a controller for each pipeline stage.
52 + self.controllers: List[Controller] = []
53 + for i in range(pipeline_parallel_size):
54 + controller = Controller(
55 + stage_id=i,
56 + stage_devices=all_stage_devices[i],
57 + world_size=self.world_size,
58 + pipeline_parallel_size=pipeline_parallel_size,
59 + tensor_parallel_size=tensor_parallel_size,
60 + distributed_init_method=distributed_init_method,
61 + model_name=model,
62 + block_size=block_size,
63 + num_gpu_blocks=self.num_gpu_blocks,
64 + num_cpu_blocks=self.num_cpu_blocks,
65 + dtype=dtype,
66 + seed=seed,
67 + model_path=model_path,
68 + )
69 + self.controllers.append(controller)
70 +
71 + # Create a scheduler.
72 + self.scheduler = Scheduler(
73 + controllers=self.controllers,
74 + block_size=block_size,
75 + num_gpu_blocks=self.num_gpu_blocks,
76 + num_cpu_blocks=self.num_cpu_blocks,
77 + max_num_batched_tokens=max_batch_size,
78 + )
79 + # Connect the controllers.
80 + for i in range(len(self.controllers) - 1):
81 + self.controllers[i].set_next(self.controllers[i + 1])
82 + self.controllers[-1].set_next(self.scheduler)
83 +
84 + def add_sequence_groups(
85 + self,
86 + sequence_groups: List[Tuple[SequenceGroup, SamplingParams]]
87 + ):
88 + self.scheduler.add_sequence_groups(sequence_groups)
89 +
90 + def step(self):
91 + return self.scheduler.step()
92 +
93 + def has_unfinished_requests(self):
94 + return (self.scheduler.pending or self.scheduler.running or
95 + self.scheduler.swapped)
96
97
98 def initialize_ray_cluster(
@@ -76,88 +161,7 @@ def initialize_ray_cluster(
161 all_stage_devices)
162
163
79 -def main(args: argparse.Namespace):
80 - # TODO(zhuohan): Support pipeline parallelism.
81 - assert args.pipeline_parallel_size == 1, (
82 - 'Pipeline parallelism is not supported yet.')
83 -
84 - (num_nodes, num_devices_per_node, distributed_init_method,
85 - all_stage_devices) = (
86 - initialize_ray_cluster(
87 - pipeline_parallel_size=args.pipeline_parallel_size,
88 - tensor_parallel_size=args.tensor_parallel_size))
89 -
90 - world_size = args.pipeline_parallel_size * args.tensor_parallel_size
91 -
92 - memory_analyzer = get_memory_analyzer(
93 - model_name=args.model,
94 - block_size=args.block_size,
95 - dtype=args.dtype,
96 - tensor_parallel_size=args.tensor_parallel_size,
97 - )
98 - num_gpu_blocks = memory_analyzer.get_max_num_gpu_blocks(
99 - max_num_batched_tokens=args.max_batch_size)
100 - num_cpu_blocks = memory_analyzer.get_max_num_cpu_blocks(
101 - swap_space=args.swap_space)
102 - print(f'# GPU blocks: {num_gpu_blocks}, # CPU blocks: {num_cpu_blocks}')
103 -
104 - # Create a controller for each pipeline stage.
105 - controllers: List[Controller] = []
106 - for i in range(args.pipeline_parallel_size):
107 - controller = Controller(
108 - stage_id=i,
109 - stage_devices=all_stage_devices[i],
110 - world_size=world_size,
111 - pipeline_parallel_size=args.pipeline_parallel_size,
112 - tensor_parallel_size=args.tensor_parallel_size,
113 - distributed_init_method=distributed_init_method,
114 - model_name=args.model,
115 - block_size=args.block_size,
116 - num_gpu_blocks=num_gpu_blocks,
117 - num_cpu_blocks=num_cpu_blocks,
118 - dtype=args.dtype,
119 - seed=args.seed,
120 - model_path=args.model_path,
121 - )
122 - controllers.append(controller)
123 -
124 - # Create a frontend.
125 - frontend = Frontend(
126 - model_name=args.model,
127 - block_size=args.block_size,
128 - )
129 -
130 - # Create a scheduler.
131 - scheduler = Scheduler(
132 - frontend=frontend,
133 - controllers=controllers,
134 - block_size=args.block_size,
135 - num_gpu_blocks=num_gpu_blocks,
136 - num_cpu_blocks=num_cpu_blocks,
137 - max_num_batched_tokens=args.max_batch_size,
138 - )
139 - # Connect the controllers.
140 - for i in range(len(controllers) - 1):
141 - controllers[i].set_next(controllers[i + 1])
142 - controllers[-1].set_next(scheduler)
143 -
144 - # Test the following inputs.
145 - test_inputs = [
146 - ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
147 - ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
148 - ('The future of cloud computing is', {}), # Use default parameters.
149 - ]
150 - while True:
151 - if test_inputs:
152 - text, sampling_params = test_inputs.pop(0)
153 - frontend.query(text, **sampling_params)
154 - scheduler.step()
155 - if not (scheduler.pending or scheduler.running or test_inputs):
156 - break
157 -
158 -
159 -if __name__ == '__main__':
160 - parser = argparse.ArgumentParser(description='CacheFlow server')
164 +def add_server_arguments(parser: argparse.ArgumentParser):
165 # Model arguments
166 parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
167 parser.add_argument('--model-path', type=str, default='~/.cacheflow/model_weights',
@@ -173,6 +177,4 @@ if __name__ == '__main__':
177 parser.add_argument('--seed', type=int, default=0, help='random seed')
178 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
179 parser.add_argument('--max-batch-size', type=int, default=2560, help='maximum number of batched tokens')
176 - args = parser.parse_args()
177 -
178 - main(args)
180 + return parser
cacheflow/master/simple_frontend.py renamed
+8 -23
@@ -3,12 +3,11 @@ from typing import List, Optional, Set, Tuple
3 from transformers import AutoTokenizer
4
5 from cacheflow.sampling_params import SamplingParams
6 -from cacheflow.sequence import Sequence
7 -from cacheflow.sequence import SequenceGroup
6 +from cacheflow.sequence import Sequence, SequenceGroup
7 from cacheflow.utils import Counter
8
9
11 -class Frontend:
10 +class SimpleFrontend:
11
12 def __init__(
13 self,
@@ -22,30 +21,16 @@ class Frontend:
21 self.seq_counter = Counter()
22 self.inputs: List[Tuple[SequenceGroup, SamplingParams]] = []
23
24 + def add_eos_token(self, sampling_params: SamplingParams) -> SamplingParams:
25 + # Stop generation when we see an EOS token.
26 + sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
27 + return sampling_params
28 +
29 def query(
30 self,
31 prompt: str,
28 - n: int = 1,
29 - temperature: float = 1.0,
30 - top_p: float = 1.0,
31 - use_beam_search: bool = False,
32 - stop_token_ids: Set[int] = set(),
33 - max_num_steps: int = 16, # From OpenAI API.
34 - num_logprobs: int = 0,
35 - context_window_size: Optional[int] = None,
32 + sampling_params: SamplingParams,
33 ) -> None:
37 - # Stop when we see an EOS token.
38 - stop_token_ids.add(self.tokenizer.eos_token_id)
39 - sampling_params = SamplingParams(
40 - n=n,
41 - temperature=temperature,
42 - top_p=top_p,
43 - use_beam_search=use_beam_search,
44 - stop_token_ids=stop_token_ids,
45 - max_num_steps=max_num_steps,
46 - num_logprobs=num_logprobs,
47 - context_window_size=context_window_size,
48 - )
34 token_ids = self.tokenizer.encode(prompt)
35 self._add_query(token_ids, sampling_params)
36
cacheflow/models/memory_analyzer.py
+9 -9
@@ -1,9 +1,7 @@
1 import torch
2 from transformers import AutoConfig
3
4 -from cacheflow.models.utils import get_cpu_memory
4 from cacheflow.models.utils import get_dtype_size
6 -from cacheflow.models.utils import get_gpu_memory
5
6 _GiB = 1 << 30
7
@@ -31,11 +29,15 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
29 model_name: str,
30 block_size: int,
31 dtype: torch.dtype,
32 + gpu_memory: int,
33 + cpu_memory: int,
34 tensor_parallel_size: int,
35 ) -> None:
36 self.model_name = model_name
37 self.block_size = block_size
38 self.dtype = dtype
39 + self.gpu_memory = gpu_memory
40 + self.cpu_memory = cpu_memory
41 self.tensor_parallel_size = tensor_parallel_size
42
43 config = AutoConfig.from_pretrained(model_name)
@@ -106,8 +108,7 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
108 memory_utilization: float = 0.95,
109 ) -> int:
110 # NOTE(woosuk): This assumes that the machine has homogeneous GPUs.
109 - gpu_memory = get_gpu_memory()
110 - usable_memory = int(memory_utilization * gpu_memory)
111 + usable_memory = int(memory_utilization * self.gpu_memory)
112
113 param_size = self._get_param_size()
114 act_size = self._get_max_act_size(max_num_batched_tokens)
@@ -122,16 +123,15 @@ class OPTMemoryAnalyzer(CacheFlowMemoryAnalyzer):
123 swap_space: int,
124 ) -> int:
125 swap_space = swap_space * _GiB
125 - cpu_memory = get_cpu_memory()
126 - if swap_space > 0.8 * cpu_memory:
126 + if swap_space > 0.8 * self.cpu_memory:
127 raise ValueError(f'The swap space ({swap_space / _GiB:.2f} GiB) '
128 'takes more than 80% of the available memory '
129 - f'({cpu_memory / _GiB:.2f} GiB).'
129 + f'({self.cpu_memory / _GiB:.2f} GiB).'
130 'Please check the swap space size.')
131 - if swap_space > 0.5 * cpu_memory:
131 + if swap_space > 0.5 * self.cpu_memory:
132 print(f'WARNING: The swap space ({swap_space / _GiB:.2f} GiB) '
133 'takes more than 50% of the available memory '
134 - f'({cpu_memory / _GiB:.2f} GiB).'
134 + f'({self.cpu_memory / _GiB:.2f} GiB).'
135 'This may slow the system performance.')
136 max_num_blocks = swap_space // self._get_cache_block_size()
137 return max_num_blocks
cacheflow/models/model_utils.py
+4 -1
@@ -44,11 +44,14 @@ def get_memory_analyzer(
44 model_name: str,
45 block_size: int,
46 dtype: Union[torch.dtype, str],
47 + gpu_memory: int,
48 + cpu_memory: int,
49 tensor_parallel_size: int = 1,
50 ) -> CacheFlowMemoryAnalyzer:
51 torch_dtype = get_torch_dtype(dtype)
52 for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
53 if model_class in model_name:
54 return memory_analyzer(
53 - model_name, block_size, torch_dtype, tensor_parallel_size)
55 + model_name, block_size, torch_dtype, gpu_memory, cpu_memory,
56 + tensor_parallel_size)
57 raise ValueError(f'Unsupported model name: {model_name}')
cacheflow/models/utils.py
-11
@@ -1,9 +1,5 @@
1 from typing import Union
2
3 -import random
4 -
5 -import numpy as np
6 -import psutil
3 import torch
4
5 _STR_DTYPE_TO_TORCH_DTYPE = {
@@ -26,10 +22,3 @@ def get_dtype_size(dtype: Union[torch.dtype, str]) -> int:
22 torch_dtype = get_torch_dtype(dtype)
23 return torch.tensor([], dtype=torch_dtype).element_size()
24
29 -
30 -def get_gpu_memory(gpu: int = 0) -> int:
31 - return torch.cuda.get_device_properties(gpu).total_memory
32 -
33 -
34 -def get_cpu_memory() -> int:
35 - return psutil.virtual_memory().total
cacheflow/sampling_params.py
+14 -1
@@ -1,4 +1,4 @@
1 -from typing import Optional, Set
1 +from typing import Optional, Set, Dict
2
3
4 class SamplingParams:
@@ -69,3 +69,16 @@ class SamplingParams:
69 f'max_num_steps={self.max_num_steps}, '
70 f'num_logprobs={self.num_logprobs}, '
71 f'context_window_size={self.context_window_size})')
72 +
73 + @classmethod
74 + def from_dict(cls, d: Dict) -> 'SamplingParams':
75 + return cls(
76 + n=d.get('n', 1),
77 + temperature=d.get('temperature', 1.0),
78 + top_p=d.get('top_p', 1.0),
79 + use_beam_search=d.get('use_beam_search', False),
80 + stop_token_ids=set(d.get('stop_token_ids', set())),
81 + max_num_steps=d.get('max_num_steps', 16),
82 + num_logprobs=d.get('num_logprobs', 0),
83 + context_window_size=d.get('context_window_size', None),
84 + )
cacheflow/utils.py
+10
@@ -1,5 +1,6 @@
1 import enum
2 import random
3 +import psutil
4
5 import numpy as np
6 import torch
@@ -26,6 +27,7 @@ class Counter:
27 def reset(self) -> None:
28 self.counter = 0
29
30 +
31 def set_random_seed(seed: int):
32 random.seed(seed)
33 np.random.seed(seed)
@@ -35,3 +37,11 @@ def set_random_seed(seed: int):
37
38 if model_parallel_is_initialized():
39 model_parallel_cuda_manual_seed(seed)
40 +
41 +
42 +def get_gpu_memory(gpu: int = 0) -> int:
43 + return torch.cuda.get_device_properties(gpu).total_memory
44 +
45 +
46 +def get_cpu_memory() -> int:
47 + return psutil.virtual_memory().total
playground/http_client.py new
+20
@@ -0,0 +1,20 @@
1 +import requests
2 +import json
3 +
4 +def http_bot():
5 + prompt = "How are you? I'm fine."
6 +
7 + headers = {"User-Agent": "Test Client"}
8 + pload = {
9 + "prompt": prompt,
10 + }
11 + response = requests.post("http://localhost:10002", headers=headers, json=pload, stream=True)
12 +
13 + for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
14 + if chunk:
15 + data = json.loads(chunk.decode("utf-8"))
16 + output = data["text"]
17 + yield output
18 +
19 +for h in http_bot():
20 + print(h, end="", flush=True)
\ No newline at end of file
playground/streaming_fastapi_worker.py new
+40
@@ -0,0 +1,40 @@
1 +import argparse
2 +import asyncio
3 +import time
4 +from typing import Union
5 +import json
6 +
7 +from fastapi import FastAPI, Request
8 +from fastapi.responses import StreamingResponse
9 +import uvicorn
10 +
11 +
12 +app = FastAPI()
13 +
14 +
15 +async def text_streamer(args):
16 + context = args["prompt"]
17 + words = context.split(" ")
18 + for word in words:
19 + await asyncio.sleep(1)
20 + print("word:", word)
21 + ret = {
22 + "text": word + " ",
23 + "error": 0,
24 + }
25 + yield (json.dumps(ret) + "\0").encode("utf-8")
26 +
27 +
28 +@app.post("/")
29 +async def read_root(request: Request):
30 + args = await request.json()
31 + return StreamingResponse(text_streamer(args))
32 +
33 +
34 +if __name__ == "__main__":
35 + parser = argparse.ArgumentParser()
36 + parser.add_argument("--host", type=str, default="localhost")
37 + parser.add_argument("--port", type=int, default=10002)
38 + args = parser.parse_args()
39 +
40 + uvicorn.run(app, host=args.host, port=args.port, log_level="info")
simple_server.py new
+71
@@ -0,0 +1,71 @@
1 +import argparse
2 +from typing import List
3 +
4 +from cacheflow.master.simple_frontend import SimpleFrontend
5 +from cacheflow.master.server import (Server, add_server_arguments,
6 + initialize_ray_cluster)
7 +from cacheflow.sampling_params import SamplingParams
8 +from cacheflow.utils import get_gpu_memory, get_cpu_memory
9 +
10 +def main(args: argparse.Namespace):
11 + # TODO(zhuohan): Support pipeline parallelism.
12 + assert args.pipeline_parallel_size == 1, (
13 + 'Pipeline parallelism is not supported yet.')
14 +
15 + (num_nodes, num_devices_per_node, distributed_init_method,
16 + all_stage_devices) = (
17 + initialize_ray_cluster(
18 + pipeline_parallel_size=args.pipeline_parallel_size,
19 + tensor_parallel_size=args.tensor_parallel_size))
20 +
21 + # Create a server.
22 + server = Server(
23 + model=args.model,
24 + model_path=args.model_path,
25 + pipeline_parallel_size=args.pipeline_parallel_size,
26 + tensor_parallel_size=args.tensor_parallel_size,
27 + block_size=args.block_size,
28 + dtype=args.dtype,
29 + seed=args.seed,
30 + swap_space=args.swap_space,
31 + max_batch_size=args.max_batch_size,
32 + num_nodes=num_nodes,
33 + num_devices_per_node=num_devices_per_node,
34 + distributed_init_method=distributed_init_method,
35 + all_stage_devices=all_stage_devices,
36 + gpu_memory=get_gpu_memory(),
37 + cpu_memory=get_cpu_memory(),
38 + )
39 +
40 + # Create a frontend.
41 + frontend = SimpleFrontend(
42 + model_name=args.model,
43 + block_size=args.block_size,
44 + )
45 +
46 + # Test the following inputs.
47 + test_inputs = [
48 + ('Ion Stoica is a', {'n': 4, 'use_beam_search': True, 'temperature': 0.0}),
49 + ('UC Berkeley is', {'n': 3, 'temperature': 0.8, 'top_p': 0.99}),
50 + ('The future of cloud computing is', {}), # Use default parameters.
51 + ]
52 + while True:
53 + if test_inputs:
54 + text, sampling_params_dict = test_inputs.pop(0)
55 + sampling_params = SamplingParams.from_dict(sampling_params_dict)
56 + sampling_params = frontend.add_eos_token(sampling_params)
57 + frontend.query(text, sampling_params)
58 + server.add_sequence_groups(frontend.get_inputs())
59 + updated_seq_groups = server.step()
60 + for seq_group in updated_seq_groups:
61 + if seq_group.is_finished():
62 + frontend.print_response(seq_group)
63 + if not (server.has_unfinished_requests() or test_inputs):
64 + break
65 +
66 +
67 +if __name__ == '__main__':
68 + parser = argparse.ArgumentParser(description='CacheFlow simple server.')
69 + parser = add_server_arguments(parser)
70 + args = parser.parse_args()
71 + main(args)