A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix potential bugs in FastAPI frontend and add comments (#28)
Zhuohan Li committed
Apr 6, 2023 at 13:44 UTC
a490aafa3671da1b6b2be6cff4568913fcb1732c
1 file changed
+30
-5
cacheflow/http_frontend/fastapi_frontend.py
+30
-5
@@ -17,8 +17,10 @@ from cacheflow.master.server import (Server, add_server_arguments,
17
from cacheflow.worker.controller import DeviceID
18
from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
19
20
+TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
21
app = FastAPI()
22
23
+
24
class FastAPIFrontend:
25
def __init__(
26
self,
@@ -30,7 +32,7 @@ class FastAPIFrontend:
32
dtype: str,
33
seed: int,
34
swap_space: int,
33
- max_batch_size: int,
35
+ max_num_batched_tokens: int,
36
num_nodes: int,
37
num_devices_per_node: int,
38
distributed_init_method: str,
@@ -51,7 +53,7 @@ class FastAPIFrontend:
53
dtype=dtype,
54
seed=seed,
55
swap_space=swap_space,
54
- max_batch_size=max_batch_size,
56
+ max_num_batched_tokens=max_num_batched_tokens,
57
num_nodes=num_nodes,
58
num_devices_per_node=num_devices_per_node,
59
distributed_init_method=distributed_init_method,
@@ -68,12 +70,14 @@ class FastAPIFrontend:
70
self.is_server_running = True
71
updated_seq_groups = await self.server.step.remote()
72
self.is_server_running = False
73
+ # Notify the waiting coroutines that there new outputs ready.
74
for seq_group in updated_seq_groups:
75
group_id = seq_group.group_id
76
self.running_seq_groups[group_id] = seq_group
77
self.sequence_group_events[group_id].set()
78
79
async def generate(self, request_dict: Dict):
80
+ # Preprocess the request.
81
prompt = request_dict["prompt"]
82
sampling_params = SamplingParams.from_dict(request_dict)
83
sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
@@ -87,15 +91,27 @@ class FastAPIFrontend:
91
arrival_time = time.time()
92
group_id = next(self.seq_group_counter)
93
seq_group = SequenceGroup(group_id, seqs, arrival_time)
94
+ # Create an event to notify us that there is new output from the
95
+ # cacheflow server.
96
group_event = asyncio.Event()
97
+ self.running_seq_groups[group_id] = seq_group
98
self.sequence_group_events[group_id] = group_event
99
+ # Add the request into the cacheflow server's waiting queue.
100
await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
101
+ # The cacheflow server does not have a background loop that keeps
102
+ # processing incoming requests. Therefore, we need to keep kicking
103
+ # the server to process the requests.
104
while True:
105
+ # Kick the server if the server is not running.
106
if not self.is_server_running:
107
await self.server_step()
96
- # Wait for new output. Add a 1s timeout to prevent dead lock.
97
- await asyncio.wait_for(group_event.wait(), timeout=1)
108
+ # Wait for new output. The group_event will be set in server_step
109
+ # when there is new output available for the sequence group.
110
+ # Added a timeout to prevent deadlock.
111
+ await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
112
+ # Reset the event to wait for the next output.
113
group_event.clear()
114
+ # Decode and return new outputs
115
seq_group = self.running_seq_groups[group_id]
116
all_outputs = []
117
for seq in seq_group.seqs:
@@ -107,7 +123,16 @@ class FastAPIFrontend:
123
"error": 0,
124
}
125
yield (json.dumps(ret) + "\0").encode("utf-8")
126
+
127
+ # Once finished, release the resources of the sequence group.
128
if seq_group.is_finished():
129
+ del self.running_seq_groups[group_id]
130
+ del self.sequence_group_events[group_id]
131
+ # Kick the server if the server is not running. This is to
132
+ # prevent that there are still requests in server's waiting
133
+ # queue to be executed.
134
+ if not self.is_server_running:
135
+ await self.server_step()
136
break
137
138
@@ -143,7 +168,7 @@ if __name__ == "__main__":
168
dtype=args.dtype,
169
seed=args.seed,
170
swap_space=args.swap_space,
146
- max_batch_size=args.max_batch_size,
171
+ max_num_batched_tokens=args.max_num_batched_tokens,
172
num_nodes=num_nodes,
173
num_devices_per_node=num_devices_per_node,
174
distributed_init_method=distributed_init_method,