@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a490aafa36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix potential bugs in FastAPI frontend and add comments (#28)

Zhuohan Li committed Apr 6, 2023 at 13:44 UTC a490aafa3671da1b6b2be6cff4568913fcb1732c
1 file changed +30 -5
cacheflow/http_frontend/fastapi_frontend.py
+30 -5
@@ -17,8 +17,10 @@ from cacheflow.master.server import (Server, add_server_arguments,
17 from cacheflow.worker.controller import DeviceID
18 from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
19
20 +TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
21 app = FastAPI()
22
23 +
24 class FastAPIFrontend:
25 def __init__(
26 self,
@@ -30,7 +32,7 @@ class FastAPIFrontend:
32 dtype: str,
33 seed: int,
34 swap_space: int,
33 - max_batch_size: int,
35 + max_num_batched_tokens: int,
36 num_nodes: int,
37 num_devices_per_node: int,
38 distributed_init_method: str,
@@ -51,7 +53,7 @@ class FastAPIFrontend:
53 dtype=dtype,
54 seed=seed,
55 swap_space=swap_space,
54 - max_batch_size=max_batch_size,
56 + max_num_batched_tokens=max_num_batched_tokens,
57 num_nodes=num_nodes,
58 num_devices_per_node=num_devices_per_node,
59 distributed_init_method=distributed_init_method,
@@ -68,12 +70,14 @@ class FastAPIFrontend:
70 self.is_server_running = True
71 updated_seq_groups = await self.server.step.remote()
72 self.is_server_running = False
73 + # Notify the waiting coroutines that there new outputs ready.
74 for seq_group in updated_seq_groups:
75 group_id = seq_group.group_id
76 self.running_seq_groups[group_id] = seq_group
77 self.sequence_group_events[group_id].set()
78
79 async def generate(self, request_dict: Dict):
80 + # Preprocess the request.
81 prompt = request_dict["prompt"]
82 sampling_params = SamplingParams.from_dict(request_dict)
83 sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
@@ -87,15 +91,27 @@ class FastAPIFrontend:
91 arrival_time = time.time()
92 group_id = next(self.seq_group_counter)
93 seq_group = SequenceGroup(group_id, seqs, arrival_time)
94 + # Create an event to notify us that there is new output from the
95 + # cacheflow server.
96 group_event = asyncio.Event()
97 + self.running_seq_groups[group_id] = seq_group
98 self.sequence_group_events[group_id] = group_event
99 + # Add the request into the cacheflow server's waiting queue.
100 await self.server.add_sequence_groups.remote([(seq_group, sampling_params)])
101 + # The cacheflow server does not have a background loop that keeps
102 + # processing incoming requests. Therefore, we need to keep kicking
103 + # the server to process the requests.
104 while True:
105 + # Kick the server if the server is not running.
106 if not self.is_server_running:
107 await self.server_step()
96 - # Wait for new output. Add a 1s timeout to prevent dead lock.
97 - await asyncio.wait_for(group_event.wait(), timeout=1)
108 + # Wait for new output. The group_event will be set in server_step
109 + # when there is new output available for the sequence group.
110 + # Added a timeout to prevent deadlock.
111 + await asyncio.wait_for(group_event.wait(), timeout=TIMEOUT_TO_PREVENT_DEADLOCK)
112 + # Reset the event to wait for the next output.
113 group_event.clear()
114 + # Decode and return new outputs
115 seq_group = self.running_seq_groups[group_id]
116 all_outputs = []
117 for seq in seq_group.seqs:
@@ -107,7 +123,16 @@ class FastAPIFrontend:
123 "error": 0,
124 }
125 yield (json.dumps(ret) + "\0").encode("utf-8")
126 +
127 + # Once finished, release the resources of the sequence group.
128 if seq_group.is_finished():
129 + del self.running_seq_groups[group_id]
130 + del self.sequence_group_events[group_id]
131 + # Kick the server if the server is not running. This is to
132 + # prevent that there are still requests in server's waiting
133 + # queue to be executed.
134 + if not self.is_server_running:
135 + await self.server_step()
136 break
137
138
@@ -143,7 +168,7 @@ if __name__ == "__main__":
168 dtype=args.dtype,
169 seed=args.seed,
170 swap_space=args.swap_space,
146 - max_batch_size=args.max_batch_size,
171 + max_num_batched_tokens=args.max_num_batched_tokens,
172 num_nodes=num_nodes,
173 num_devices_per_node=num_devices_per_node,
174 distributed_init_method=distributed_init_method,