A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
OpenAI Compatible Frontend (#116)
Zhuohan Li committed
May 23, 2023 at 21:39 UTC
057daef778ec4e951841f44afda1cd0b1eb50ee4
20 files changed
+645
-170
cacheflow/core/block_manager.py
+3
-3
@@ -148,7 +148,7 @@ class BlockSpaceManager:
148
# the sequences in the same group.
149
blocks: Set[PhysicalTokenBlock] = set()
150
for seq in seq_group.get_seqs():
151
- if seq.status == SequenceStatus.FINISHED:
151
+ if SequenceStatus.is_finished(seq.status):
152
continue
153
block_table = self.block_tables[seq.seq_id]
154
for block in block_table:
@@ -169,7 +169,7 @@ class BlockSpaceManager:
169
# CPU block -> GPU block.
170
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
171
for seq in seq_group.get_seqs():
172
- if seq.status == SequenceStatus.FINISHED:
172
+ if SequenceStatus.is_finished(seq.status):
173
continue
174
new_block_table: BlockTable = []
175
block_table = self.block_tables[seq.seq_id]
@@ -200,7 +200,7 @@ class BlockSpaceManager:
200
# GPU block -> CPU block.
201
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
202
for seq in seq_group.get_seqs():
203
- if seq.status == SequenceStatus.FINISHED:
203
+ if SequenceStatus.is_finished(seq.status):
204
continue
205
new_block_table: BlockTable = []
206
block_table = self.block_tables[seq.seq_id]
cacheflow/core/scheduler.py
+4
-2
@@ -292,10 +292,12 @@ class Scheduler:
292
# Append a new token to the sequence.
293
output = seq_outputs[seq.seq_id]
294
seq.append_token_id(output.output_token, output.logprobs)
295
+ # Return a shallow copy of the running queue to prevent the queue
296
+ # from being modified by the caller.
297
return self.running.copy()
298
297
- def free_seq(self, seq: Sequence) -> None:
298
- seq.status = SequenceStatus.FINISHED
299
+ def free_seq(self, seq: Sequence, finish_status: SequenceStatus) -> None:
300
+ seq.status = finish_status
301
self.block_manager.free(seq)
302
303
def free_finished_seq_groups(self) -> None:
cacheflow/entrypoints/openai/openai_frontend.py
new
+300
@@ -0,0 +1,300 @@
1
+# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
2
+
3
+import argparse
4
+from http import HTTPStatus
5
+import json
6
+import time
7
+from typing import AsyncGenerator, Dict, List, Optional
8
+
9
+import fastapi
10
+from fastapi.exceptions import RequestValidationError
11
+from fastapi.middleware.cors import CORSMiddleware
12
+from fastapi.responses import StreamingResponse, JSONResponse
13
+import uvicorn
14
+
15
+from cacheflow.outputs import RequestOutput
16
+from cacheflow.server.arg_utils import ServerArgs
17
+from cacheflow.server.async_llm_server import AsyncLLMServer
18
+from cacheflow.server.tokenizer_utils import get_tokenizer
19
+from cacheflow.logger import init_logger
20
+from cacheflow.sampling_params import SamplingParams
21
+from cacheflow.utils import random_uuid
22
+from cacheflow.entrypoints.openai.protocol import (
23
+ CompletionRequest,
24
+ CompletionResponse,
25
+ CompletionResponseChoice,
26
+ CompletionResponseStreamChoice,
27
+ CompletionStreamResponse,
28
+ ErrorResponse,
29
+ LogProbs,
30
+ ModelCard,
31
+ ModelList,
32
+ ModelPermission,
33
+ UsageInfo,
34
+)
35
+
36
+
37
+logger = init_logger(__name__)
38
+served_model = None
39
+app = fastapi.FastAPI()
40
+
41
+
42
+def create_error_response(status_code: HTTPStatus,
43
+ message: str) -> JSONResponse:
44
+ return JSONResponse(
45
+ ErrorResponse(message=message, type="invalid_request_error").dict(),
46
+ status_code=status_code.value
47
+ )
48
+
49
+
50
+@app.exception_handler(RequestValidationError)
51
+async def validation_exception_handler(request, exc):
52
+ return create_error_response(HTTPStatus.BAD_REQUEST, str(exc))
53
+
54
+
55
+async def check_model(request) -> Optional[JSONResponse]:
56
+ if request.model == served_model:
57
+ return
58
+ ret = create_error_response(
59
+ HTTPStatus.NOT_FOUND,
60
+ f"The model `{request.model}` does not exist.",
61
+ )
62
+ return ret
63
+
64
+
65
+@app.get("/v1/models")
66
+async def show_available_models():
67
+ """Show available models. Right now we only have one model."""
68
+ model_cards = [ModelCard(id=served_model, root=served_model,
69
+ permission=[ModelPermission()])]
70
+ return ModelList(data=model_cards)
71
+
72
+
73
+def create_logprobs(token_ids: List[int],
74
+ id_logprobs: List[Dict[int, float]],
75
+ initial_text_offset: int = 0) -> LogProbs:
76
+ """Create OpenAI-style logprobs."""
77
+ logprobs = LogProbs()
78
+ last_token_len = 0
79
+ for token_id, id_logprob in zip(token_ids, id_logprobs):
80
+ token = tokenizer.convert_ids_to_tokens(token_id)
81
+ logprobs.tokens.append(token)
82
+ logprobs.token_logprobs.append(id_logprob[token_id])
83
+ if len(logprobs.text_offset) == 0:
84
+ logprobs.text_offset.append(initial_text_offset)
85
+ else:
86
+ logprobs.text_offset.append(logprobs.text_offset[-1] + last_token_len)
87
+ last_token_len = len(token)
88
+
89
+ logprobs.top_logprobs.append(
90
+ {tokenizer.convert_ids_to_tokens(i): p
91
+ for i, p in id_logprob.items()})
92
+ return logprobs
93
+
94
+
95
+@app.post("/v1/completions")
96
+async def create_completion(request: CompletionRequest):
97
+ logger.info(f"Received completion request: {request}")
98
+
99
+ error_check_ret = await check_model(request)
100
+ if error_check_ret is not None:
101
+ return error_check_ret
102
+
103
+ if request.echo:
104
+ # We do not support echo since the cacheflow server does not
105
+ # currently support getting the logprobs of prompt tokens.
106
+ return create_error_response(HTTPStatus.BAD_REQUEST,
107
+ "echo is not currently supported")
108
+
109
+ if request.suffix is not None:
110
+ # The language models we currently support do not support suffix.
111
+ return create_error_response(HTTPStatus.BAD_REQUEST,
112
+ "suffix is not currently supported")
113
+
114
+ if request.logit_bias is not None:
115
+ # TODO: support logit_bias in cacheflow server.
116
+ return create_error_response(HTTPStatus.BAD_REQUEST,
117
+ "logit_bias is not currently supported")
118
+
119
+ model_name = request.model
120
+ request_id = f"cmpl-{random_uuid()}"
121
+ prompt = request.prompt
122
+ created_time = int(time.time())
123
+ try:
124
+ sampling_params = SamplingParams(
125
+ n=request.n,
126
+ best_of=request.best_of,
127
+ presence_penalty=request.presence_penalty,
128
+ frequency_penalty=request.frequency_penalty,
129
+ temperature=request.temperature,
130
+ top_p=request.top_p,
131
+ top_k=request.top_k,
132
+ stop=request.stop,
133
+ ignore_eos=request.ignore_eos,
134
+ max_tokens=request.max_tokens,
135
+ logprobs=request.logprobs,
136
+ use_beam_search=request.use_beam_search,
137
+ )
138
+ except ValueError as e:
139
+ return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
140
+
141
+ result_generator = server.generate(prompt, sampling_params,
142
+ request_id=request_id)
143
+
144
+ # Similar to the OpenAI API, when n != best_of, we do not stream the
145
+ # results. In addition, we do not stream the results when use beam search.
146
+ stream = (request.stream and
147
+ (request.best_of is None or request.n == request.best_of) and
148
+ not request.use_beam_search)
149
+
150
+ def create_stream_response_json(index: int,
151
+ text: str,
152
+ logprobs: Optional[LogProbs] = None,
153
+ finish_reason: Optional[str] = None) -> str:
154
+ choice_data = CompletionResponseStreamChoice(
155
+ index=index,
156
+ text=text,
157
+ logprobs=logprobs,
158
+ finish_reason=finish_reason,
159
+ )
160
+ response = CompletionStreamResponse(
161
+ id=request_id,
162
+ created=created_time,
163
+ model=model_name,
164
+ choices=[choice_data],
165
+ )
166
+ response_json = response.json(ensure_ascii=False)
167
+
168
+ return response_json
169
+
170
+ async def completion_stream_generator() -> AsyncGenerator[str, None]:
171
+ previous_texts = [""] * request.n
172
+ previous_num_tokens = [0] * request.n
173
+ async for res in result_generator:
174
+ res: RequestOutput
175
+ for output in res.outputs:
176
+ i = output.index
177
+ delta_text = output.text[len(previous_texts[i]):]
178
+ if request.logprobs is not None:
179
+ logprobs = create_logprobs(
180
+ output.token_ids[previous_num_tokens[i]:],
181
+ output.logprobs[previous_num_tokens[i]:],
182
+ len(previous_texts[i]))
183
+ else:
184
+ logprobs = None
185
+ previous_texts[i] = output.text
186
+ previous_num_tokens[i] = len(output.token_ids)
187
+ response_json = create_stream_response_json(
188
+ index=i,
189
+ text=delta_text,
190
+ logprobs=logprobs,
191
+ )
192
+ yield f"data: {response_json}\n\n"
193
+ if output.finish_reason is not None:
194
+ logprobs = LogProbs() if request.logprobs is not None else None
195
+ response_json = create_stream_response_json(
196
+ index=i,
197
+ text="",
198
+ logprobs=logprobs,
199
+ finish_reason=output.finish_reason,
200
+ )
201
+ yield f"data: {response_json}\n\n"
202
+ yield "data: [DONE]\n\n"
203
+
204
+ # Streaming response
205
+ if stream:
206
+ return StreamingResponse(completion_stream_generator(),
207
+ media_type="text/event-stream")
208
+
209
+ # Non-streaming response
210
+ final_res: RequestOutput = None
211
+ async for res in result_generator:
212
+ final_res = res
213
+ assert final_res is not None
214
+ choices = []
215
+ for output in final_res.outputs:
216
+ if request.logprobs is not None:
217
+ logprobs = create_logprobs(output.token_ids, output.logprobs)
218
+ else:
219
+ logprobs = None
220
+ choice_data = CompletionResponseChoice(
221
+ index=output.index,
222
+ text=output.text,
223
+ logprobs=logprobs,
224
+ finish_reason=output.finish_reason,
225
+ )
226
+ choices.append(choice_data)
227
+
228
+ num_prompt_tokens = len(final_res.prompt_token_ids)
229
+ num_generated_tokens = sum(len(output.token_ids)
230
+ for output in final_res.outputs)
231
+ usage = UsageInfo(
232
+ prompt_tokens=num_prompt_tokens,
233
+ completion_tokens=num_generated_tokens,
234
+ total_tokens=num_prompt_tokens + num_generated_tokens,
235
+ )
236
+ response = CompletionResponse(
237
+ id=request_id,
238
+ created=created_time,
239
+ model=model_name,
240
+ choices=choices,
241
+ usage=usage,
242
+ )
243
+
244
+ if request.stream:
245
+ # When user requests streaming but we don't stream, we still need to
246
+ # return a streaming response with a single event.
247
+ response_json = response.json(ensure_ascii=False)
248
+ async def fake_stream_generator() -> AsyncGenerator[str, None]:
249
+ yield f"data: {response_json}\n\n"
250
+ yield "data: [DONE]\n\n"
251
+ return StreamingResponse(fake_stream_generator(),
252
+ media_type="text/event-stream")
253
+
254
+ return response
255
+
256
+
257
+if __name__ == "__main__":
258
+ parser = argparse.ArgumentParser(
259
+ description="CacheFlow OpenAI-Compatible RESTful API server."
260
+ )
261
+ parser.add_argument("--host", type=str, default="localhost", help="host name")
262
+ parser.add_argument("--port", type=int, default=8000, help="port number")
263
+ parser.add_argument(
264
+ "--allow-credentials", action="store_true", help="allow credentials"
265
+ )
266
+ parser.add_argument(
267
+ "--allowed-origins", type=json.loads, default=["*"], help="allowed origins"
268
+ )
269
+ parser.add_argument(
270
+ "--allowed-methods", type=json.loads, default=["*"], help="allowed methods"
271
+ )
272
+ parser.add_argument(
273
+ "--allowed-headers", type=json.loads, default=["*"], help="allowed headers"
274
+ )
275
+ parser.add_argument("--served-model-name", type=str, default=None,
276
+ help="The model name used in the API. If not specified, "
277
+ "the model name will be the same as the "
278
+ "huggingface name.")
279
+ parser = ServerArgs.add_cli_args(parser)
280
+ args = parser.parse_args()
281
+
282
+ app.add_middleware(
283
+ CORSMiddleware,
284
+ allow_origins=args.allowed_origins,
285
+ allow_credentials=args.allow_credentials,
286
+ allow_methods=args.allowed_methods,
287
+ allow_headers=args.allowed_headers,
288
+ )
289
+
290
+ logger.info(f"args: {args}")
291
+
292
+ served_model = args.served_model_name or args.model
293
+
294
+ server_args = ServerArgs.from_cli_args(args)
295
+ server = AsyncLLMServer.from_server_args(server_args)
296
+
297
+ # A separate tokenizer to map token IDs to strings.
298
+ tokenizer = get_tokenizer(args.model)
299
+
300
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/entrypoints/openai/protocol.py
new
+126
@@ -0,0 +1,126 @@
1
+# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
2
+import time
3
+from typing import Dict, List, Literal, Optional, Union
4
+
5
+from pydantic import BaseModel, Field
6
+
7
+from cacheflow.utils import random_uuid
8
+
9
+
10
+class ErrorResponse(BaseModel):
11
+ object: str = "error"
12
+ message: str
13
+ type: str
14
+ param: Optional[str] = None
15
+ code: Optional[str] = None
16
+
17
+
18
+class ModelPermission(BaseModel):
19
+ id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
20
+ object: str = "model_permission"
21
+ created: int = Field(default_factory=lambda: int(time.time()))
22
+ allow_create_engine: bool = False
23
+ allow_sampling: bool = True
24
+ allow_logprobs: bool = True
25
+ allow_search_indices: bool = False
26
+ allow_view: bool = True
27
+ allow_fine_tuning: bool = False
28
+ organization: str = "*"
29
+ group: Optional[str] = None
30
+ is_blocking: str = False
31
+
32
+
33
+class ModelCard(BaseModel):
34
+ id: str
35
+ object: str = "model"
36
+ created: int = Field(default_factory=lambda: int(time.time()))
37
+ owned_by: str = "cacheflow"
38
+ root: Optional[str] = None
39
+ parent: Optional[str] = None
40
+ permission: List[ModelPermission] = Field(default_factory=list)
41
+
42
+
43
+class ModelList(BaseModel):
44
+ object: str = "list"
45
+ data: List[ModelCard] = Field(default_factory=list)
46
+
47
+
48
+class UsageInfo(BaseModel):
49
+ prompt_tokens: int = 0
50
+ total_tokens: int = 0
51
+ completion_tokens: Optional[int] = 0
52
+
53
+
54
+class ChatCompletionRequest(BaseModel):
55
+ model: str
56
+ messages: List[Dict[str, str]]
57
+ temperature: Optional[float] = 0.7
58
+ top_p: Optional[float] = 1.0
59
+ n: Optional[int] = 1
60
+ max_tokens: Optional[int] = None
61
+ stop: Optional[Union[str, List[str]]] = None
62
+ stream: Optional[bool] = False
63
+ presence_penalty: Optional[float] = 0.0
64
+ frequency_penalty: Optional[float] = 0.0
65
+ user: Optional[str] = None
66
+
67
+
68
+class CompletionRequest(BaseModel):
69
+ model: str
70
+ prompt: str
71
+ suffix: Optional[str] = None
72
+ max_tokens: Optional[int] = 16
73
+ temperature: Optional[float] = 1.0
74
+ top_p: Optional[float] = 1.0
75
+ n: Optional[int] = 1
76
+ stream: Optional[bool] = False
77
+ logprobs: Optional[int] = None
78
+ echo: Optional[bool] = False
79
+ stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
80
+ presence_penalty: Optional[float] = 0.0
81
+ frequency_penalty: Optional[float] = 0.0
82
+ best_of: Optional[int] = None
83
+ logit_bias: Optional[Dict[str, float]] = None
84
+ user: Optional[str] = None
85
+ # Additional parameters supported by cacheflow
86
+ top_k: Optional[int] = -1
87
+ ignore_eos: Optional[bool] = False
88
+ use_beam_search: Optional[bool] = False
89
+
90
+
91
+class LogProbs(BaseModel):
92
+ text_offset: List[int] = Field(default_factory=list)
93
+ token_logprobs: List[Optional[float]] = Field(default_factory=list)
94
+ tokens: List[str] = Field(default_factory=list)
95
+ top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list)
96
+
97
+
98
+class CompletionResponseChoice(BaseModel):
99
+ index: int
100
+ text: str
101
+ logprobs: Optional[LogProbs] = None
102
+ finish_reason: Optional[Literal["stop", "length"]] = None
103
+
104
+
105
+class CompletionResponse(BaseModel):
106
+ id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
107
+ object: str = "text_completion"
108
+ created: int = Field(default_factory=lambda: int(time.time()))
109
+ model: str
110
+ choices: List[CompletionResponseChoice]
111
+ usage: UsageInfo
112
+
113
+
114
+class CompletionResponseStreamChoice(BaseModel):
115
+ index: int
116
+ text: str
117
+ logprobs: Optional[LogProbs] = None
118
+ finish_reason: Optional[Literal["stop", "length"]] = None
119
+
120
+
121
+class CompletionStreamResponse(BaseModel):
122
+ id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
123
+ object: str = "text_completion"
124
+ created: int = Field(default_factory=lambda: int(time.time()))
125
+ model: str
126
+ choices: List[CompletionResponseStreamChoice]
cacheflow/entrypoints/simple_fastapi_frontend.py
new
+51
@@ -0,0 +1,51 @@
1
+import argparse
2
+import json
3
+from typing import AsyncGenerator
4
+
5
+from fastapi import FastAPI, Request
6
+from fastapi.responses import StreamingResponse
7
+import uvicorn
8
+
9
+from cacheflow.sampling_params import SamplingParams
10
+from cacheflow.server.arg_utils import ServerArgs
11
+from cacheflow.server.async_llm_server import AsyncLLMServer
12
+from cacheflow.server.ray_utils import initialize_cluster
13
+
14
+TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
+app = FastAPI()
16
+
17
+
18
+@app.post("/generate")
19
+async def generate_stream(request: Request) -> StreamingResponse:
20
+ request_dict = await request.json()
21
+ prompt = request_dict.pop("prompt")
22
+ sampling_params = SamplingParams(**request_dict)
23
+ results_generator = server.generate(prompt, sampling_params)
24
+
25
+ async def stream_results() -> AsyncGenerator[bytes, None]:
26
+ async for request_output in results_generator:
27
+ prompt = request_output.prompt
28
+ text_outputs = [
29
+ prompt + output.text
30
+ for output in request_output.outputs
31
+ ]
32
+ ret = {
33
+ "text": text_outputs,
34
+ "error": 0,
35
+ }
36
+ yield (json.dumps(ret) + "\0").encode("utf-8")
37
+
38
+ return StreamingResponse(stream_results())
39
+
40
+
41
+if __name__ == "__main__":
42
+ parser = argparse.ArgumentParser()
43
+ parser.add_argument("--host", type=str, default="localhost")
44
+ parser.add_argument("--port", type=int, default=8001)
45
+ parser = ServerArgs.add_cli_args(parser)
46
+ args = parser.parse_args()
47
+
48
+ server_args = ServerArgs.from_cli_args(args)
49
+ server = AsyncLLMServer.from_server_args(server_args)
50
+
51
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/model_executor/layers/sampler.py
+3
-3
@@ -1,5 +1,5 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
-from typing import Dict, List, Tuple
2
+from typing import Dict, List, Tuple, Optional
3
4
import numpy as np
5
import torch
@@ -258,9 +258,9 @@ def _apply_top_p_top_k(
258
259
def _get_topk_logprobs(
260
logprobs: torch.Tensor,
261
- num_logprobs: int,
261
+ num_logprobs: Optional[int],
262
) -> Dict[int, float]:
263
- if num_logprobs == 0:
263
+ if num_logprobs is None or num_logprobs == 0:
264
return {}
265
266
topk_logprobs, topk_ids = torch.topk(logprobs, num_logprobs)
cacheflow/outputs.py
+18
-11
@@ -1,6 +1,6 @@
1
-from typing import Dict, List
1
+from typing import Dict, List, Optional
2
3
-from cacheflow.sequence import SequenceGroup
3
+from cacheflow.sequence import SequenceGroup, SequenceStatus
4
5
6
class CompletionOutput:
@@ -12,19 +12,25 @@ class CompletionOutput:
12
token_ids: List[int],
13
cumulative_logprob: float,
14
logprobs: List[Dict[int, float]],
15
+ finish_reason: Optional[str] = None,
16
) -> None:
17
self.index = index
18
self.text = text
19
self.token_ids = token_ids
20
self.cumulative_logprob = cumulative_logprob
21
self.logprobs = logprobs
22
+ self.finish_reason = finish_reason
23
+
24
+ def finished(self) -> bool:
25
+ return self.finish_reason is not None
26
27
def __repr__(self) -> str:
28
return (f"CompletionOutput(index={self.index}, "
29
f"text={self.text!r}, "
30
f"token_ids={self.token_ids}, "
31
f"cumulative_logprob={self.cumulative_logprob}, "
27
- f"logprobs={self.logprobs})")
32
+ f"logprobs={self.logprobs},"
33
+ f"finish_reason={self.finish_reason})")
34
35
36
class RequestOutput:
@@ -35,13 +41,11 @@ class RequestOutput:
41
prompt: str,
42
prompt_token_ids: List[int],
43
outputs: List[CompletionOutput],
38
- done: bool,
44
) -> None:
45
self.request_id = request_id
46
self.prompt = prompt
47
self.prompt_token_ids = prompt_token_ids
48
self.outputs = outputs
44
- self.done = done
49
50
@classmethod
51
def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -57,25 +61,28 @@ class RequestOutput:
61
outputs: List[CompletionOutput] = []
62
for seq in top_n_seqs:
63
logprobs = seq.output_logprobs
60
- if seq_group.sampling_params.logprobs == 0:
64
+ if seq_group.sampling_params.logprobs is None:
65
# NOTE: We need to take care of this case because the sequence
66
# always has the logprobs of the sampled tokens even if the
67
# logprobs are not requested.
68
logprobs = {}
69
+ finshed_reason = SequenceStatus.get_finished_reason(seq.status)
70
output = CompletionOutput(seqs.index(seq), seq.output_text,
71
seq.get_output_token_ids(),
67
- seq.get_cumulative_logprob(), logprobs)
72
+ seq.get_cumulative_logprob(), logprobs,
73
+ finshed_reason)
74
outputs.append(output)
75
76
# Every sequence in the sequence group should have the same prompt.
77
prompt = top_n_seqs[0].prompt
78
prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
73
- return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
74
- seq_group.is_finished())
79
+ return cls(seq_group.request_id, prompt, prompt_token_ids, outputs)
80
81
def __repr__(self) -> str:
82
return (f"RequestOutput(request_id={self.request_id}, "
83
f"prompt={self.prompt!r}, "
84
f"prompt_token_ids={self.prompt_token_ids}, "
80
- f"outputs={self.outputs}, "
81
- f"done={self.done})")
85
+ f"outputs={self.outputs})")
86
+
87
+ def finished(self) -> bool:
88
+ return all(output.finished() for output in self.outputs)
cacheflow/sampling_params.py
+2
-2
@@ -53,7 +53,7 @@ class SamplingParams:
53
stop: Union[str, List[str]] = [],
54
ignore_eos: bool = False,
55
max_tokens: int = 16,
56
- logprobs: int = 0,
56
+ logprobs: Optional[int] = None,
57
) -> None:
58
self.n = n
59
self.best_of = best_of if best_of is not None else n
@@ -98,7 +98,7 @@ class SamplingParams:
98
if self.max_tokens < 1:
99
raise ValueError(
100
f"max_tokens must be at least 1, got {self.max_tokens}.")
101
- if self.logprobs < 0:
101
+ if self.logprobs is not None and self.logprobs < 0:
102
raise ValueError(
103
f"logprobs must be non-negative, got {self.logprobs}.")
104
cacheflow/sequence.py
+21
-5
@@ -1,6 +1,6 @@
1
import copy
2
import enum
3
-from typing import Dict, List, Optional
3
+from typing import Dict, List, Optional, Union
4
5
from cacheflow.block import LogicalTokenBlock
6
from cacheflow.sampling_params import SamplingParams
@@ -10,8 +10,25 @@ class SequenceStatus(enum.Enum):
10
WAITING = enum.auto()
11
RUNNING = enum.auto()
12
SWAPPED = enum.auto()
13
- FINISHED = enum.auto()
14
-
13
+ FINISHED_STOPPED = enum.auto()
14
+ FINISHED_LENGTH_CAPPED = enum.auto()
15
+
16
+ @staticmethod
17
+ def is_finished(status: "SequenceStatus") -> bool:
18
+ return status in [
19
+ SequenceStatus.FINISHED_STOPPED,
20
+ SequenceStatus.FINISHED_LENGTH_CAPPED,
21
+ ]
22
+
23
+ @staticmethod
24
+ def get_finished_reason(status: "SequenceStatus") -> Union[str, None]:
25
+ if status == SequenceStatus.FINISHED_STOPPED:
26
+ finish_reason = "stop"
27
+ elif status == SequenceStatus.FINISHED_LENGTH_CAPPED:
28
+ finish_reason = "length"
29
+ else:
30
+ finish_reason = None
31
+ return finish_reason
32
33
class SequenceData:
34
@@ -20,7 +37,6 @@ class SequenceData:
37
prompt_token_ids: List[int],
38
) -> None:
39
self.prompt_token_ids = prompt_token_ids
23
-
40
self.output_token_ids: List[int] = []
41
self.cumulative_logprob = 0.0
42
@@ -166,7 +182,7 @@ class SequenceGroup:
182
raise ValueError(f'Sequence {seq_id} not found.')
183
184
def is_finished(self) -> bool:
169
- return all(seq.status == SequenceStatus.FINISHED for seq in self.seqs)
185
+ return all(SequenceStatus.is_finished(seq.status) for seq in self.seqs)
186
187
def __repr__(self) -> str:
188
return (f"SequenceGroup(request_id={self.request_id}, "
cacheflow/server/async_llm_server.py
renamed
+21
-46
@@ -1,26 +1,20 @@
1
-import argparse
1
import asyncio
3
-import json
2
import time
5
-from typing import Any, Dict
6
-import uuid
3
+from typing import Dict, Optional
4
8
-from fastapi import FastAPI, Request
9
-from fastapi.responses import StreamingResponse
5
import ray
11
-import uvicorn
6
7
from cacheflow.outputs import RequestOutput
8
from cacheflow.sampling_params import SamplingParams
9
from cacheflow.server.arg_utils import ServerArgs
10
from cacheflow.server.llm_server import LLMServer
11
from cacheflow.server.ray_utils import initialize_cluster
12
+from cacheflow.utils import random_uuid
13
14
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
20
-app = FastAPI()
15
16
23
-class FastAPIServer:
17
+class AsyncLLMServer:
18
19
def __init__(self, server_use_ray: bool, *args, **kwargs) -> None:
20
if server_use_ray:
@@ -45,15 +39,15 @@ class FastAPIServer:
39
self.request_outputs[request_id] = request_output
40
self.request_events[request_id].set()
41
48
- async def generate(self, request_dict: Dict[str, Any]):
42
+ async def generate(self, prompt: str, sampling_params: SamplingParams,
43
+ request_id: Optional[str] = None) -> RequestOutput:
44
# Preprocess the request.
45
arrival_time = time.time()
51
- prompt = request_dict.pop("prompt")
52
- sampling_params = SamplingParams(**request_dict)
46
47
# Create an event to notify us that there is new output from the
48
# cacheflow server.
56
- request_id = str(uuid.uuid4().hex[:8])
49
+ if request_id is None:
50
+ request_id = random_uuid()
51
request_event = asyncio.Event()
52
self.request_events[request_id] = request_event
53
@@ -82,19 +76,10 @@ class FastAPIServer:
76
77
# Decode and return new outputs.
78
request_output = self.request_outputs[request_id]
85
- prompt = request_output.prompt
86
- text_outputs = [
87
- prompt + output.text
88
- for output in request_output.outputs
89
- ]
90
- ret = {
91
- "text": text_outputs,
92
- "error": 0,
93
- }
94
- yield (json.dumps(ret) + "\0").encode("utf-8")
79
+ yield request_output
80
81
# Once finished, release the resources of the sequence group.
97
- if request_output.done:
82
+ if request_output.finished():
83
del self.request_outputs[request_id]
84
del self.request_events[request_id]
85
# Kick the server if the server is not running. This is to
@@ -104,25 +89,15 @@ class FastAPIServer:
89
await self.server_step()
90
break
91
107
-
108
-@app.post("/generate")
109
-async def generate_stream(request: Request):
110
- request_dict = await request.json()
111
- return StreamingResponse(server.generate(request_dict))
112
-
113
-
114
-if __name__ == "__main__":
115
- parser = argparse.ArgumentParser()
116
- parser.add_argument("--host", type=str, default="localhost")
117
- parser.add_argument("--port", type=int, default=10002)
118
- parser = ServerArgs.add_cli_args(parser)
119
- args = parser.parse_args()
120
-
121
- server_configs = ServerArgs.from_cli_args(args).create_server_configs()
122
- parallel_config = server_configs[2]
123
- distributed_init_method, stage_devices = initialize_cluster(parallel_config)
124
-
125
- server = FastAPIServer(args.use_ray, *server_configs,
126
- distributed_init_method, stage_devices,
127
- log_stats=not args.disable_log_stats)
128
- uvicorn.run(app, host=args.host, port=args.port, log_level="info")
92
+ @classmethod
93
+ def from_server_args(cls, server_args: ServerArgs) -> "AsyncLLMServer":
94
+ # Create the server configs.
95
+ server_configs = server_args.create_server_configs()
96
+ parallel_config = server_configs[2]
97
+ # Initialize the cluster.
98
+ distributed_init_method, devices = initialize_cluster(parallel_config)
99
+ # Create the LLM server.
100
+ server = cls(server_args.use_ray, *server_configs,
101
+ distributed_init_method, devices,
102
+ log_stats=not server_args.disable_log_stats)
103
+ return server
cacheflow/server/llm_server.py
+8
-5
@@ -210,7 +210,8 @@ class LLMServer:
210
# Truncate the output text so that the stop string is
211
# not included in the output.
212
seq.output_text = seq.output_text[:-len(stop_str)]
213
- self.scheduler.free_seq(seq)
213
+ self.scheduler.free_seq(seq,
214
+ SequenceStatus.FINISHED_STOPPED)
215
stopped = True
216
break
217
if stopped:
@@ -218,12 +219,14 @@ class LLMServer:
219
220
# Check if the sequence has reached max_tokens.
221
if seq.get_output_len() == sampling_params.max_tokens:
221
- self.scheduler.free_seq(seq)
222
+ self.scheduler.free_seq(
223
+ seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
224
continue
225
# Check if the sequence has generated the EOS token.
226
if not sampling_params.ignore_eos:
227
if seq.get_last_token_id() == self.tokenizer.eos_token_id:
226
- self.scheduler.free_seq(seq)
228
+ self.scheduler.free_seq(seq,
229
+ SequenceStatus.FINISHED_STOPPED)
230
continue
231
232
def _run_workers(
@@ -238,10 +241,10 @@ class LLMServer:
241
executor = getattr(worker, method)
242
if self.parallel_config.use_ray:
243
executor = executor.remote
241
-
244
+
245
output = executor(*args, **kwargs)
246
all_outputs.append(output)
244
-
247
+
248
if self.parallel_config.use_ray:
249
all_outputs = ray.get(all_outputs)
250
cacheflow/utils.py
+5
@@ -1,4 +1,5 @@
1
import enum
2
+import uuid
3
4
import psutil
5
import torch
@@ -31,3 +32,7 @@ def get_gpu_memory(gpu: int = 0) -> int:
32
def get_cpu_memory() -> int:
33
"""Returns the total CPU memory of the node in bytes."""
34
return psutil.virtual_memory().total
35
+
36
+
37
+def random_uuid() -> str:
38
+ return str(uuid.uuid4().hex)
examples/gradio_webserver.py
renamed
+7
-6
@@ -1,6 +1,5 @@
1
import argparse
2
import json
3
-import time
3
4
import gradio as gr
5
import requests
@@ -24,9 +23,9 @@ def http_bot(prompt):
23
def build_demo():
24
with gr.Blocks() as demo:
25
gr.Markdown(
27
- "# Cacheflow demo\n"
26
+ "# Cacheflow text completion demo\n"
27
)
29
- inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")# .style(container=False)
28
+ inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")
29
outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
30
inputbox.submit(http_bot, [inputbox], [outputbox])
31
return demo
@@ -35,9 +34,11 @@ def build_demo():
34
if __name__ == "__main__":
35
parser = argparse.ArgumentParser()
36
parser.add_argument("--host", type=str, default="localhost")
38
- parser.add_argument("--port", type=int, default=10003)
39
- parser.add_argument("--model-url", type=str, default="http://localhost:10002/generate")
37
+ parser.add_argument("--port", type=int, default=8002)
38
+ parser.add_argument("--model-url", type=str, default="http://localhost:8001/generate")
39
args = parser.parse_args()
40
41
demo = build_demo()
43
- demo.queue(concurrency_count=100).launch(server_name=args.host, server_port=args.port)
\ No newline at end of file
42
+ demo.queue(concurrency_count=100).launch(server_name=args.host,
43
+ server_port=args.port,
44
+ share=True)
examples/openai_client.py
new
+22
@@ -0,0 +1,22 @@
1
+import openai
2
+openai.api_key = "EMPTY"
3
+openai.api_base = "http://localhost:8000/v1"
4
+model = "facebook/opt-125m"
5
+
6
+# list models
7
+models = openai.Model.list()
8
+print(models)
9
+
10
+# create a completion
11
+
12
+stream = True
13
+completion = openai.Completion.create(
14
+ model=model, prompt="A robot may not injure a human being", echo=False, n=2,
15
+ best_of=3, stream=stream, logprobs=3)
16
+
17
+# print the completion
18
+if stream:
19
+ for c in completion:
20
+ print(c)
21
+else:
22
+ print("completion:", completion)
examples/simple_fastapi_client.py
new
+48
@@ -0,0 +1,48 @@
1
+import argparse
2
+import requests
3
+import json
4
+
5
+def clear_line(n=1):
6
+ LINE_UP = '\033[1A'
7
+ LINE_CLEAR = '\x1b[2K'
8
+ for i in range(n):
9
+ print(LINE_UP, end=LINE_CLEAR, flush=True)
10
+
11
+
12
+def http_request(prompt: str, api_url: str, n: int = 1):
13
+ headers = {"User-Agent": "Test Client"}
14
+ pload = {
15
+ "prompt": prompt,
16
+ "n": n,
17
+ "use_beam_search": True,
18
+ "temperature": 0.0,
19
+ "max_tokens": 16,
20
+ }
21
+ response = requests.post(api_url, headers=headers, json=pload, stream=True)
22
+
23
+ for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
24
+ if chunk:
25
+ data = json.loads(chunk.decode("utf-8"))
26
+ output = data["text"]
27
+ yield output
28
+
29
+
30
+if __name__ == "__main__":
31
+ parser = argparse.ArgumentParser()
32
+ parser.add_argument("--host", type=str, default="localhost")
33
+ parser.add_argument("--port", type=int, default=8001)
34
+ parser.add_argument("--n", type=int, default=4)
35
+ parser.add_argument("--prompt", type=str, default="San Francisco is a")
36
+ args = parser.parse_args()
37
+ prompt = args.prompt
38
+ api_url = f"http://{args.host}:{args.port}/generate"
39
+ n = args.n
40
+
41
+ print(f"Prompt: {prompt}\n", flush=True)
42
+ num_printed_lines = 0
43
+ for h in http_request(prompt, api_url, n):
44
+ clear_line(num_printed_lines)
45
+ num_printed_lines = 0
46
+ for i, line in enumerate(h):
47
+ num_printed_lines += 1
48
+ print(f"Beam candidate {i}: {line}", flush=True)
examples/simple_server.py
+5
-4
@@ -1,5 +1,4 @@
1
import argparse
2
-import uuid
2
3
from cacheflow import ServerArgs, LLMServer, SamplingParams
4
@@ -20,17 +19,19 @@ def main(args: argparse.Namespace):
19
SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20
]
21
22
+ request_id = 0
23
+
24
# Run the server.
25
while True:
26
# To test iteration-level scheduling, we add one request at each step.
27
if test_prompts:
28
prompt, sampling_params = test_prompts.pop(0)
28
- request_id = str(uuid.uuid4().hex[:8])
29
- server.add_request(request_id, prompt, sampling_params)
29
+ server.add_request(str(request_id), prompt, sampling_params)
30
+ request_id += 1
31
32
request_outputs = server.step()
33
for request_output in request_outputs:
33
- if request_output.done:
34
+ if request_output.finished():
35
print(request_output)
36
37
if not (server.has_unfinished_requests() or test_prompts):
playground/http_client.py
deleted
-20
@@ -1,20 +0,0 @@
1
-import requests
2
-import json
3
-
4
-def http_bot():
5
- prompt = "How are you? I'm fine."
6
-
7
- headers = {"User-Agent": "Test Client"}
8
- pload = {
9
- "prompt": prompt,
10
- }
11
- response = requests.post("http://localhost:10002", headers=headers, json=pload, stream=True)
12
-
13
- for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
14
- if chunk:
15
- data = json.loads(chunk.decode("utf-8"))
16
- output = data["text"]
17
- yield output
18
-
19
-for h in http_bot():
20
- print(h, end="", flush=True)
\ No newline at end of file
playground/streaming_fastapi_worker.py
deleted
-40
@@ -1,40 +0,0 @@
1
-import argparse
2
-import asyncio
3
-import time
4
-from typing import Union
5
-import json
6
-
7
-from fastapi import FastAPI, Request
8
-from fastapi.responses import StreamingResponse
9
-import uvicorn
10
-
11
-
12
-app = FastAPI()
13
-
14
-
15
-async def text_streamer(args):
16
- context = args["prompt"]
17
- words = context.split(" ")
18
- for word in words:
19
- await asyncio.sleep(1)
20
- print("word:", word)
21
- ret = {
22
- "text": word + " ",
23
- "error": 0,
24
- }
25
- yield (json.dumps(ret) + "\0").encode("utf-8")
26
-
27
-
28
-@app.post("/")
29
-async def read_root(request: Request):
30
- args = await request.json()
31
- return StreamingResponse(text_streamer(args))
32
-
33
-
34
-if __name__ == "__main__":
35
- parser = argparse.ArgumentParser()
36
- parser.add_argument("--host", type=str, default="localhost")
37
- parser.add_argument("--port", type=int, default=10002)
38
- args = parser.parse_args()
39
-
40
- uvicorn.run(app, host=args.host, port=args.port, log_level="info")
requirements.txt
+1
@@ -8,3 +8,4 @@ transformers >= 4.28.0 # Required for LLaMA.
8
xformers >= 0.0.19
9
fastapi
10
uvicorn
11
+pydantic # Required for OpenAI server.
test_cli_client.py
deleted
-23
@@ -1,23 +0,0 @@
1
-import requests
2
-import json
3
-
4
-def http_request():
5
- prompt = "Ion Stoica is a"
6
-
7
- headers = {"User-Agent": "Test Client"}
8
- pload = {
9
- "prompt": prompt,
10
- "n": 4,
11
- "use_beam_search": True,
12
- "temperature": 0.0,
13
- }
14
- response = requests.post("http://localhost:10002/generate", headers=headers, json=pload, stream=True)
15
-
16
- for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
17
- if chunk:
18
- data = json.loads(chunk.decode("utf-8"))
19
- output = data["text"]
20
- yield output
21
-
22
-for h in http_request():
23
- print(h, flush=True)