@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 057daef778
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

OpenAI Compatible Frontend (#116)

Zhuohan Li committed May 23, 2023 at 21:39 UTC 057daef778ec4e951841f44afda1cd0b1eb50ee4
20 files changed +645 -170
cacheflow/core/block_manager.py
+3 -3
@@ -148,7 +148,7 @@ class BlockSpaceManager:
148 # the sequences in the same group.
149 blocks: Set[PhysicalTokenBlock] = set()
150 for seq in seq_group.get_seqs():
151 - if seq.status == SequenceStatus.FINISHED:
151 + if SequenceStatus.is_finished(seq.status):
152 continue
153 block_table = self.block_tables[seq.seq_id]
154 for block in block_table:
@@ -169,7 +169,7 @@ class BlockSpaceManager:
169 # CPU block -> GPU block.
170 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
171 for seq in seq_group.get_seqs():
172 - if seq.status == SequenceStatus.FINISHED:
172 + if SequenceStatus.is_finished(seq.status):
173 continue
174 new_block_table: BlockTable = []
175 block_table = self.block_tables[seq.seq_id]
@@ -200,7 +200,7 @@ class BlockSpaceManager:
200 # GPU block -> CPU block.
201 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
202 for seq in seq_group.get_seqs():
203 - if seq.status == SequenceStatus.FINISHED:
203 + if SequenceStatus.is_finished(seq.status):
204 continue
205 new_block_table: BlockTable = []
206 block_table = self.block_tables[seq.seq_id]
cacheflow/core/scheduler.py
+4 -2
@@ -292,10 +292,12 @@ class Scheduler:
292 # Append a new token to the sequence.
293 output = seq_outputs[seq.seq_id]
294 seq.append_token_id(output.output_token, output.logprobs)
295 + # Return a shallow copy of the running queue to prevent the queue
296 + # from being modified by the caller.
297 return self.running.copy()
298
297 - def free_seq(self, seq: Sequence) -> None:
298 - seq.status = SequenceStatus.FINISHED
299 + def free_seq(self, seq: Sequence, finish_status: SequenceStatus) -> None:
300 + seq.status = finish_status
301 self.block_manager.free(seq)
302
303 def free_finished_seq_groups(self) -> None:
cacheflow/entrypoints/openai/openai_frontend.py new
+300
@@ -0,0 +1,300 @@
1 +# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/serve/openai_api_server.py
2 +
3 +import argparse
4 +from http import HTTPStatus
5 +import json
6 +import time
7 +from typing import AsyncGenerator, Dict, List, Optional
8 +
9 +import fastapi
10 +from fastapi.exceptions import RequestValidationError
11 +from fastapi.middleware.cors import CORSMiddleware
12 +from fastapi.responses import StreamingResponse, JSONResponse
13 +import uvicorn
14 +
15 +from cacheflow.outputs import RequestOutput
16 +from cacheflow.server.arg_utils import ServerArgs
17 +from cacheflow.server.async_llm_server import AsyncLLMServer
18 +from cacheflow.server.tokenizer_utils import get_tokenizer
19 +from cacheflow.logger import init_logger
20 +from cacheflow.sampling_params import SamplingParams
21 +from cacheflow.utils import random_uuid
22 +from cacheflow.entrypoints.openai.protocol import (
23 + CompletionRequest,
24 + CompletionResponse,
25 + CompletionResponseChoice,
26 + CompletionResponseStreamChoice,
27 + CompletionStreamResponse,
28 + ErrorResponse,
29 + LogProbs,
30 + ModelCard,
31 + ModelList,
32 + ModelPermission,
33 + UsageInfo,
34 +)
35 +
36 +
37 +logger = init_logger(__name__)
38 +served_model = None
39 +app = fastapi.FastAPI()
40 +
41 +
42 +def create_error_response(status_code: HTTPStatus,
43 + message: str) -> JSONResponse:
44 + return JSONResponse(
45 + ErrorResponse(message=message, type="invalid_request_error").dict(),
46 + status_code=status_code.value
47 + )
48 +
49 +
50 +@app.exception_handler(RequestValidationError)
51 +async def validation_exception_handler(request, exc):
52 + return create_error_response(HTTPStatus.BAD_REQUEST, str(exc))
53 +
54 +
55 +async def check_model(request) -> Optional[JSONResponse]:
56 + if request.model == served_model:
57 + return
58 + ret = create_error_response(
59 + HTTPStatus.NOT_FOUND,
60 + f"The model `{request.model}` does not exist.",
61 + )
62 + return ret
63 +
64 +
65 +@app.get("/v1/models")
66 +async def show_available_models():
67 + """Show available models. Right now we only have one model."""
68 + model_cards = [ModelCard(id=served_model, root=served_model,
69 + permission=[ModelPermission()])]
70 + return ModelList(data=model_cards)
71 +
72 +
73 +def create_logprobs(token_ids: List[int],
74 + id_logprobs: List[Dict[int, float]],
75 + initial_text_offset: int = 0) -> LogProbs:
76 + """Create OpenAI-style logprobs."""
77 + logprobs = LogProbs()
78 + last_token_len = 0
79 + for token_id, id_logprob in zip(token_ids, id_logprobs):
80 + token = tokenizer.convert_ids_to_tokens(token_id)
81 + logprobs.tokens.append(token)
82 + logprobs.token_logprobs.append(id_logprob[token_id])
83 + if len(logprobs.text_offset) == 0:
84 + logprobs.text_offset.append(initial_text_offset)
85 + else:
86 + logprobs.text_offset.append(logprobs.text_offset[-1] + last_token_len)
87 + last_token_len = len(token)
88 +
89 + logprobs.top_logprobs.append(
90 + {tokenizer.convert_ids_to_tokens(i): p
91 + for i, p in id_logprob.items()})
92 + return logprobs
93 +
94 +
95 +@app.post("/v1/completions")
96 +async def create_completion(request: CompletionRequest):
97 + logger.info(f"Received completion request: {request}")
98 +
99 + error_check_ret = await check_model(request)
100 + if error_check_ret is not None:
101 + return error_check_ret
102 +
103 + if request.echo:
104 + # We do not support echo since the cacheflow server does not
105 + # currently support getting the logprobs of prompt tokens.
106 + return create_error_response(HTTPStatus.BAD_REQUEST,
107 + "echo is not currently supported")
108 +
109 + if request.suffix is not None:
110 + # The language models we currently support do not support suffix.
111 + return create_error_response(HTTPStatus.BAD_REQUEST,
112 + "suffix is not currently supported")
113 +
114 + if request.logit_bias is not None:
115 + # TODO: support logit_bias in cacheflow server.
116 + return create_error_response(HTTPStatus.BAD_REQUEST,
117 + "logit_bias is not currently supported")
118 +
119 + model_name = request.model
120 + request_id = f"cmpl-{random_uuid()}"
121 + prompt = request.prompt
122 + created_time = int(time.time())
123 + try:
124 + sampling_params = SamplingParams(
125 + n=request.n,
126 + best_of=request.best_of,
127 + presence_penalty=request.presence_penalty,
128 + frequency_penalty=request.frequency_penalty,
129 + temperature=request.temperature,
130 + top_p=request.top_p,
131 + top_k=request.top_k,
132 + stop=request.stop,
133 + ignore_eos=request.ignore_eos,
134 + max_tokens=request.max_tokens,
135 + logprobs=request.logprobs,
136 + use_beam_search=request.use_beam_search,
137 + )
138 + except ValueError as e:
139 + return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
140 +
141 + result_generator = server.generate(prompt, sampling_params,
142 + request_id=request_id)
143 +
144 + # Similar to the OpenAI API, when n != best_of, we do not stream the
145 + # results. In addition, we do not stream the results when use beam search.
146 + stream = (request.stream and
147 + (request.best_of is None or request.n == request.best_of) and
148 + not request.use_beam_search)
149 +
150 + def create_stream_response_json(index: int,
151 + text: str,
152 + logprobs: Optional[LogProbs] = None,
153 + finish_reason: Optional[str] = None) -> str:
154 + choice_data = CompletionResponseStreamChoice(
155 + index=index,
156 + text=text,
157 + logprobs=logprobs,
158 + finish_reason=finish_reason,
159 + )
160 + response = CompletionStreamResponse(
161 + id=request_id,
162 + created=created_time,
163 + model=model_name,
164 + choices=[choice_data],
165 + )
166 + response_json = response.json(ensure_ascii=False)
167 +
168 + return response_json
169 +
170 + async def completion_stream_generator() -> AsyncGenerator[str, None]:
171 + previous_texts = [""] * request.n
172 + previous_num_tokens = [0] * request.n
173 + async for res in result_generator:
174 + res: RequestOutput
175 + for output in res.outputs:
176 + i = output.index
177 + delta_text = output.text[len(previous_texts[i]):]
178 + if request.logprobs is not None:
179 + logprobs = create_logprobs(
180 + output.token_ids[previous_num_tokens[i]:],
181 + output.logprobs[previous_num_tokens[i]:],
182 + len(previous_texts[i]))
183 + else:
184 + logprobs = None
185 + previous_texts[i] = output.text
186 + previous_num_tokens[i] = len(output.token_ids)
187 + response_json = create_stream_response_json(
188 + index=i,
189 + text=delta_text,
190 + logprobs=logprobs,
191 + )
192 + yield f"data: {response_json}\n\n"
193 + if output.finish_reason is not None:
194 + logprobs = LogProbs() if request.logprobs is not None else None
195 + response_json = create_stream_response_json(
196 + index=i,
197 + text="",
198 + logprobs=logprobs,
199 + finish_reason=output.finish_reason,
200 + )
201 + yield f"data: {response_json}\n\n"
202 + yield "data: [DONE]\n\n"
203 +
204 + # Streaming response
205 + if stream:
206 + return StreamingResponse(completion_stream_generator(),
207 + media_type="text/event-stream")
208 +
209 + # Non-streaming response
210 + final_res: RequestOutput = None
211 + async for res in result_generator:
212 + final_res = res
213 + assert final_res is not None
214 + choices = []
215 + for output in final_res.outputs:
216 + if request.logprobs is not None:
217 + logprobs = create_logprobs(output.token_ids, output.logprobs)
218 + else:
219 + logprobs = None
220 + choice_data = CompletionResponseChoice(
221 + index=output.index,
222 + text=output.text,
223 + logprobs=logprobs,
224 + finish_reason=output.finish_reason,
225 + )
226 + choices.append(choice_data)
227 +
228 + num_prompt_tokens = len(final_res.prompt_token_ids)
229 + num_generated_tokens = sum(len(output.token_ids)
230 + for output in final_res.outputs)
231 + usage = UsageInfo(
232 + prompt_tokens=num_prompt_tokens,
233 + completion_tokens=num_generated_tokens,
234 + total_tokens=num_prompt_tokens + num_generated_tokens,
235 + )
236 + response = CompletionResponse(
237 + id=request_id,
238 + created=created_time,
239 + model=model_name,
240 + choices=choices,
241 + usage=usage,
242 + )
243 +
244 + if request.stream:
245 + # When user requests streaming but we don't stream, we still need to
246 + # return a streaming response with a single event.
247 + response_json = response.json(ensure_ascii=False)
248 + async def fake_stream_generator() -> AsyncGenerator[str, None]:
249 + yield f"data: {response_json}\n\n"
250 + yield "data: [DONE]\n\n"
251 + return StreamingResponse(fake_stream_generator(),
252 + media_type="text/event-stream")
253 +
254 + return response
255 +
256 +
257 +if __name__ == "__main__":
258 + parser = argparse.ArgumentParser(
259 + description="CacheFlow OpenAI-Compatible RESTful API server."
260 + )
261 + parser.add_argument("--host", type=str, default="localhost", help="host name")
262 + parser.add_argument("--port", type=int, default=8000, help="port number")
263 + parser.add_argument(
264 + "--allow-credentials", action="store_true", help="allow credentials"
265 + )
266 + parser.add_argument(
267 + "--allowed-origins", type=json.loads, default=["*"], help="allowed origins"
268 + )
269 + parser.add_argument(
270 + "--allowed-methods", type=json.loads, default=["*"], help="allowed methods"
271 + )
272 + parser.add_argument(
273 + "--allowed-headers", type=json.loads, default=["*"], help="allowed headers"
274 + )
275 + parser.add_argument("--served-model-name", type=str, default=None,
276 + help="The model name used in the API. If not specified, "
277 + "the model name will be the same as the "
278 + "huggingface name.")
279 + parser = ServerArgs.add_cli_args(parser)
280 + args = parser.parse_args()
281 +
282 + app.add_middleware(
283 + CORSMiddleware,
284 + allow_origins=args.allowed_origins,
285 + allow_credentials=args.allow_credentials,
286 + allow_methods=args.allowed_methods,
287 + allow_headers=args.allowed_headers,
288 + )
289 +
290 + logger.info(f"args: {args}")
291 +
292 + served_model = args.served_model_name or args.model
293 +
294 + server_args = ServerArgs.from_cli_args(args)
295 + server = AsyncLLMServer.from_server_args(server_args)
296 +
297 + # A separate tokenizer to map token IDs to strings.
298 + tokenizer = get_tokenizer(args.model)
299 +
300 + uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/entrypoints/openai/protocol.py new
+126
@@ -0,0 +1,126 @@
1 +# Adapted from https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
2 +import time
3 +from typing import Dict, List, Literal, Optional, Union
4 +
5 +from pydantic import BaseModel, Field
6 +
7 +from cacheflow.utils import random_uuid
8 +
9 +
10 +class ErrorResponse(BaseModel):
11 + object: str = "error"
12 + message: str
13 + type: str
14 + param: Optional[str] = None
15 + code: Optional[str] = None
16 +
17 +
18 +class ModelPermission(BaseModel):
19 + id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
20 + object: str = "model_permission"
21 + created: int = Field(default_factory=lambda: int(time.time()))
22 + allow_create_engine: bool = False
23 + allow_sampling: bool = True
24 + allow_logprobs: bool = True
25 + allow_search_indices: bool = False
26 + allow_view: bool = True
27 + allow_fine_tuning: bool = False
28 + organization: str = "*"
29 + group: Optional[str] = None
30 + is_blocking: str = False
31 +
32 +
33 +class ModelCard(BaseModel):
34 + id: str
35 + object: str = "model"
36 + created: int = Field(default_factory=lambda: int(time.time()))
37 + owned_by: str = "cacheflow"
38 + root: Optional[str] = None
39 + parent: Optional[str] = None
40 + permission: List[ModelPermission] = Field(default_factory=list)
41 +
42 +
43 +class ModelList(BaseModel):
44 + object: str = "list"
45 + data: List[ModelCard] = Field(default_factory=list)
46 +
47 +
48 +class UsageInfo(BaseModel):
49 + prompt_tokens: int = 0
50 + total_tokens: int = 0
51 + completion_tokens: Optional[int] = 0
52 +
53 +
54 +class ChatCompletionRequest(BaseModel):
55 + model: str
56 + messages: List[Dict[str, str]]
57 + temperature: Optional[float] = 0.7
58 + top_p: Optional[float] = 1.0
59 + n: Optional[int] = 1
60 + max_tokens: Optional[int] = None
61 + stop: Optional[Union[str, List[str]]] = None
62 + stream: Optional[bool] = False
63 + presence_penalty: Optional[float] = 0.0
64 + frequency_penalty: Optional[float] = 0.0
65 + user: Optional[str] = None
66 +
67 +
68 +class CompletionRequest(BaseModel):
69 + model: str
70 + prompt: str
71 + suffix: Optional[str] = None
72 + max_tokens: Optional[int] = 16
73 + temperature: Optional[float] = 1.0
74 + top_p: Optional[float] = 1.0
75 + n: Optional[int] = 1
76 + stream: Optional[bool] = False
77 + logprobs: Optional[int] = None
78 + echo: Optional[bool] = False
79 + stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
80 + presence_penalty: Optional[float] = 0.0
81 + frequency_penalty: Optional[float] = 0.0
82 + best_of: Optional[int] = None
83 + logit_bias: Optional[Dict[str, float]] = None
84 + user: Optional[str] = None
85 + # Additional parameters supported by cacheflow
86 + top_k: Optional[int] = -1
87 + ignore_eos: Optional[bool] = False
88 + use_beam_search: Optional[bool] = False
89 +
90 +
91 +class LogProbs(BaseModel):
92 + text_offset: List[int] = Field(default_factory=list)
93 + token_logprobs: List[Optional[float]] = Field(default_factory=list)
94 + tokens: List[str] = Field(default_factory=list)
95 + top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list)
96 +
97 +
98 +class CompletionResponseChoice(BaseModel):
99 + index: int
100 + text: str
101 + logprobs: Optional[LogProbs] = None
102 + finish_reason: Optional[Literal["stop", "length"]] = None
103 +
104 +
105 +class CompletionResponse(BaseModel):
106 + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
107 + object: str = "text_completion"
108 + created: int = Field(default_factory=lambda: int(time.time()))
109 + model: str
110 + choices: List[CompletionResponseChoice]
111 + usage: UsageInfo
112 +
113 +
114 +class CompletionResponseStreamChoice(BaseModel):
115 + index: int
116 + text: str
117 + logprobs: Optional[LogProbs] = None
118 + finish_reason: Optional[Literal["stop", "length"]] = None
119 +
120 +
121 +class CompletionStreamResponse(BaseModel):
122 + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
123 + object: str = "text_completion"
124 + created: int = Field(default_factory=lambda: int(time.time()))
125 + model: str
126 + choices: List[CompletionResponseStreamChoice]
cacheflow/entrypoints/simple_fastapi_frontend.py new
+51
@@ -0,0 +1,51 @@
1 +import argparse
2 +import json
3 +from typing import AsyncGenerator
4 +
5 +from fastapi import FastAPI, Request
6 +from fastapi.responses import StreamingResponse
7 +import uvicorn
8 +
9 +from cacheflow.sampling_params import SamplingParams
10 +from cacheflow.server.arg_utils import ServerArgs
11 +from cacheflow.server.async_llm_server import AsyncLLMServer
12 +from cacheflow.server.ray_utils import initialize_cluster
13 +
14 +TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15 +app = FastAPI()
16 +
17 +
18 +@app.post("/generate")
19 +async def generate_stream(request: Request) -> StreamingResponse:
20 + request_dict = await request.json()
21 + prompt = request_dict.pop("prompt")
22 + sampling_params = SamplingParams(**request_dict)
23 + results_generator = server.generate(prompt, sampling_params)
24 +
25 + async def stream_results() -> AsyncGenerator[bytes, None]:
26 + async for request_output in results_generator:
27 + prompt = request_output.prompt
28 + text_outputs = [
29 + prompt + output.text
30 + for output in request_output.outputs
31 + ]
32 + ret = {
33 + "text": text_outputs,
34 + "error": 0,
35 + }
36 + yield (json.dumps(ret) + "\0").encode("utf-8")
37 +
38 + return StreamingResponse(stream_results())
39 +
40 +
41 +if __name__ == "__main__":
42 + parser = argparse.ArgumentParser()
43 + parser.add_argument("--host", type=str, default="localhost")
44 + parser.add_argument("--port", type=int, default=8001)
45 + parser = ServerArgs.add_cli_args(parser)
46 + args = parser.parse_args()
47 +
48 + server_args = ServerArgs.from_cli_args(args)
49 + server = AsyncLLMServer.from_server_args(server_args)
50 +
51 + uvicorn.run(app, host=args.host, port=args.port, log_level="info")
cacheflow/model_executor/layers/sampler.py
+3 -3
@@ -1,5 +1,5 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 -from typing import Dict, List, Tuple
2 +from typing import Dict, List, Tuple, Optional
3
4 import numpy as np
5 import torch
@@ -258,9 +258,9 @@ def _apply_top_p_top_k(
258
259 def _get_topk_logprobs(
260 logprobs: torch.Tensor,
261 - num_logprobs: int,
261 + num_logprobs: Optional[int],
262 ) -> Dict[int, float]:
263 - if num_logprobs == 0:
263 + if num_logprobs is None or num_logprobs == 0:
264 return {}
265
266 topk_logprobs, topk_ids = torch.topk(logprobs, num_logprobs)
cacheflow/outputs.py
+18 -11
@@ -1,6 +1,6 @@
1 -from typing import Dict, List
1 +from typing import Dict, List, Optional
2
3 -from cacheflow.sequence import SequenceGroup
3 +from cacheflow.sequence import SequenceGroup, SequenceStatus
4
5
6 class CompletionOutput:
@@ -12,19 +12,25 @@ class CompletionOutput:
12 token_ids: List[int],
13 cumulative_logprob: float,
14 logprobs: List[Dict[int, float]],
15 + finish_reason: Optional[str] = None,
16 ) -> None:
17 self.index = index
18 self.text = text
19 self.token_ids = token_ids
20 self.cumulative_logprob = cumulative_logprob
21 self.logprobs = logprobs
22 + self.finish_reason = finish_reason
23 +
24 + def finished(self) -> bool:
25 + return self.finish_reason is not None
26
27 def __repr__(self) -> str:
28 return (f"CompletionOutput(index={self.index}, "
29 f"text={self.text!r}, "
30 f"token_ids={self.token_ids}, "
31 f"cumulative_logprob={self.cumulative_logprob}, "
27 - f"logprobs={self.logprobs})")
32 + f"logprobs={self.logprobs},"
33 + f"finish_reason={self.finish_reason})")
34
35
36 class RequestOutput:
@@ -35,13 +41,11 @@ class RequestOutput:
41 prompt: str,
42 prompt_token_ids: List[int],
43 outputs: List[CompletionOutput],
38 - done: bool,
44 ) -> None:
45 self.request_id = request_id
46 self.prompt = prompt
47 self.prompt_token_ids = prompt_token_ids
48 self.outputs = outputs
44 - self.done = done
49
50 @classmethod
51 def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
@@ -57,25 +61,28 @@ class RequestOutput:
61 outputs: List[CompletionOutput] = []
62 for seq in top_n_seqs:
63 logprobs = seq.output_logprobs
60 - if seq_group.sampling_params.logprobs == 0:
64 + if seq_group.sampling_params.logprobs is None:
65 # NOTE: We need to take care of this case because the sequence
66 # always has the logprobs of the sampled tokens even if the
67 # logprobs are not requested.
68 logprobs = {}
69 + finshed_reason = SequenceStatus.get_finished_reason(seq.status)
70 output = CompletionOutput(seqs.index(seq), seq.output_text,
71 seq.get_output_token_ids(),
67 - seq.get_cumulative_logprob(), logprobs)
72 + seq.get_cumulative_logprob(), logprobs,
73 + finshed_reason)
74 outputs.append(output)
75
76 # Every sequence in the sequence group should have the same prompt.
77 prompt = top_n_seqs[0].prompt
78 prompt_token_ids = top_n_seqs[0].data.prompt_token_ids
73 - return cls(seq_group.request_id, prompt, prompt_token_ids, outputs,
74 - seq_group.is_finished())
79 + return cls(seq_group.request_id, prompt, prompt_token_ids, outputs)
80
81 def __repr__(self) -> str:
82 return (f"RequestOutput(request_id={self.request_id}, "
83 f"prompt={self.prompt!r}, "
84 f"prompt_token_ids={self.prompt_token_ids}, "
80 - f"outputs={self.outputs}, "
81 - f"done={self.done})")
85 + f"outputs={self.outputs})")
86 +
87 + def finished(self) -> bool:
88 + return all(output.finished() for output in self.outputs)
cacheflow/sampling_params.py
+2 -2
@@ -53,7 +53,7 @@ class SamplingParams:
53 stop: Union[str, List[str]] = [],
54 ignore_eos: bool = False,
55 max_tokens: int = 16,
56 - logprobs: int = 0,
56 + logprobs: Optional[int] = None,
57 ) -> None:
58 self.n = n
59 self.best_of = best_of if best_of is not None else n
@@ -98,7 +98,7 @@ class SamplingParams:
98 if self.max_tokens < 1:
99 raise ValueError(
100 f"max_tokens must be at least 1, got {self.max_tokens}.")
101 - if self.logprobs < 0:
101 + if self.logprobs is not None and self.logprobs < 0:
102 raise ValueError(
103 f"logprobs must be non-negative, got {self.logprobs}.")
104
cacheflow/sequence.py
+21 -5
@@ -1,6 +1,6 @@
1 import copy
2 import enum
3 -from typing import Dict, List, Optional
3 +from typing import Dict, List, Optional, Union
4
5 from cacheflow.block import LogicalTokenBlock
6 from cacheflow.sampling_params import SamplingParams
@@ -10,8 +10,25 @@ class SequenceStatus(enum.Enum):
10 WAITING = enum.auto()
11 RUNNING = enum.auto()
12 SWAPPED = enum.auto()
13 - FINISHED = enum.auto()
14 -
13 + FINISHED_STOPPED = enum.auto()
14 + FINISHED_LENGTH_CAPPED = enum.auto()
15 +
16 + @staticmethod
17 + def is_finished(status: "SequenceStatus") -> bool:
18 + return status in [
19 + SequenceStatus.FINISHED_STOPPED,
20 + SequenceStatus.FINISHED_LENGTH_CAPPED,
21 + ]
22 +
23 + @staticmethod
24 + def get_finished_reason(status: "SequenceStatus") -> Union[str, None]:
25 + if status == SequenceStatus.FINISHED_STOPPED:
26 + finish_reason = "stop"
27 + elif status == SequenceStatus.FINISHED_LENGTH_CAPPED:
28 + finish_reason = "length"
29 + else:
30 + finish_reason = None
31 + return finish_reason
32
33 class SequenceData:
34
@@ -20,7 +37,6 @@ class SequenceData:
37 prompt_token_ids: List[int],
38 ) -> None:
39 self.prompt_token_ids = prompt_token_ids
23 -
40 self.output_token_ids: List[int] = []
41 self.cumulative_logprob = 0.0
42
@@ -166,7 +182,7 @@ class SequenceGroup:
182 raise ValueError(f'Sequence {seq_id} not found.')
183
184 def is_finished(self) -> bool:
169 - return all(seq.status == SequenceStatus.FINISHED for seq in self.seqs)
185 + return all(SequenceStatus.is_finished(seq.status) for seq in self.seqs)
186
187 def __repr__(self) -> str:
188 return (f"SequenceGroup(request_id={self.request_id}, "
cacheflow/server/async_llm_server.py renamed
+21 -46
@@ -1,26 +1,20 @@
1 -import argparse
1 import asyncio
3 -import json
2 import time
5 -from typing import Any, Dict
6 -import uuid
3 +from typing import Dict, Optional
4
8 -from fastapi import FastAPI, Request
9 -from fastapi.responses import StreamingResponse
5 import ray
11 -import uvicorn
6
7 from cacheflow.outputs import RequestOutput
8 from cacheflow.sampling_params import SamplingParams
9 from cacheflow.server.arg_utils import ServerArgs
10 from cacheflow.server.llm_server import LLMServer
11 from cacheflow.server.ray_utils import initialize_cluster
12 +from cacheflow.utils import random_uuid
13
14 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
20 -app = FastAPI()
15
16
23 -class FastAPIServer:
17 +class AsyncLLMServer:
18
19 def __init__(self, server_use_ray: bool, *args, **kwargs) -> None:
20 if server_use_ray:
@@ -45,15 +39,15 @@ class FastAPIServer:
39 self.request_outputs[request_id] = request_output
40 self.request_events[request_id].set()
41
48 - async def generate(self, request_dict: Dict[str, Any]):
42 + async def generate(self, prompt: str, sampling_params: SamplingParams,
43 + request_id: Optional[str] = None) -> RequestOutput:
44 # Preprocess the request.
45 arrival_time = time.time()
51 - prompt = request_dict.pop("prompt")
52 - sampling_params = SamplingParams(**request_dict)
46
47 # Create an event to notify us that there is new output from the
48 # cacheflow server.
56 - request_id = str(uuid.uuid4().hex[:8])
49 + if request_id is None:
50 + request_id = random_uuid()
51 request_event = asyncio.Event()
52 self.request_events[request_id] = request_event
53
@@ -82,19 +76,10 @@ class FastAPIServer:
76
77 # Decode and return new outputs.
78 request_output = self.request_outputs[request_id]
85 - prompt = request_output.prompt
86 - text_outputs = [
87 - prompt + output.text
88 - for output in request_output.outputs
89 - ]
90 - ret = {
91 - "text": text_outputs,
92 - "error": 0,
93 - }
94 - yield (json.dumps(ret) + "\0").encode("utf-8")
79 + yield request_output
80
81 # Once finished, release the resources of the sequence group.
97 - if request_output.done:
82 + if request_output.finished():
83 del self.request_outputs[request_id]
84 del self.request_events[request_id]
85 # Kick the server if the server is not running. This is to
@@ -104,25 +89,15 @@ class FastAPIServer:
89 await self.server_step()
90 break
91
107 -
108 -@app.post("/generate")
109 -async def generate_stream(request: Request):
110 - request_dict = await request.json()
111 - return StreamingResponse(server.generate(request_dict))
112 -
113 -
114 -if __name__ == "__main__":
115 - parser = argparse.ArgumentParser()
116 - parser.add_argument("--host", type=str, default="localhost")
117 - parser.add_argument("--port", type=int, default=10002)
118 - parser = ServerArgs.add_cli_args(parser)
119 - args = parser.parse_args()
120 -
121 - server_configs = ServerArgs.from_cli_args(args).create_server_configs()
122 - parallel_config = server_configs[2]
123 - distributed_init_method, stage_devices = initialize_cluster(parallel_config)
124 -
125 - server = FastAPIServer(args.use_ray, *server_configs,
126 - distributed_init_method, stage_devices,
127 - log_stats=not args.disable_log_stats)
128 - uvicorn.run(app, host=args.host, port=args.port, log_level="info")
92 + @classmethod
93 + def from_server_args(cls, server_args: ServerArgs) -> "AsyncLLMServer":
94 + # Create the server configs.
95 + server_configs = server_args.create_server_configs()
96 + parallel_config = server_configs[2]
97 + # Initialize the cluster.
98 + distributed_init_method, devices = initialize_cluster(parallel_config)
99 + # Create the LLM server.
100 + server = cls(server_args.use_ray, *server_configs,
101 + distributed_init_method, devices,
102 + log_stats=not server_args.disable_log_stats)
103 + return server
cacheflow/server/llm_server.py
+8 -5
@@ -210,7 +210,8 @@ class LLMServer:
210 # Truncate the output text so that the stop string is
211 # not included in the output.
212 seq.output_text = seq.output_text[:-len(stop_str)]
213 - self.scheduler.free_seq(seq)
213 + self.scheduler.free_seq(seq,
214 + SequenceStatus.FINISHED_STOPPED)
215 stopped = True
216 break
217 if stopped:
@@ -218,12 +219,14 @@ class LLMServer:
219
220 # Check if the sequence has reached max_tokens.
221 if seq.get_output_len() == sampling_params.max_tokens:
221 - self.scheduler.free_seq(seq)
222 + self.scheduler.free_seq(
223 + seq, SequenceStatus.FINISHED_LENGTH_CAPPED)
224 continue
225 # Check if the sequence has generated the EOS token.
226 if not sampling_params.ignore_eos:
227 if seq.get_last_token_id() == self.tokenizer.eos_token_id:
226 - self.scheduler.free_seq(seq)
228 + self.scheduler.free_seq(seq,
229 + SequenceStatus.FINISHED_STOPPED)
230 continue
231
232 def _run_workers(
@@ -238,10 +241,10 @@ class LLMServer:
241 executor = getattr(worker, method)
242 if self.parallel_config.use_ray:
243 executor = executor.remote
241 -
244 +
245 output = executor(*args, **kwargs)
246 all_outputs.append(output)
244 -
247 +
248 if self.parallel_config.use_ray:
249 all_outputs = ray.get(all_outputs)
250
cacheflow/utils.py
+5
@@ -1,4 +1,5 @@
1 import enum
2 +import uuid
3
4 import psutil
5 import torch
@@ -31,3 +32,7 @@ def get_gpu_memory(gpu: int = 0) -> int:
32 def get_cpu_memory() -> int:
33 """Returns the total CPU memory of the node in bytes."""
34 return psutil.virtual_memory().total
35 +
36 +
37 +def random_uuid() -> str:
38 + return str(uuid.uuid4().hex)
examples/gradio_webserver.py renamed
+7 -6
@@ -1,6 +1,5 @@
1 import argparse
2 import json
3 -import time
3
4 import gradio as gr
5 import requests
@@ -24,9 +23,9 @@ def http_bot(prompt):
23 def build_demo():
24 with gr.Blocks() as demo:
25 gr.Markdown(
27 - "# Cacheflow demo\n"
26 + "# Cacheflow text completion demo\n"
27 )
29 - inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")# .style(container=False)
28 + inputbox = gr.Textbox(label="Input", placeholder="Enter text and press ENTER")
29 outputbox = gr.Textbox(label="Output", placeholder="Generated result from the model")
30 inputbox.submit(http_bot, [inputbox], [outputbox])
31 return demo
@@ -35,9 +34,11 @@ def build_demo():
34 if __name__ == "__main__":
35 parser = argparse.ArgumentParser()
36 parser.add_argument("--host", type=str, default="localhost")
38 - parser.add_argument("--port", type=int, default=10003)
39 - parser.add_argument("--model-url", type=str, default="http://localhost:10002/generate")
37 + parser.add_argument("--port", type=int, default=8002)
38 + parser.add_argument("--model-url", type=str, default="http://localhost:8001/generate")
39 args = parser.parse_args()
40
41 demo = build_demo()
43 - demo.queue(concurrency_count=100).launch(server_name=args.host, server_port=args.port)
\ No newline at end of file
42 + demo.queue(concurrency_count=100).launch(server_name=args.host,
43 + server_port=args.port,
44 + share=True)
examples/openai_client.py new
+22
@@ -0,0 +1,22 @@
1 +import openai
2 +openai.api_key = "EMPTY"
3 +openai.api_base = "http://localhost:8000/v1"
4 +model = "facebook/opt-125m"
5 +
6 +# list models
7 +models = openai.Model.list()
8 +print(models)
9 +
10 +# create a completion
11 +
12 +stream = True
13 +completion = openai.Completion.create(
14 + model=model, prompt="A robot may not injure a human being", echo=False, n=2,
15 + best_of=3, stream=stream, logprobs=3)
16 +
17 +# print the completion
18 +if stream:
19 + for c in completion:
20 + print(c)
21 +else:
22 + print("completion:", completion)
examples/simple_fastapi_client.py new
+48
@@ -0,0 +1,48 @@
1 +import argparse
2 +import requests
3 +import json
4 +
5 +def clear_line(n=1):
6 + LINE_UP = '\033[1A'
7 + LINE_CLEAR = '\x1b[2K'
8 + for i in range(n):
9 + print(LINE_UP, end=LINE_CLEAR, flush=True)
10 +
11 +
12 +def http_request(prompt: str, api_url: str, n: int = 1):
13 + headers = {"User-Agent": "Test Client"}
14 + pload = {
15 + "prompt": prompt,
16 + "n": n,
17 + "use_beam_search": True,
18 + "temperature": 0.0,
19 + "max_tokens": 16,
20 + }
21 + response = requests.post(api_url, headers=headers, json=pload, stream=True)
22 +
23 + for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
24 + if chunk:
25 + data = json.loads(chunk.decode("utf-8"))
26 + output = data["text"]
27 + yield output
28 +
29 +
30 +if __name__ == "__main__":
31 + parser = argparse.ArgumentParser()
32 + parser.add_argument("--host", type=str, default="localhost")
33 + parser.add_argument("--port", type=int, default=8001)
34 + parser.add_argument("--n", type=int, default=4)
35 + parser.add_argument("--prompt", type=str, default="San Francisco is a")
36 + args = parser.parse_args()
37 + prompt = args.prompt
38 + api_url = f"http://{args.host}:{args.port}/generate"
39 + n = args.n
40 +
41 + print(f"Prompt: {prompt}\n", flush=True)
42 + num_printed_lines = 0
43 + for h in http_request(prompt, api_url, n):
44 + clear_line(num_printed_lines)
45 + num_printed_lines = 0
46 + for i, line in enumerate(h):
47 + num_printed_lines += 1
48 + print(f"Beam candidate {i}: {line}", flush=True)
examples/simple_server.py
+5 -4
@@ -1,5 +1,4 @@
1 import argparse
2 -import uuid
2
3 from cacheflow import ServerArgs, LLMServer, SamplingParams
4
@@ -20,17 +19,19 @@ def main(args: argparse.Namespace):
19 SamplingParams(n=3, best_of=3, use_beam_search=True, temperature=0.0)),
20 ]
21
22 + request_id = 0
23 +
24 # Run the server.
25 while True:
26 # To test iteration-level scheduling, we add one request at each step.
27 if test_prompts:
28 prompt, sampling_params = test_prompts.pop(0)
28 - request_id = str(uuid.uuid4().hex[:8])
29 - server.add_request(request_id, prompt, sampling_params)
29 + server.add_request(str(request_id), prompt, sampling_params)
30 + request_id += 1
31
32 request_outputs = server.step()
33 for request_output in request_outputs:
33 - if request_output.done:
34 + if request_output.finished():
35 print(request_output)
36
37 if not (server.has_unfinished_requests() or test_prompts):
playground/http_client.py deleted
-20
@@ -1,20 +0,0 @@
1 -import requests
2 -import json
3 -
4 -def http_bot():
5 - prompt = "How are you? I'm fine."
6 -
7 - headers = {"User-Agent": "Test Client"}
8 - pload = {
9 - "prompt": prompt,
10 - }
11 - response = requests.post("http://localhost:10002", headers=headers, json=pload, stream=True)
12 -
13 - for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
14 - if chunk:
15 - data = json.loads(chunk.decode("utf-8"))
16 - output = data["text"]
17 - yield output
18 -
19 -for h in http_bot():
20 - print(h, end="", flush=True)
\ No newline at end of file
playground/streaming_fastapi_worker.py deleted
-40
@@ -1,40 +0,0 @@
1 -import argparse
2 -import asyncio
3 -import time
4 -from typing import Union
5 -import json
6 -
7 -from fastapi import FastAPI, Request
8 -from fastapi.responses import StreamingResponse
9 -import uvicorn
10 -
11 -
12 -app = FastAPI()
13 -
14 -
15 -async def text_streamer(args):
16 - context = args["prompt"]
17 - words = context.split(" ")
18 - for word in words:
19 - await asyncio.sleep(1)
20 - print("word:", word)
21 - ret = {
22 - "text": word + " ",
23 - "error": 0,
24 - }
25 - yield (json.dumps(ret) + "\0").encode("utf-8")
26 -
27 -
28 -@app.post("/")
29 -async def read_root(request: Request):
30 - args = await request.json()
31 - return StreamingResponse(text_streamer(args))
32 -
33 -
34 -if __name__ == "__main__":
35 - parser = argparse.ArgumentParser()
36 - parser.add_argument("--host", type=str, default="localhost")
37 - parser.add_argument("--port", type=int, default=10002)
38 - args = parser.parse_args()
39 -
40 - uvicorn.run(app, host=args.host, port=args.port, log_level="info")
requirements.txt
+1
@@ -8,3 +8,4 @@ transformers >= 4.28.0 # Required for LLaMA.
8 xformers >= 0.0.19
9 fastapi
10 uvicorn
11 +pydantic # Required for OpenAI server.
test_cli_client.py deleted
-23
@@ -1,23 +0,0 @@
1 -import requests
2 -import json
3 -
4 -def http_request():
5 - prompt = "Ion Stoica is a"
6 -
7 - headers = {"User-Agent": "Test Client"}
8 - pload = {
9 - "prompt": prompt,
10 - "n": 4,
11 - "use_beam_search": True,
12 - "temperature": 0.0,
13 - }
14 - response = requests.post("http://localhost:10002/generate", headers=headers, json=pload, stream=True)
15 -
16 - for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
17 - if chunk:
18 - data = json.loads(chunk.decode("utf-8"))
19 - output = data["text"]
20 - yield output
21 -
22 -for h in http_request():
23 - print(h, flush=True)