A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix various issues of async servers (#135)
Zhuohan Li committed
Jun 5, 2023 at 23:44 UTC
1a956e136beae057746af6257ffa8da601730f10
11 files changed
+289
-121
benchmarks/benchmark_async_llm_server.py
new
+58
@@ -0,0 +1,58 @@
1
+import argparse
2
+import json
3
+import threading
4
+import time
5
+
6
+import requests
7
+
8
+
9
+def main(args: argparse.Namespace):
10
+ prompts = [f"Tell me a story with more than {''.join([str(i+1)] * 5)} words"
11
+ for i in range(args.n_threads)]
12
+
13
+ headers = {"User-Agent": "CacheFlow Benchmark Client"}
14
+ ploads = [{
15
+ "prompt": p,
16
+ "max_tokens": args.max_tokens,
17
+ "temperature": 0.0,
18
+ "ignore_eos": True,
19
+ } for p in prompts]
20
+
21
+ def send_request(results, i):
22
+ response = requests.post(args.api_url, headers=headers,
23
+ json=ploads[i], stream=True)
24
+ results[i] = response
25
+
26
+ # use args.n_threads to prompt the backend
27
+ tik = time.time()
28
+ threads = []
29
+ results = [None] * args.n_threads
30
+ for i in range(args.n_threads):
31
+ t = threading.Thread(target=send_request, args=(results, i))
32
+ t.start()
33
+ threads.append(t)
34
+
35
+ for t in threads:
36
+ t.join()
37
+
38
+ print(f"Time (POST): {time.time() - tik} s")
39
+ n_words = 0
40
+
41
+ for i, response in enumerate(results):
42
+ k = list(response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"))
43
+ response_new_words = json.loads(k[-2].decode("utf-8"))["text"][0]
44
+ n_words += len(response_new_words.split(" ")) - len(prompts[i].split(" "))
45
+
46
+ time_seconds = time.time() - tik
47
+ print(f"Time (total): {time_seconds:.3f}s to finish, n_threads: {args.n_threads}, "
48
+ f"throughput: {n_words / time_seconds} words/s.")
49
+
50
+
51
+if __name__ == "__main__":
52
+ parser = argparse.ArgumentParser()
53
+ parser.add_argument("--api-url", type=str, default="http://localhost:8001/generate")
54
+ parser.add_argument("--max-tokens", type=int, default=128)
55
+ parser.add_argument("--n-threads", type=int, default=128)
56
+ args = parser.parse_args()
57
+
58
+ main(args)
cacheflow/config.py
+3
-3
@@ -116,15 +116,15 @@ class ParallelConfig:
116
self,
117
pipeline_parallel_size: int,
118
tensor_parallel_size: int,
119
- use_ray: bool,
119
+ worker_use_ray: bool,
120
) -> None:
121
self.pipeline_parallel_size = pipeline_parallel_size
122
self.tensor_parallel_size = tensor_parallel_size
123
- self.use_ray = use_ray
123
+ self.worker_use_ray = worker_use_ray
124
125
self.world_size = pipeline_parallel_size * tensor_parallel_size
126
if self.world_size > 1:
127
- self.use_ray = True
127
+ self.worker_use_ray = True
128
self._verify_args()
129
130
def _verify_args(self) -> None:
cacheflow/core/block_manager.py
+6
-3
@@ -148,7 +148,7 @@ class BlockSpaceManager:
148
# the sequences in the same group.
149
blocks: Set[PhysicalTokenBlock] = set()
150
for seq in seq_group.get_seqs():
151
- if SequenceStatus.is_finished(seq.status):
151
+ if seq.is_finished():
152
continue
153
block_table = self.block_tables[seq.seq_id]
154
for block in block_table:
@@ -169,7 +169,7 @@ class BlockSpaceManager:
169
# CPU block -> GPU block.
170
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
171
for seq in seq_group.get_seqs():
172
- if SequenceStatus.is_finished(seq.status):
172
+ if seq.is_finished():
173
continue
174
new_block_table: BlockTable = []
175
block_table = self.block_tables[seq.seq_id]
@@ -200,7 +200,7 @@ class BlockSpaceManager:
200
# GPU block -> CPU block.
201
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
202
for seq in seq_group.get_seqs():
203
- if SequenceStatus.is_finished(seq.status):
203
+ if seq.is_finished():
204
continue
205
new_block_table: BlockTable = []
206
block_table = self.block_tables[seq.seq_id]
@@ -231,6 +231,9 @@ class BlockSpaceManager:
231
self.cpu_allocator.free(block)
232
233
def free(self, seq: Sequence) -> None:
234
+ if seq.seq_id not in self.block_tables:
235
+ # Already freed or haven't been scheduled yet.
236
+ return
237
block_table = self.block_tables[seq.seq_id]
238
self._free_block_table(block_table)
239
del self.block_tables[seq.seq_id]
cacheflow/core/scheduler.py
+13
-1
@@ -12,7 +12,7 @@ from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
12
13
logger = init_logger(__name__)
14
15
-_LOGGING_INTERVAL_SEC = 10
15
+_LOGGING_INTERVAL_SEC = 5
16
17
18
class PreemptionMode(enum.Enum):
@@ -84,6 +84,18 @@ class Scheduler:
84
# Add sequence groups to the waiting queue.
85
self.waiting.append(seq_group)
86
87
+ def abort_seq_group(self, request_id: str) -> None:
88
+ for state_queue in [self.waiting, self.running, self.swapped]:
89
+ for seq_group in state_queue:
90
+ if seq_group.request_id == request_id:
91
+ # Remove the sequence group from the state queue.
92
+ state_queue.remove(seq_group)
93
+ for seq in seq_group.seqs:
94
+ if seq.is_finished():
95
+ continue
96
+ self.free_seq(seq, SequenceStatus.FINISHED_ABORTED)
97
+ return
98
+
99
def has_unfinished_seqs(self) -> bool:
100
return self.waiting or self.running or self.swapped
101
cacheflow/entrypoints/openai/openai_frontend.py
+23
-7
@@ -7,13 +7,14 @@ import time
7
from typing import AsyncGenerator, Dict, List, Optional
8
9
import fastapi
10
+from fastapi import BackgroundTasks, Request
11
from fastapi.exceptions import RequestValidationError
12
from fastapi.middleware.cors import CORSMiddleware
13
from fastapi.responses import StreamingResponse, JSONResponse
14
import uvicorn
15
16
from cacheflow.outputs import RequestOutput
16
-from cacheflow.server.arg_utils import ServerArgs
17
+from cacheflow.server.arg_utils import AsyncServerArgs
18
from cacheflow.server.async_llm_server import AsyncLLMServer
19
from cacheflow.server.tokenizer_utils import get_tokenizer
20
from cacheflow.logger import init_logger
@@ -33,6 +34,7 @@ from cacheflow.entrypoints.openai.protocol import (
34
UsageInfo,
35
)
36
37
+TIMEOUT_KEEP_ALIVE = 5 # seconds
38
39
logger = init_logger(__name__)
40
served_model = None
@@ -93,7 +95,8 @@ def create_logprobs(token_ids: List[int],
95
96
97
@app.post("/v1/completions")
96
-async def create_completion(request: CompletionRequest):
98
+async def create_completion(raw_request: Request):
99
+ request = CompletionRequest(**await raw_request.json())
100
logger.info(f"Received completion request: {request}")
101
102
error_check_ret = await check_model(request)
@@ -139,7 +142,7 @@ async def create_completion(request: CompletionRequest):
142
return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
143
144
result_generator = server.generate(prompt, sampling_params,
142
- request_id=request_id)
145
+ request_id)
146
147
# Similar to the OpenAI API, when n != best_of, we do not stream the
148
# results. In addition, we do not stream the results when use beam search.
@@ -147,6 +150,9 @@ async def create_completion(request: CompletionRequest):
150
(request.best_of is None or request.n == request.best_of) and
151
not request.use_beam_search)
152
153
+ async def abort_request() -> None:
154
+ await server.abort(request_id)
155
+
156
def create_stream_response_json(index: int,
157
text: str,
158
logprobs: Optional[LogProbs] = None,
@@ -203,12 +209,21 @@ async def create_completion(request: CompletionRequest):
209
210
# Streaming response
211
if stream:
212
+ background_tasks = BackgroundTasks()
213
+ # Abort the request if the client disconnects.
214
+ background_tasks.add_task(abort_request)
215
return StreamingResponse(completion_stream_generator(),
207
- media_type="text/event-stream")
216
+ media_type="text/event-stream",
217
+ background=background_tasks)
218
219
# Non-streaming response
220
final_res: RequestOutput = None
221
async for res in result_generator:
222
+ if await raw_request.is_disconnected():
223
+ # Abort the request if the client disconnects.
224
+ await server.abort(request_id)
225
+ return create_error_response(HTTPStatus.BAD_REQUEST,
226
+ "Client disconnected")
227
final_res = res
228
assert final_res is not None
229
choices = []
@@ -276,7 +291,7 @@ if __name__ == "__main__":
291
help="The model name used in the API. If not specified, "
292
"the model name will be the same as the "
293
"huggingface name.")
279
- parser = ServerArgs.add_cli_args(parser)
294
+ parser = AsyncServerArgs.add_cli_args(parser)
295
args = parser.parse_args()
296
297
app.add_middleware(
@@ -291,10 +306,11 @@ if __name__ == "__main__":
306
307
served_model = args.served_model_name or args.model
308
294
- server_args = ServerArgs.from_cli_args(args)
309
+ server_args = AsyncServerArgs.from_cli_args(args)
310
server = AsyncLLMServer.from_server_args(server_args)
311
312
# A separate tokenizer to map token IDs to strings.
313
tokenizer = get_tokenizer(args.model)
314
300
- uvicorn.run(app, host=args.host, port=args.port, log_level="info")
315
+ uvicorn.run(app, host=args.host, port=args.port, log_level="info",
316
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/entrypoints/simple_fastapi_frontend.py
+17
-8
@@ -2,15 +2,16 @@ import argparse
2
import json
3
from typing import AsyncGenerator
4
5
-from fastapi import FastAPI, Request
5
+from fastapi import BackgroundTasks, FastAPI, Request
6
from fastapi.responses import StreamingResponse
7
import uvicorn
8
9
from cacheflow.sampling_params import SamplingParams
10
-from cacheflow.server.arg_utils import ServerArgs
10
+from cacheflow.server.arg_utils import AsyncServerArgs
11
from cacheflow.server.async_llm_server import AsyncLLMServer
12
-from cacheflow.server.ray_utils import initialize_cluster
12
+from cacheflow.utils import random_uuid
13
14
+TIMEOUT_KEEP_ALIVE = 5 # seconds.
15
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
16
app = FastAPI()
17
@@ -20,7 +21,8 @@ async def generate_stream(request: Request) -> StreamingResponse:
21
request_dict = await request.json()
22
prompt = request_dict.pop("prompt")
23
sampling_params = SamplingParams(**request_dict)
23
- results_generator = server.generate(prompt, sampling_params)
24
+ request_id = random_uuid()
25
+ results_generator = server.generate(prompt, sampling_params, request_id)
26
27
async def stream_results() -> AsyncGenerator[bytes, None]:
28
async for request_output in results_generator:
@@ -35,17 +37,24 @@ async def generate_stream(request: Request) -> StreamingResponse:
37
}
38
yield (json.dumps(ret) + "\0").encode("utf-8")
39
38
- return StreamingResponse(stream_results())
40
+ async def abort_request() -> None:
41
+ await server.abort(request_id)
42
+
43
+ background_tasks = BackgroundTasks()
44
+ # Abort the request if the client disconnects.
45
+ background_tasks.add_task(abort_request)
46
+ return StreamingResponse(stream_results(), background=background_tasks)
47
48
49
if __name__ == "__main__":
50
parser = argparse.ArgumentParser()
51
parser.add_argument("--host", type=str, default="localhost")
52
parser.add_argument("--port", type=int, default=8001)
45
- parser = ServerArgs.add_cli_args(parser)
53
+ parser = AsyncServerArgs.add_cli_args(parser)
54
args = parser.parse_args()
55
48
- server_args = ServerArgs.from_cli_args(args)
56
+ server_args = AsyncServerArgs.from_cli_args(args)
57
server = AsyncLLMServer.from_server_args(server_args)
58
51
- uvicorn.run(app, host=args.host, port=args.port, log_level="info")
59
+ uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
60
+ timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/sequence.py
+9
-1
@@ -12,12 +12,14 @@ class SequenceStatus(enum.Enum):
12
SWAPPED = enum.auto()
13
FINISHED_STOPPED = enum.auto()
14
FINISHED_LENGTH_CAPPED = enum.auto()
15
+ FINISHED_ABORTED = enum.auto()
16
17
@staticmethod
18
def is_finished(status: "SequenceStatus") -> bool:
19
return status in [
20
SequenceStatus.FINISHED_STOPPED,
21
SequenceStatus.FINISHED_LENGTH_CAPPED,
22
+ SequenceStatus.FINISHED_ABORTED,
23
]
24
25
@staticmethod
@@ -26,10 +28,13 @@ class SequenceStatus(enum.Enum):
28
finish_reason = "stop"
29
elif status == SequenceStatus.FINISHED_LENGTH_CAPPED:
30
finish_reason = "length"
31
+ elif status == SequenceStatus.FINISHED_ABORTED:
32
+ finish_reason = "abort"
33
else:
34
finish_reason = None
35
return finish_reason
36
37
+
38
class SequenceData:
39
40
def __init__(
@@ -137,6 +142,9 @@ class Sequence:
142
def get_cumulative_logprob(self) -> float:
143
return self.data.cumulative_logprob
144
145
+ def is_finished(self) -> bool:
146
+ return SequenceStatus.is_finished(self.status)
147
+
148
def fork(self, child_seq: 'Sequence') -> None:
149
child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
150
child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
@@ -182,7 +190,7 @@ class SequenceGroup:
190
raise ValueError(f'Sequence {seq_id} not found.')
191
192
def is_finished(self) -> bool:
185
- return all(SequenceStatus.is_finished(seq.status) for seq in self.seqs)
193
+ return all(seq.is_finished() for seq in self.seqs)
194
195
def __repr__(self) -> str:
196
return (f"SequenceGroup(request_id={self.request_id}, "
cacheflow/server/arg_utils.py
+72
-59
@@ -15,7 +15,7 @@ class ServerArgs:
15
use_dummy_weights: bool = False
16
dtype: str = "default"
17
seed: int = 0
18
- use_ray: bool = False
18
+ worker_use_ray: bool = False
19
pipeline_parallel_size: int = 1
20
tensor_parallel_size: int = 1
21
block_size: int = 16
@@ -32,7 +32,63 @@ class ServerArgs:
32
def add_cli_args(
33
parser: argparse.ArgumentParser,
34
) -> argparse.ArgumentParser:
35
- return _add_server_arguments(parser)
35
+ """Shared CLI arguments for CacheFlow servers."""
36
+ # Model arguments
37
+ parser.add_argument('--model', type=str, default='facebook/opt-125m',
38
+ help='name or path of the huggingface model to use')
39
+ parser.add_argument('--download-dir', type=str,
40
+ default=ServerArgs.download_dir,
41
+ help='directory to download and load the weights, '
42
+ 'default to the default cache dir of '
43
+ 'huggingface')
44
+ parser.add_argument('--use-np-weights', action='store_true',
45
+ help='save a numpy copy of model weights for '
46
+ 'faster loading. This can increase the disk '
47
+ 'usage by up to 2x.')
48
+ parser.add_argument('--use-dummy-weights', action='store_true',
49
+ help='use dummy values for model weights')
50
+ # TODO(woosuk): Support FP32.
51
+ parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
52
+ choices=['default', 'half', 'bfloat16'],
53
+ help='data type for model weights and activations. '
54
+ 'The "default" option will use FP16 precision '
55
+ 'for FP32 and FP16 models, and BF16 precision '
56
+ 'for BF16 models.')
57
+ # Parallel arguments
58
+ parser.add_argument('--worker-use-ray', action='store_true',
59
+ help='use Ray for distributed serving, will be '
60
+ 'automatically set when using more than 1 GPU')
61
+ parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
62
+ default=ServerArgs.pipeline_parallel_size,
63
+ help='number of pipeline stages')
64
+ parser.add_argument('--tensor-parallel-size', '-tp', type=int,
65
+ default=ServerArgs.tensor_parallel_size,
66
+ help='number of tensor parallel replicas')
67
+ # KV cache arguments
68
+ parser.add_argument('--block-size', type=int,
69
+ default=ServerArgs.block_size,
70
+ choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
71
+ help='token block size')
72
+ # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
73
+ parser.add_argument('--seed', type=int, default=ServerArgs.seed,
74
+ help='random seed')
75
+ parser.add_argument('--swap-space', type=int,
76
+ default=ServerArgs.swap_space,
77
+ help='CPU swap space size (GiB) per GPU')
78
+ parser.add_argument('--gpu-memory-utilization', type=float,
79
+ default=ServerArgs.gpu_memory_utilization,
80
+ help='the percentage of GPU memory to be used for'
81
+ 'the model executor')
82
+ parser.add_argument('--max-num-batched-tokens', type=int,
83
+ default=ServerArgs.max_num_batched_tokens,
84
+ help='maximum number of batched tokens per '
85
+ 'iteration')
86
+ parser.add_argument('--max-num-seqs', type=int,
87
+ default=ServerArgs.max_num_seqs,
88
+ help='maximum number of sequences per iteration')
89
+ parser.add_argument('--disable-log-stats', action='store_true',
90
+ help='disable logging statistics')
91
+ return parser
92
93
@classmethod
94
def from_cli_args(cls, args: argparse.Namespace) -> "ServerArgs":
@@ -53,65 +109,22 @@ class ServerArgs:
109
self.swap_space)
110
parallel_config = ParallelConfig(self.pipeline_parallel_size,
111
self.tensor_parallel_size,
56
- self.use_ray)
112
+ self.worker_use_ray)
113
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
114
self.max_num_seqs)
115
return model_config, cache_config, parallel_config, scheduler_config
116
117
62
-def _add_server_arguments(
63
- parser: argparse.ArgumentParser,
64
-)-> argparse.ArgumentParser:
65
- """Shared CLI arguments for CacheFlow servers."""
66
- # Model arguments
67
- parser.add_argument('--model', type=str, default='facebook/opt-125m',
68
- help='name or path of the huggingface model to use')
69
- parser.add_argument('--download-dir', type=str,
70
- default=ServerArgs.download_dir,
71
- help='directory to download and load the weights, '
72
- 'default to the default cache dir of huggingface')
73
- parser.add_argument('--use-np-weights', action='store_true',
74
- help='save a numpy copy of model weights for faster '
75
- 'loading. This can increase the disk usage by up '
76
- 'to 2x.')
77
- parser.add_argument('--use-dummy-weights', action='store_true',
78
- help='use dummy values for model weights')
79
- # TODO(woosuk): Support FP32.
80
- parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
81
- choices=['default', 'half', 'bfloat16'],
82
- help=('data type for model weights and activations. '
83
- 'The "default" option will use FP16 precision '
84
- 'for FP32 and FP16 models, and BF16 precision '
85
- 'for BF16 models.'))
86
- # Parallel arguments
87
- parser.add_argument('--use-ray', action='store_true',
88
- help='use Ray for distributed serving, will be '
89
- 'automatically set when using more than 1 GPU')
90
- parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
91
- default=ServerArgs.pipeline_parallel_size,
92
- help='number of pipeline stages')
93
- parser.add_argument('--tensor-parallel-size', '-tp', type=int,
94
- default=ServerArgs.tensor_parallel_size,
95
- help='number of tensor parallel replicas')
96
- # KV cache arguments
97
- parser.add_argument('--block-size', type=int, default=ServerArgs.block_size,
98
- choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
99
- help='token block size')
100
- # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
101
- parser.add_argument('--seed', type=int, default=ServerArgs.seed,
102
- help='random seed')
103
- parser.add_argument('--swap-space', type=int, default=ServerArgs.swap_space,
104
- help='CPU swap space size (GiB) per GPU')
105
- parser.add_argument('--gpu-memory-utilization', type=float,
106
- default=ServerArgs.gpu_memory_utilization,
107
- help='the percentage of GPU memory to be used for the '
108
- 'model executor')
109
- parser.add_argument('--max-num-batched-tokens', type=int,
110
- default=ServerArgs.max_num_batched_tokens,
111
- help='maximum number of batched tokens per iteration')
112
- parser.add_argument('--max-num-seqs', type=int,
113
- default=ServerArgs.max_num_seqs,
114
- help='maximum number of sequences per iteration')
115
- parser.add_argument('--disable-log-stats', action='store_true',
116
- help='disable logging statistics')
117
- return parser
118
+@dataclass
119
+class AsyncServerArgs(ServerArgs):
120
+ server_use_ray: bool = False
121
+
122
+ @staticmethod
123
+ def add_cli_args(
124
+ parser: argparse.ArgumentParser,
125
+ ) -> argparse.ArgumentParser:
126
+ parser = ServerArgs.add_cli_args(parser)
127
+ parser.add_argument('--server-use-ray', action='store_true',
128
+ help='use Ray to start the LLM server in a '
129
+ 'separate process as the web server process.')
130
+ return parser
cacheflow/server/async_llm_server.py
+71
-22
@@ -2,37 +2,52 @@ import asyncio
2
import time
3
from typing import Dict, Optional
4
5
-import ray
6
-
5
+from cacheflow.logger import init_logger
6
from cacheflow.outputs import RequestOutput
7
from cacheflow.sampling_params import SamplingParams
9
-from cacheflow.server.arg_utils import ServerArgs
8
+from cacheflow.server.arg_utils import AsyncServerArgs
9
from cacheflow.server.llm_server import LLMServer
11
-from cacheflow.server.ray_utils import initialize_cluster
12
-from cacheflow.utils import random_uuid
10
+from cacheflow.server.ray_utils import ray, initialize_cluster
11
+
12
+logger = init_logger(__name__)
13
14
TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17
class AsyncLLMServer:
18
19
- def __init__(self, server_use_ray: bool, *args, **kwargs) -> None:
20
- if server_use_ray:
21
- remote_server_class = ray.remote(num_cpus=0)(LLMServer)
19
+ def __init__(self, worker_use_ray: bool, server_use_ray: bool,
20
+ *args, **kwargs) -> None:
21
+ self.worker_use_ray = worker_use_ray
22
+ self.server_use_ray = server_use_ray
23
+ if not self.server_use_ray:
24
+ server_class = LLMServer
25
+ elif self.worker_use_ray:
26
+ server_class = ray.remote(num_cpus=0)(LLMServer).remote
27
else:
23
- remote_server_class = ray.remote(num_gpus=1)(LLMServer)
24
- self.server = remote_server_class.remote(*args, **kwargs)
25
-
28
+ server_class = ray.remote(num_gpus=1)(LLMServer).remote
29
+ self.server = server_class(*args, **kwargs)
30
# Request id -> request output.
31
self.request_outputs: Dict[str, RequestOutput] = {}
32
# Request id -> event to notify that there is new output.
33
self.request_events: Dict[str, asyncio.Event] = {}
34
self.is_server_running = False
35
+ self.kicking_request_id: Optional[str] = None
36
32
- async def server_step(self):
37
+ async def server_step(self, kicking_request_id: Optional[str] = None):
38
self.is_server_running = True
34
- request_outputs = await self.server.step.remote()
39
+ self.kicking_request_id = kicking_request_id
40
+ if self.server_use_ray:
41
+ request_outputs = await self.server.step.remote()
42
+ else:
43
+ # Yield to the event loop to allow other coroutines to run
44
+ # while is_server_running is True. This let the server to add new
45
+ # requests into the queue.
46
+ await asyncio.sleep(0)
47
+ request_outputs = self.server.step()
48
self.is_server_running = False
49
+ self.kicking_request_id = None
50
+
51
# Notify the waiting coroutines that there are new outputs ready.
52
for request_output in request_outputs:
53
request_id = request_output.request_id
@@ -40,20 +55,26 @@ class AsyncLLMServer:
55
self.request_events[request_id].set()
56
57
async def generate(self, prompt: str, sampling_params: SamplingParams,
43
- request_id: Optional[str] = None) -> RequestOutput:
58
+ request_id: str) -> RequestOutput:
59
# Preprocess the request.
60
arrival_time = time.time()
61
62
# Create an event to notify us that there is new output from the
63
# cacheflow server.
49
- if request_id is None:
50
- request_id = random_uuid()
64
request_event = asyncio.Event()
65
self.request_events[request_id] = request_event
66
67
+ logger.info(f"Received request {request_id}: "
68
+ f"prompt: {prompt!r}, "
69
+ f"sampling params: {sampling_params}.")
70
+
71
# Add the request into the cacheflow server's waiting queue.
55
- await self.server.add_request.remote(
56
- request_id, prompt, sampling_params, arrival_time=arrival_time)
72
+ if self.server_use_ray:
73
+ await self.server.add_request.remote(
74
+ request_id, prompt, sampling_params, arrival_time=arrival_time)
75
+ else:
76
+ self.server.add_request(
77
+ request_id, prompt, sampling_params, arrival_time=arrival_time)
78
79
# The cacheflow server does not have a background loop that keeps
80
# processing incoming requests. Therefore, we need to keep kicking
@@ -61,7 +82,7 @@ class AsyncLLMServer:
82
while True:
83
# Kick the server if the server is not running.
84
if not self.is_server_running:
64
- await self.server_step()
85
+ await self.server_step(request_id)
86
87
# Wait for new output. The group_event will be set in server_step
88
# when there is new output available for the sequence group.
@@ -80,6 +101,8 @@ class AsyncLLMServer:
101
102
# Once finished, release the resources of the sequence group.
103
if request_output.finished():
104
+ logger.info(f"Finished request {request_id}.")
105
+
106
del self.request_outputs[request_id]
107
del self.request_events[request_id]
108
# Kick the server if the server is not running. This is to
@@ -89,15 +112,41 @@ class AsyncLLMServer:
112
await self.server_step()
113
break
114
115
+ async def abort(self, request_id: str) -> None:
116
+ if request_id not in self.request_events:
117
+ # The request has already finished or been aborted.
118
+ return
119
+
120
+ logger.info(f"Aborted request {request_id}.")
121
+
122
+ if self.server_use_ray:
123
+ await self.server.abort_request.remote(request_id)
124
+ else:
125
+ self.server.abort_request(request_id)
126
+
127
+ if request_id in self.request_events:
128
+ del self.request_events[request_id]
129
+ if request_id in self.request_outputs:
130
+ del self.request_outputs[request_id]
131
+
132
+ # To prevent deadlock when a request is aborted while the server is
133
+ # running.
134
+ if self.kicking_request_id == request_id:
135
+ self.is_server_running = False
136
+ self.kicking_request_id = None
137
+
138
@classmethod
93
- def from_server_args(cls, server_args: ServerArgs) -> "AsyncLLMServer":
139
+ def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMServer":
140
# Create the server configs.
141
server_configs = server_args.create_server_configs()
142
parallel_config = server_configs[2]
143
# Initialize the cluster.
98
- distributed_init_method, devices = initialize_cluster(parallel_config)
144
+ distributed_init_method, devices = initialize_cluster(
145
+ parallel_config, server_args.server_use_ray)
146
# Create the LLM server.
100
- server = cls(server_args.use_ray, *server_configs,
147
+ server = cls(server_args.worker_use_ray,
148
+ server_args.server_use_ray,
149
+ *server_configs,
150
distributed_init_method, devices,
151
log_stats=not server_args.disable_log_stats)
152
return server
cacheflow/server/llm_server.py
+7
-9
@@ -1,11 +1,6 @@
1
import time
2
from typing import Any, List, Optional
3
4
-try:
5
- import ray
6
-except ImportError:
7
- ray = None
8
-
4
from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5
SchedulerConfig)
6
from cacheflow.core.scheduler import Scheduler
@@ -13,7 +8,7 @@ from cacheflow.logger import init_logger
8
from cacheflow.outputs import RequestOutput
9
from cacheflow.sampling_params import SamplingParams
10
from cacheflow.server.arg_utils import ServerArgs
16
-from cacheflow.server.ray_utils import initialize_cluster
11
+from cacheflow.server.ray_utils import ray, initialize_cluster
12
from cacheflow.server.tokenizer_utils import (get_tokenizer,
13
detokenize_incrementally)
14
from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -62,7 +57,7 @@ class LLMServer:
57
assert len(stage_devices) == 1, "Only support one stage for now."
58
for rank, node_resource, _ in stage_devices[0]:
59
worker_cls = Worker
65
- if self.parallel_config.use_ray:
60
+ if self.parallel_config.worker_use_ray:
61
worker_cls = ray.remote(
62
num_cpus=0,
63
num_gpus=1,
@@ -152,6 +147,9 @@ class LLMServer:
147
# Add the sequence group to the scheduler.
148
self.scheduler.add_seq_group(seq_group)
149
150
+ def abort_request(self, request_id: str) -> None:
151
+ self.scheduler.abort_seq_group(request_id)
152
+
153
def get_num_unfinished_requests(self) -> int:
154
return self.scheduler.get_num_unfinished_seq_groups()
155
@@ -243,13 +241,13 @@ class LLMServer:
241
all_outputs = []
242
for worker in self.workers:
243
executor = getattr(worker, method)
246
- if self.parallel_config.use_ray:
244
+ if self.parallel_config.worker_use_ray:
245
executor = executor.remote
246
247
output = executor(*args, **kwargs)
248
all_outputs.append(output)
249
252
- if self.parallel_config.use_ray:
250
+ if self.parallel_config.worker_use_ray:
251
all_outputs = ray.get(all_outputs)
252
253
if get_all_outputs:
cacheflow/server/ray_utils.py
+10
-8
@@ -13,9 +13,18 @@ DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), devi
13
14
def initialize_cluster(
15
parallel_config: ParallelConfig,
16
+ server_use_ray: bool = False,
17
address: Optional[str] = None,
18
) -> Tuple[str, List[List[DeviceID]]]:
18
- if not parallel_config.use_ray:
19
+ if parallel_config.worker_use_ray or server_use_ray:
20
+ if ray is None:
21
+ raise ImportError(
22
+ "Ray is not installed. Please install Ray to use distributed "
23
+ "serving.")
24
+ # Connect to a ray cluster.
25
+ ray.init(address=address)
26
+
27
+ if not parallel_config.worker_use_ray:
28
# Initialize cluster locally.
29
port = random.randint(10000, 20000)
30
# We need to setup the distributed init method to make sure
@@ -24,13 +33,6 @@ def initialize_cluster(
33
all_stage_devices = [[(0, None, 0)]]
34
return distributed_init_method, all_stage_devices
35
27
- if ray is None:
28
- raise ImportError(
29
- "Ray is not installed. Please install Ray to use distributed "
30
- "serving.")
31
- # Connect to a ray cluster.
32
- ray.init(address=address)
33
-
36
# Assume we have a uniform cluster that each node has the same number of
37
# GPUs for now.
38
valid_node_resources = []