@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a956e136b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix various issues of async servers (#135)

Zhuohan Li committed Jun 5, 2023 at 23:44 UTC 1a956e136beae057746af6257ffa8da601730f10
11 files changed +289 -121
benchmarks/benchmark_async_llm_server.py new
+58
@@ -0,0 +1,58 @@
1 +import argparse
2 +import json
3 +import threading
4 +import time
5 +
6 +import requests
7 +
8 +
9 +def main(args: argparse.Namespace):
10 + prompts = [f"Tell me a story with more than {''.join([str(i+1)] * 5)} words"
11 + for i in range(args.n_threads)]
12 +
13 + headers = {"User-Agent": "CacheFlow Benchmark Client"}
14 + ploads = [{
15 + "prompt": p,
16 + "max_tokens": args.max_tokens,
17 + "temperature": 0.0,
18 + "ignore_eos": True,
19 + } for p in prompts]
20 +
21 + def send_request(results, i):
22 + response = requests.post(args.api_url, headers=headers,
23 + json=ploads[i], stream=True)
24 + results[i] = response
25 +
26 + # use args.n_threads to prompt the backend
27 + tik = time.time()
28 + threads = []
29 + results = [None] * args.n_threads
30 + for i in range(args.n_threads):
31 + t = threading.Thread(target=send_request, args=(results, i))
32 + t.start()
33 + threads.append(t)
34 +
35 + for t in threads:
36 + t.join()
37 +
38 + print(f"Time (POST): {time.time() - tik} s")
39 + n_words = 0
40 +
41 + for i, response in enumerate(results):
42 + k = list(response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"))
43 + response_new_words = json.loads(k[-2].decode("utf-8"))["text"][0]
44 + n_words += len(response_new_words.split(" ")) - len(prompts[i].split(" "))
45 +
46 + time_seconds = time.time() - tik
47 + print(f"Time (total): {time_seconds:.3f}s to finish, n_threads: {args.n_threads}, "
48 + f"throughput: {n_words / time_seconds} words/s.")
49 +
50 +
51 +if __name__ == "__main__":
52 + parser = argparse.ArgumentParser()
53 + parser.add_argument("--api-url", type=str, default="http://localhost:8001/generate")
54 + parser.add_argument("--max-tokens", type=int, default=128)
55 + parser.add_argument("--n-threads", type=int, default=128)
56 + args = parser.parse_args()
57 +
58 + main(args)
cacheflow/config.py
+3 -3
@@ -116,15 +116,15 @@ class ParallelConfig:
116 self,
117 pipeline_parallel_size: int,
118 tensor_parallel_size: int,
119 - use_ray: bool,
119 + worker_use_ray: bool,
120 ) -> None:
121 self.pipeline_parallel_size = pipeline_parallel_size
122 self.tensor_parallel_size = tensor_parallel_size
123 - self.use_ray = use_ray
123 + self.worker_use_ray = worker_use_ray
124
125 self.world_size = pipeline_parallel_size * tensor_parallel_size
126 if self.world_size > 1:
127 - self.use_ray = True
127 + self.worker_use_ray = True
128 self._verify_args()
129
130 def _verify_args(self) -> None:
cacheflow/core/block_manager.py
+6 -3
@@ -148,7 +148,7 @@ class BlockSpaceManager:
148 # the sequences in the same group.
149 blocks: Set[PhysicalTokenBlock] = set()
150 for seq in seq_group.get_seqs():
151 - if SequenceStatus.is_finished(seq.status):
151 + if seq.is_finished():
152 continue
153 block_table = self.block_tables[seq.seq_id]
154 for block in block_table:
@@ -169,7 +169,7 @@ class BlockSpaceManager:
169 # CPU block -> GPU block.
170 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
171 for seq in seq_group.get_seqs():
172 - if SequenceStatus.is_finished(seq.status):
172 + if seq.is_finished():
173 continue
174 new_block_table: BlockTable = []
175 block_table = self.block_tables[seq.seq_id]
@@ -200,7 +200,7 @@ class BlockSpaceManager:
200 # GPU block -> CPU block.
201 mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
202 for seq in seq_group.get_seqs():
203 - if SequenceStatus.is_finished(seq.status):
203 + if seq.is_finished():
204 continue
205 new_block_table: BlockTable = []
206 block_table = self.block_tables[seq.seq_id]
@@ -231,6 +231,9 @@ class BlockSpaceManager:
231 self.cpu_allocator.free(block)
232
233 def free(self, seq: Sequence) -> None:
234 + if seq.seq_id not in self.block_tables:
235 + # Already freed or haven't been scheduled yet.
236 + return
237 block_table = self.block_tables[seq.seq_id]
238 self._free_block_table(block_table)
239 del self.block_tables[seq.seq_id]
cacheflow/core/scheduler.py
+13 -1
@@ -12,7 +12,7 @@ from cacheflow.sequence import (Sequence, SequenceData, SequenceGroup,
12
13 logger = init_logger(__name__)
14
15 -_LOGGING_INTERVAL_SEC = 10
15 +_LOGGING_INTERVAL_SEC = 5
16
17
18 class PreemptionMode(enum.Enum):
@@ -84,6 +84,18 @@ class Scheduler:
84 # Add sequence groups to the waiting queue.
85 self.waiting.append(seq_group)
86
87 + def abort_seq_group(self, request_id: str) -> None:
88 + for state_queue in [self.waiting, self.running, self.swapped]:
89 + for seq_group in state_queue:
90 + if seq_group.request_id == request_id:
91 + # Remove the sequence group from the state queue.
92 + state_queue.remove(seq_group)
93 + for seq in seq_group.seqs:
94 + if seq.is_finished():
95 + continue
96 + self.free_seq(seq, SequenceStatus.FINISHED_ABORTED)
97 + return
98 +
99 def has_unfinished_seqs(self) -> bool:
100 return self.waiting or self.running or self.swapped
101
cacheflow/entrypoints/openai/openai_frontend.py
+23 -7
@@ -7,13 +7,14 @@ import time
7 from typing import AsyncGenerator, Dict, List, Optional
8
9 import fastapi
10 +from fastapi import BackgroundTasks, Request
11 from fastapi.exceptions import RequestValidationError
12 from fastapi.middleware.cors import CORSMiddleware
13 from fastapi.responses import StreamingResponse, JSONResponse
14 import uvicorn
15
16 from cacheflow.outputs import RequestOutput
16 -from cacheflow.server.arg_utils import ServerArgs
17 +from cacheflow.server.arg_utils import AsyncServerArgs
18 from cacheflow.server.async_llm_server import AsyncLLMServer
19 from cacheflow.server.tokenizer_utils import get_tokenizer
20 from cacheflow.logger import init_logger
@@ -33,6 +34,7 @@ from cacheflow.entrypoints.openai.protocol import (
34 UsageInfo,
35 )
36
37 +TIMEOUT_KEEP_ALIVE = 5 # seconds
38
39 logger = init_logger(__name__)
40 served_model = None
@@ -93,7 +95,8 @@ def create_logprobs(token_ids: List[int],
95
96
97 @app.post("/v1/completions")
96 -async def create_completion(request: CompletionRequest):
98 +async def create_completion(raw_request: Request):
99 + request = CompletionRequest(**await raw_request.json())
100 logger.info(f"Received completion request: {request}")
101
102 error_check_ret = await check_model(request)
@@ -139,7 +142,7 @@ async def create_completion(request: CompletionRequest):
142 return create_error_response(HTTPStatus.BAD_REQUEST, str(e))
143
144 result_generator = server.generate(prompt, sampling_params,
142 - request_id=request_id)
145 + request_id)
146
147 # Similar to the OpenAI API, when n != best_of, we do not stream the
148 # results. In addition, we do not stream the results when use beam search.
@@ -147,6 +150,9 @@ async def create_completion(request: CompletionRequest):
150 (request.best_of is None or request.n == request.best_of) and
151 not request.use_beam_search)
152
153 + async def abort_request() -> None:
154 + await server.abort(request_id)
155 +
156 def create_stream_response_json(index: int,
157 text: str,
158 logprobs: Optional[LogProbs] = None,
@@ -203,12 +209,21 @@ async def create_completion(request: CompletionRequest):
209
210 # Streaming response
211 if stream:
212 + background_tasks = BackgroundTasks()
213 + # Abort the request if the client disconnects.
214 + background_tasks.add_task(abort_request)
215 return StreamingResponse(completion_stream_generator(),
207 - media_type="text/event-stream")
216 + media_type="text/event-stream",
217 + background=background_tasks)
218
219 # Non-streaming response
220 final_res: RequestOutput = None
221 async for res in result_generator:
222 + if await raw_request.is_disconnected():
223 + # Abort the request if the client disconnects.
224 + await server.abort(request_id)
225 + return create_error_response(HTTPStatus.BAD_REQUEST,
226 + "Client disconnected")
227 final_res = res
228 assert final_res is not None
229 choices = []
@@ -276,7 +291,7 @@ if __name__ == "__main__":
291 help="The model name used in the API. If not specified, "
292 "the model name will be the same as the "
293 "huggingface name.")
279 - parser = ServerArgs.add_cli_args(parser)
294 + parser = AsyncServerArgs.add_cli_args(parser)
295 args = parser.parse_args()
296
297 app.add_middleware(
@@ -291,10 +306,11 @@ if __name__ == "__main__":
306
307 served_model = args.served_model_name or args.model
308
294 - server_args = ServerArgs.from_cli_args(args)
309 + server_args = AsyncServerArgs.from_cli_args(args)
310 server = AsyncLLMServer.from_server_args(server_args)
311
312 # A separate tokenizer to map token IDs to strings.
313 tokenizer = get_tokenizer(args.model)
314
300 - uvicorn.run(app, host=args.host, port=args.port, log_level="info")
315 + uvicorn.run(app, host=args.host, port=args.port, log_level="info",
316 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/entrypoints/simple_fastapi_frontend.py
+17 -8
@@ -2,15 +2,16 @@ import argparse
2 import json
3 from typing import AsyncGenerator
4
5 -from fastapi import FastAPI, Request
5 +from fastapi import BackgroundTasks, FastAPI, Request
6 from fastapi.responses import StreamingResponse
7 import uvicorn
8
9 from cacheflow.sampling_params import SamplingParams
10 -from cacheflow.server.arg_utils import ServerArgs
10 +from cacheflow.server.arg_utils import AsyncServerArgs
11 from cacheflow.server.async_llm_server import AsyncLLMServer
12 -from cacheflow.server.ray_utils import initialize_cluster
12 +from cacheflow.utils import random_uuid
13
14 +TIMEOUT_KEEP_ALIVE = 5 # seconds.
15 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
16 app = FastAPI()
17
@@ -20,7 +21,8 @@ async def generate_stream(request: Request) -> StreamingResponse:
21 request_dict = await request.json()
22 prompt = request_dict.pop("prompt")
23 sampling_params = SamplingParams(**request_dict)
23 - results_generator = server.generate(prompt, sampling_params)
24 + request_id = random_uuid()
25 + results_generator = server.generate(prompt, sampling_params, request_id)
26
27 async def stream_results() -> AsyncGenerator[bytes, None]:
28 async for request_output in results_generator:
@@ -35,17 +37,24 @@ async def generate_stream(request: Request) -> StreamingResponse:
37 }
38 yield (json.dumps(ret) + "\0").encode("utf-8")
39
38 - return StreamingResponse(stream_results())
40 + async def abort_request() -> None:
41 + await server.abort(request_id)
42 +
43 + background_tasks = BackgroundTasks()
44 + # Abort the request if the client disconnects.
45 + background_tasks.add_task(abort_request)
46 + return StreamingResponse(stream_results(), background=background_tasks)
47
48
49 if __name__ == "__main__":
50 parser = argparse.ArgumentParser()
51 parser.add_argument("--host", type=str, default="localhost")
52 parser.add_argument("--port", type=int, default=8001)
45 - parser = ServerArgs.add_cli_args(parser)
53 + parser = AsyncServerArgs.add_cli_args(parser)
54 args = parser.parse_args()
55
48 - server_args = ServerArgs.from_cli_args(args)
56 + server_args = AsyncServerArgs.from_cli_args(args)
57 server = AsyncLLMServer.from_server_args(server_args)
58
51 - uvicorn.run(app, host=args.host, port=args.port, log_level="info")
59 + uvicorn.run(app, host=args.host, port=args.port, log_level="debug",
60 + timeout_keep_alive=TIMEOUT_KEEP_ALIVE)
cacheflow/sequence.py
+9 -1
@@ -12,12 +12,14 @@ class SequenceStatus(enum.Enum):
12 SWAPPED = enum.auto()
13 FINISHED_STOPPED = enum.auto()
14 FINISHED_LENGTH_CAPPED = enum.auto()
15 + FINISHED_ABORTED = enum.auto()
16
17 @staticmethod
18 def is_finished(status: "SequenceStatus") -> bool:
19 return status in [
20 SequenceStatus.FINISHED_STOPPED,
21 SequenceStatus.FINISHED_LENGTH_CAPPED,
22 + SequenceStatus.FINISHED_ABORTED,
23 ]
24
25 @staticmethod
@@ -26,10 +28,13 @@ class SequenceStatus(enum.Enum):
28 finish_reason = "stop"
29 elif status == SequenceStatus.FINISHED_LENGTH_CAPPED:
30 finish_reason = "length"
31 + elif status == SequenceStatus.FINISHED_ABORTED:
32 + finish_reason = "abort"
33 else:
34 finish_reason = None
35 return finish_reason
36
37 +
38 class SequenceData:
39
40 def __init__(
@@ -137,6 +142,9 @@ class Sequence:
142 def get_cumulative_logprob(self) -> float:
143 return self.data.cumulative_logprob
144
145 + def is_finished(self) -> bool:
146 + return SequenceStatus.is_finished(self.status)
147 +
148 def fork(self, child_seq: 'Sequence') -> None:
149 child_seq.logical_token_blocks = copy.deepcopy(self.logical_token_blocks)
150 child_seq.output_logprobs = copy.deepcopy(self.output_logprobs)
@@ -182,7 +190,7 @@ class SequenceGroup:
190 raise ValueError(f'Sequence {seq_id} not found.')
191
192 def is_finished(self) -> bool:
185 - return all(SequenceStatus.is_finished(seq.status) for seq in self.seqs)
193 + return all(seq.is_finished() for seq in self.seqs)
194
195 def __repr__(self) -> str:
196 return (f"SequenceGroup(request_id={self.request_id}, "
cacheflow/server/arg_utils.py
+72 -59
@@ -15,7 +15,7 @@ class ServerArgs:
15 use_dummy_weights: bool = False
16 dtype: str = "default"
17 seed: int = 0
18 - use_ray: bool = False
18 + worker_use_ray: bool = False
19 pipeline_parallel_size: int = 1
20 tensor_parallel_size: int = 1
21 block_size: int = 16
@@ -32,7 +32,63 @@ class ServerArgs:
32 def add_cli_args(
33 parser: argparse.ArgumentParser,
34 ) -> argparse.ArgumentParser:
35 - return _add_server_arguments(parser)
35 + """Shared CLI arguments for CacheFlow servers."""
36 + # Model arguments
37 + parser.add_argument('--model', type=str, default='facebook/opt-125m',
38 + help='name or path of the huggingface model to use')
39 + parser.add_argument('--download-dir', type=str,
40 + default=ServerArgs.download_dir,
41 + help='directory to download and load the weights, '
42 + 'default to the default cache dir of '
43 + 'huggingface')
44 + parser.add_argument('--use-np-weights', action='store_true',
45 + help='save a numpy copy of model weights for '
46 + 'faster loading. This can increase the disk '
47 + 'usage by up to 2x.')
48 + parser.add_argument('--use-dummy-weights', action='store_true',
49 + help='use dummy values for model weights')
50 + # TODO(woosuk): Support FP32.
51 + parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
52 + choices=['default', 'half', 'bfloat16'],
53 + help='data type for model weights and activations. '
54 + 'The "default" option will use FP16 precision '
55 + 'for FP32 and FP16 models, and BF16 precision '
56 + 'for BF16 models.')
57 + # Parallel arguments
58 + parser.add_argument('--worker-use-ray', action='store_true',
59 + help='use Ray for distributed serving, will be '
60 + 'automatically set when using more than 1 GPU')
61 + parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
62 + default=ServerArgs.pipeline_parallel_size,
63 + help='number of pipeline stages')
64 + parser.add_argument('--tensor-parallel-size', '-tp', type=int,
65 + default=ServerArgs.tensor_parallel_size,
66 + help='number of tensor parallel replicas')
67 + # KV cache arguments
68 + parser.add_argument('--block-size', type=int,
69 + default=ServerArgs.block_size,
70 + choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
71 + help='token block size')
72 + # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
73 + parser.add_argument('--seed', type=int, default=ServerArgs.seed,
74 + help='random seed')
75 + parser.add_argument('--swap-space', type=int,
76 + default=ServerArgs.swap_space,
77 + help='CPU swap space size (GiB) per GPU')
78 + parser.add_argument('--gpu-memory-utilization', type=float,
79 + default=ServerArgs.gpu_memory_utilization,
80 + help='the percentage of GPU memory to be used for'
81 + 'the model executor')
82 + parser.add_argument('--max-num-batched-tokens', type=int,
83 + default=ServerArgs.max_num_batched_tokens,
84 + help='maximum number of batched tokens per '
85 + 'iteration')
86 + parser.add_argument('--max-num-seqs', type=int,
87 + default=ServerArgs.max_num_seqs,
88 + help='maximum number of sequences per iteration')
89 + parser.add_argument('--disable-log-stats', action='store_true',
90 + help='disable logging statistics')
91 + return parser
92
93 @classmethod
94 def from_cli_args(cls, args: argparse.Namespace) -> "ServerArgs":
@@ -53,65 +109,22 @@ class ServerArgs:
109 self.swap_space)
110 parallel_config = ParallelConfig(self.pipeline_parallel_size,
111 self.tensor_parallel_size,
56 - self.use_ray)
112 + self.worker_use_ray)
113 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
114 self.max_num_seqs)
115 return model_config, cache_config, parallel_config, scheduler_config
116
117
62 -def _add_server_arguments(
63 - parser: argparse.ArgumentParser,
64 -)-> argparse.ArgumentParser:
65 - """Shared CLI arguments for CacheFlow servers."""
66 - # Model arguments
67 - parser.add_argument('--model', type=str, default='facebook/opt-125m',
68 - help='name or path of the huggingface model to use')
69 - parser.add_argument('--download-dir', type=str,
70 - default=ServerArgs.download_dir,
71 - help='directory to download and load the weights, '
72 - 'default to the default cache dir of huggingface')
73 - parser.add_argument('--use-np-weights', action='store_true',
74 - help='save a numpy copy of model weights for faster '
75 - 'loading. This can increase the disk usage by up '
76 - 'to 2x.')
77 - parser.add_argument('--use-dummy-weights', action='store_true',
78 - help='use dummy values for model weights')
79 - # TODO(woosuk): Support FP32.
80 - parser.add_argument('--dtype', type=str, default=ServerArgs.dtype,
81 - choices=['default', 'half', 'bfloat16'],
82 - help=('data type for model weights and activations. '
83 - 'The "default" option will use FP16 precision '
84 - 'for FP32 and FP16 models, and BF16 precision '
85 - 'for BF16 models.'))
86 - # Parallel arguments
87 - parser.add_argument('--use-ray', action='store_true',
88 - help='use Ray for distributed serving, will be '
89 - 'automatically set when using more than 1 GPU')
90 - parser.add_argument('--pipeline-parallel-size', '-pp', type=int,
91 - default=ServerArgs.pipeline_parallel_size,
92 - help='number of pipeline stages')
93 - parser.add_argument('--tensor-parallel-size', '-tp', type=int,
94 - default=ServerArgs.tensor_parallel_size,
95 - help='number of tensor parallel replicas')
96 - # KV cache arguments
97 - parser.add_argument('--block-size', type=int, default=ServerArgs.block_size,
98 - choices=[1, 2, 4, 8, 16, 32, 64, 128, 256],
99 - help='token block size')
100 - # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
101 - parser.add_argument('--seed', type=int, default=ServerArgs.seed,
102 - help='random seed')
103 - parser.add_argument('--swap-space', type=int, default=ServerArgs.swap_space,
104 - help='CPU swap space size (GiB) per GPU')
105 - parser.add_argument('--gpu-memory-utilization', type=float,
106 - default=ServerArgs.gpu_memory_utilization,
107 - help='the percentage of GPU memory to be used for the '
108 - 'model executor')
109 - parser.add_argument('--max-num-batched-tokens', type=int,
110 - default=ServerArgs.max_num_batched_tokens,
111 - help='maximum number of batched tokens per iteration')
112 - parser.add_argument('--max-num-seqs', type=int,
113 - default=ServerArgs.max_num_seqs,
114 - help='maximum number of sequences per iteration')
115 - parser.add_argument('--disable-log-stats', action='store_true',
116 - help='disable logging statistics')
117 - return parser
118 +@dataclass
119 +class AsyncServerArgs(ServerArgs):
120 + server_use_ray: bool = False
121 +
122 + @staticmethod
123 + def add_cli_args(
124 + parser: argparse.ArgumentParser,
125 + ) -> argparse.ArgumentParser:
126 + parser = ServerArgs.add_cli_args(parser)
127 + parser.add_argument('--server-use-ray', action='store_true',
128 + help='use Ray to start the LLM server in a '
129 + 'separate process as the web server process.')
130 + return parser
cacheflow/server/async_llm_server.py
+71 -22
@@ -2,37 +2,52 @@ import asyncio
2 import time
3 from typing import Dict, Optional
4
5 -import ray
6 -
5 +from cacheflow.logger import init_logger
6 from cacheflow.outputs import RequestOutput
7 from cacheflow.sampling_params import SamplingParams
9 -from cacheflow.server.arg_utils import ServerArgs
8 +from cacheflow.server.arg_utils import AsyncServerArgs
9 from cacheflow.server.llm_server import LLMServer
11 -from cacheflow.server.ray_utils import initialize_cluster
12 -from cacheflow.utils import random_uuid
10 +from cacheflow.server.ray_utils import ray, initialize_cluster
11 +
12 +logger = init_logger(__name__)
13
14 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
15
16
17 class AsyncLLMServer:
18
19 - def __init__(self, server_use_ray: bool, *args, **kwargs) -> None:
20 - if server_use_ray:
21 - remote_server_class = ray.remote(num_cpus=0)(LLMServer)
19 + def __init__(self, worker_use_ray: bool, server_use_ray: bool,
20 + *args, **kwargs) -> None:
21 + self.worker_use_ray = worker_use_ray
22 + self.server_use_ray = server_use_ray
23 + if not self.server_use_ray:
24 + server_class = LLMServer
25 + elif self.worker_use_ray:
26 + server_class = ray.remote(num_cpus=0)(LLMServer).remote
27 else:
23 - remote_server_class = ray.remote(num_gpus=1)(LLMServer)
24 - self.server = remote_server_class.remote(*args, **kwargs)
25 -
28 + server_class = ray.remote(num_gpus=1)(LLMServer).remote
29 + self.server = server_class(*args, **kwargs)
30 # Request id -> request output.
31 self.request_outputs: Dict[str, RequestOutput] = {}
32 # Request id -> event to notify that there is new output.
33 self.request_events: Dict[str, asyncio.Event] = {}
34 self.is_server_running = False
35 + self.kicking_request_id: Optional[str] = None
36
32 - async def server_step(self):
37 + async def server_step(self, kicking_request_id: Optional[str] = None):
38 self.is_server_running = True
34 - request_outputs = await self.server.step.remote()
39 + self.kicking_request_id = kicking_request_id
40 + if self.server_use_ray:
41 + request_outputs = await self.server.step.remote()
42 + else:
43 + # Yield to the event loop to allow other coroutines to run
44 + # while is_server_running is True. This let the server to add new
45 + # requests into the queue.
46 + await asyncio.sleep(0)
47 + request_outputs = self.server.step()
48 self.is_server_running = False
49 + self.kicking_request_id = None
50 +
51 # Notify the waiting coroutines that there are new outputs ready.
52 for request_output in request_outputs:
53 request_id = request_output.request_id
@@ -40,20 +55,26 @@ class AsyncLLMServer:
55 self.request_events[request_id].set()
56
57 async def generate(self, prompt: str, sampling_params: SamplingParams,
43 - request_id: Optional[str] = None) -> RequestOutput:
58 + request_id: str) -> RequestOutput:
59 # Preprocess the request.
60 arrival_time = time.time()
61
62 # Create an event to notify us that there is new output from the
63 # cacheflow server.
49 - if request_id is None:
50 - request_id = random_uuid()
64 request_event = asyncio.Event()
65 self.request_events[request_id] = request_event
66
67 + logger.info(f"Received request {request_id}: "
68 + f"prompt: {prompt!r}, "
69 + f"sampling params: {sampling_params}.")
70 +
71 # Add the request into the cacheflow server's waiting queue.
55 - await self.server.add_request.remote(
56 - request_id, prompt, sampling_params, arrival_time=arrival_time)
72 + if self.server_use_ray:
73 + await self.server.add_request.remote(
74 + request_id, prompt, sampling_params, arrival_time=arrival_time)
75 + else:
76 + self.server.add_request(
77 + request_id, prompt, sampling_params, arrival_time=arrival_time)
78
79 # The cacheflow server does not have a background loop that keeps
80 # processing incoming requests. Therefore, we need to keep kicking
@@ -61,7 +82,7 @@ class AsyncLLMServer:
82 while True:
83 # Kick the server if the server is not running.
84 if not self.is_server_running:
64 - await self.server_step()
85 + await self.server_step(request_id)
86
87 # Wait for new output. The group_event will be set in server_step
88 # when there is new output available for the sequence group.
@@ -80,6 +101,8 @@ class AsyncLLMServer:
101
102 # Once finished, release the resources of the sequence group.
103 if request_output.finished():
104 + logger.info(f"Finished request {request_id}.")
105 +
106 del self.request_outputs[request_id]
107 del self.request_events[request_id]
108 # Kick the server if the server is not running. This is to
@@ -89,15 +112,41 @@ class AsyncLLMServer:
112 await self.server_step()
113 break
114
115 + async def abort(self, request_id: str) -> None:
116 + if request_id not in self.request_events:
117 + # The request has already finished or been aborted.
118 + return
119 +
120 + logger.info(f"Aborted request {request_id}.")
121 +
122 + if self.server_use_ray:
123 + await self.server.abort_request.remote(request_id)
124 + else:
125 + self.server.abort_request(request_id)
126 +
127 + if request_id in self.request_events:
128 + del self.request_events[request_id]
129 + if request_id in self.request_outputs:
130 + del self.request_outputs[request_id]
131 +
132 + # To prevent deadlock when a request is aborted while the server is
133 + # running.
134 + if self.kicking_request_id == request_id:
135 + self.is_server_running = False
136 + self.kicking_request_id = None
137 +
138 @classmethod
93 - def from_server_args(cls, server_args: ServerArgs) -> "AsyncLLMServer":
139 + def from_server_args(cls, server_args: AsyncServerArgs) -> "AsyncLLMServer":
140 # Create the server configs.
141 server_configs = server_args.create_server_configs()
142 parallel_config = server_configs[2]
143 # Initialize the cluster.
98 - distributed_init_method, devices = initialize_cluster(parallel_config)
144 + distributed_init_method, devices = initialize_cluster(
145 + parallel_config, server_args.server_use_ray)
146 # Create the LLM server.
100 - server = cls(server_args.use_ray, *server_configs,
147 + server = cls(server_args.worker_use_ray,
148 + server_args.server_use_ray,
149 + *server_configs,
150 distributed_init_method, devices,
151 log_stats=not server_args.disable_log_stats)
152 return server
cacheflow/server/llm_server.py
+7 -9
@@ -1,11 +1,6 @@
1 import time
2 from typing import Any, List, Optional
3
4 -try:
5 - import ray
6 -except ImportError:
7 - ray = None
8 -
4 from cacheflow.config import (CacheConfig, ModelConfig, ParallelConfig,
5 SchedulerConfig)
6 from cacheflow.core.scheduler import Scheduler
@@ -13,7 +8,7 @@ from cacheflow.logger import init_logger
8 from cacheflow.outputs import RequestOutput
9 from cacheflow.sampling_params import SamplingParams
10 from cacheflow.server.arg_utils import ServerArgs
16 -from cacheflow.server.ray_utils import initialize_cluster
11 +from cacheflow.server.ray_utils import ray, initialize_cluster
12 from cacheflow.server.tokenizer_utils import (get_tokenizer,
13 detokenize_incrementally)
14 from cacheflow.sequence import Sequence, SequenceGroup, SequenceStatus
@@ -62,7 +57,7 @@ class LLMServer:
57 assert len(stage_devices) == 1, "Only support one stage for now."
58 for rank, node_resource, _ in stage_devices[0]:
59 worker_cls = Worker
65 - if self.parallel_config.use_ray:
60 + if self.parallel_config.worker_use_ray:
61 worker_cls = ray.remote(
62 num_cpus=0,
63 num_gpus=1,
@@ -152,6 +147,9 @@ class LLMServer:
147 # Add the sequence group to the scheduler.
148 self.scheduler.add_seq_group(seq_group)
149
150 + def abort_request(self, request_id: str) -> None:
151 + self.scheduler.abort_seq_group(request_id)
152 +
153 def get_num_unfinished_requests(self) -> int:
154 return self.scheduler.get_num_unfinished_seq_groups()
155
@@ -243,13 +241,13 @@ class LLMServer:
241 all_outputs = []
242 for worker in self.workers:
243 executor = getattr(worker, method)
246 - if self.parallel_config.use_ray:
244 + if self.parallel_config.worker_use_ray:
245 executor = executor.remote
246
247 output = executor(*args, **kwargs)
248 all_outputs.append(output)
249
252 - if self.parallel_config.use_ray:
250 + if self.parallel_config.worker_use_ray:
251 all_outputs = ray.get(all_outputs)
252
253 if get_all_outputs:
cacheflow/server/ray_utils.py
+10 -8
@@ -13,9 +13,18 @@ DeviceID = Tuple[int, Optional[str], int] # rank, node resource (node IP), devi
13
14 def initialize_cluster(
15 parallel_config: ParallelConfig,
16 + server_use_ray: bool = False,
17 address: Optional[str] = None,
18 ) -> Tuple[str, List[List[DeviceID]]]:
18 - if not parallel_config.use_ray:
19 + if parallel_config.worker_use_ray or server_use_ray:
20 + if ray is None:
21 + raise ImportError(
22 + "Ray is not installed. Please install Ray to use distributed "
23 + "serving.")
24 + # Connect to a ray cluster.
25 + ray.init(address=address)
26 +
27 + if not parallel_config.worker_use_ray:
28 # Initialize cluster locally.
29 port = random.randint(10000, 20000)
30 # We need to setup the distributed init method to make sure
@@ -24,13 +33,6 @@ def initialize_cluster(
33 all_stage_devices = [[(0, None, 0)]]
34 return distributed_init_method, all_stage_devices
35
27 - if ray is None:
28 - raise ImportError(
29 - "Ray is not installed. Please install Ray to use distributed "
30 - "serving.")
31 - # Connect to a ray cluster.
32 - ray.init(address=address)
33 -
36 # Assume we have a uniform cluster that each node has the same number of
37 # GPUs for now.
38 valid_node_resources = []