@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4db9f53c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Benchmark] Add `--async-engine` option to benchmark_throughput.py (#7964)

Nick Hill committed Sep 3, 2024 at 17:57 UTC d4db9f53c8a50a2b0788cf1e03b5b91f20de4313
3 files changed +143 -19
benchmarks/benchmark_throughput.py
+109 -4
@@ -6,13 +6,16 @@ import time
6 from typing import List, Optional, Tuple
7
8 import torch
9 +import uvloop
10 from tqdm import tqdm
11 from transformers import (AutoModelForCausalLM, AutoTokenizer,
12 PreTrainedTokenizerBase)
13
13 -from vllm.engine.arg_utils import EngineArgs
14 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
15 +from vllm.entrypoints.openai.api_server import (
16 + build_async_engine_client_from_engine_args)
17 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15 -from vllm.utils import FlexibleArgumentParser
18 +from vllm.utils import FlexibleArgumentParser, merge_async_iterators
19
20
21 def sample_requests(
@@ -135,6 +138,93 @@ def run_vllm(
138 return end - start
139
140
141 +async def run_vllm_async(
142 + requests: List[Tuple[str, int, int]],
143 + model: str,
144 + tokenizer: str,
145 + quantization: Optional[str],
146 + tensor_parallel_size: int,
147 + seed: int,
148 + n: int,
149 + use_beam_search: bool,
150 + trust_remote_code: bool,
151 + dtype: str,
152 + max_model_len: Optional[int],
153 + enforce_eager: bool,
154 + kv_cache_dtype: str,
155 + quantization_param_path: Optional[str],
156 + device: str,
157 + enable_prefix_caching: bool,
158 + enable_chunked_prefill: bool,
159 + max_num_batched_tokens: int,
160 + distributed_executor_backend: Optional[str],
161 + gpu_memory_utilization: float = 0.9,
162 + num_scheduler_steps: int = 1,
163 + use_v2_block_manager: bool = False,
164 + download_dir: Optional[str] = None,
165 + load_format: str = EngineArgs.load_format,
166 + disable_async_output_proc: bool = False,
167 + disable_frontend_multiprocessing: bool = False,
168 +) -> float:
169 + from vllm import SamplingParams
170 + engine_args = AsyncEngineArgs(
171 + model=model,
172 + tokenizer=tokenizer,
173 + quantization=quantization,
174 + tensor_parallel_size=tensor_parallel_size,
175 + seed=seed,
176 + trust_remote_code=trust_remote_code,
177 + dtype=dtype,
178 + max_model_len=max_model_len,
179 + gpu_memory_utilization=gpu_memory_utilization,
180 + enforce_eager=enforce_eager,
181 + kv_cache_dtype=kv_cache_dtype,
182 + quantization_param_path=quantization_param_path,
183 + device=device,
184 + enable_prefix_caching=enable_prefix_caching,
185 + download_dir=download_dir,
186 + enable_chunked_prefill=enable_chunked_prefill,
187 + max_num_batched_tokens=max_num_batched_tokens,
188 + distributed_executor_backend=distributed_executor_backend,
189 + load_format=load_format,
190 + num_scheduler_steps=num_scheduler_steps,
191 + use_v2_block_manager=use_v2_block_manager,
192 + disable_async_output_proc=disable_async_output_proc,
193 + worker_use_ray=False,
194 + engine_use_ray=False,
195 + disable_log_requests=True,
196 + )
197 +
198 + async with build_async_engine_client_from_engine_args(
199 + engine_args, disable_frontend_multiprocessing) as llm:
200 +
201 + # Add the requests to the engine.
202 + prompts: List[str] = []
203 + sampling_params: List[SamplingParams] = []
204 + for prompt, _, output_len in requests:
205 + prompts.append(prompt)
206 + sampling_params.append(
207 + SamplingParams(
208 + n=n,
209 + temperature=0.0 if use_beam_search else 1.0,
210 + top_p=1.0,
211 + use_beam_search=use_beam_search,
212 + ignore_eos=True,
213 + max_tokens=output_len,
214 + ))
215 +
216 + generators = []
217 + start = time.perf_counter()
218 + for i, (prompt, sp) in enumerate(zip(prompts, sampling_params)):
219 + generator = llm.generate(prompt, sp, request_id=f"test{i}")
220 + generators.append(generator)
221 + all_gens = merge_async_iterators(*generators)
222 + async for i, res in all_gens:
223 + pass
224 + end = time.perf_counter()
225 + return end - start
226 +
227 +
228 def run_hf(
229 requests: List[Tuple[str, int, int]],
230 model: str,
@@ -230,7 +320,7 @@ def main(args: argparse.Namespace):
320 args.output_len)
321
322 if args.backend == "vllm":
233 - elapsed_time = run_vllm(
323 + run_args = [
324 requests, args.model, args.tokenizer, args.quantization,
325 args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
326 args.trust_remote_code, args.dtype, args.max_model_len,
@@ -240,7 +330,14 @@ def main(args: argparse.Namespace):
330 args.max_num_batched_tokens, args.distributed_executor_backend,
331 args.gpu_memory_utilization, args.num_scheduler_steps,
332 args.use_v2_block_manager, args.download_dir, args.load_format,
243 - args.disable_async_output_proc)
333 + args.disable_async_output_proc
334 + ]
335 +
336 + if args.async_engine:
337 + run_args.append(args.disable_frontend_multiprocessing)
338 + elapsed_time = uvloop.run(run_vllm_async(*run_args))
339 + else:
340 + elapsed_time = run_vllm(*run_args)
341 elif args.backend == "hf":
342 assert args.tensor_parallel_size == 1
343 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -426,6 +523,14 @@ if __name__ == "__main__":
523 action='store_true',
524 default=False,
525 help="Disable async output processor for vLLM backend.")
526 + parser.add_argument("--async-engine",
527 + action='store_true',
528 + default=False,
529 + help="Use vLLM async engine rather than LLM class.")
530 + parser.add_argument("--disable-frontend-multiprocessing",
531 + action='store_true',
532 + default=False,
533 + help="Disable decoupled async engine frontend.")
534 args = parser.parse_args()
535 if args.tokenizer is None:
536 args.tokenizer = args.model
vllm/entrypoints/openai/api_server.py
+30 -15
@@ -67,7 +67,7 @@ _running_tasks: Set[asyncio.Task] = set()
67
68
69 def model_is_embedding(model_name: str, trust_remote_code: bool,
70 - quantization: str) -> bool:
70 + quantization: Optional[str]) -> bool:
71 return ModelConfig(model=model_name,
72 tokenizer=model_name,
73 tokenizer_mode="auto",
@@ -96,13 +96,6 @@ async def lifespan(app: FastAPI):
96 @asynccontextmanager
97 async def build_async_engine_client(
98 args: Namespace) -> AsyncIterator[Optional[AsyncEngineClient]]:
99 - """
100 - Create AsyncEngineClient, either:
101 - - in-process using the AsyncLLMEngine Directly
102 - - multiprocess using AsyncLLMEngine RPC
103 -
104 - Returns the Client or None if the creation failed.
105 - """
99
100 # Context manager to handle async_engine_client lifecycle
101 # Ensures everything is shutdown and cleaned up on error/exit
@@ -112,14 +105,37 @@ async def build_async_engine_client(
105 # Backend itself still global for the silly lil' health handler
106 global async_engine_client
107
108 + async with build_async_engine_client_from_engine_args(
109 + engine_args, args.disable_frontend_multiprocessing) as engine:
110 +
111 + async_engine_client = engine # type: ignore[assignment]
112 + yield engine
113 +
114 +
115 +@asynccontextmanager
116 +async def build_async_engine_client_from_engine_args(
117 + engine_args: AsyncEngineArgs,
118 + disable_frontend_multiprocessing: bool = False,
119 +) -> AsyncIterator[Optional[AsyncEngineClient]]:
120 + """
121 + Create AsyncEngineClient, either:
122 + - in-process using the AsyncLLMEngine Directly
123 + - multiprocess using AsyncLLMEngine RPC
124 +
125 + Returns the Client or None if the creation failed.
126 + """
127 +
128 # If manually triggered or embedding model, use AsyncLLMEngine in process.
129 # TODO: support embedding model via RPC.
117 - if (model_is_embedding(args.model, args.trust_remote_code,
118 - args.quantization)
119 - or args.disable_frontend_multiprocessing):
120 - async_engine_client = AsyncLLMEngine.from_engine_args(
130 + if (model_is_embedding(engine_args.model, engine_args.trust_remote_code,
131 + engine_args.quantization)
132 + or disable_frontend_multiprocessing):
133 + engine_client = AsyncLLMEngine.from_engine_args(
134 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
122 - yield async_engine_client
135 + try:
136 + yield engine_client
137 + finally:
138 + engine_client.shutdown_background_loop()
139 return
140
141 # Otherwise, use the multiprocessing AsyncLLMEngine.
@@ -148,7 +164,6 @@ async def build_async_engine_client(
164 # NOTE: Actually, this is not true yet. We still need to support
165 # embedding models via RPC (see TODO above)
166 rpc_client = AsyncEngineRPCClient(rpc_path)
151 - async_engine_client = rpc_client # type: ignore
167
168 # Start RPCServer in separate process (holds the AsyncLLMEngine).
169 context = multiprocessing.get_context("spawn")
@@ -174,7 +189,7 @@ async def build_async_engine_client(
189 yield None
190 return
191
177 - yield async_engine_client
192 + yield rpc_client # type: ignore[misc]
193 finally:
194 # Ensure rpc server process was terminated
195 rpc_server_process.terminate()
vllm/entrypoints/openai/rpc/client.py
+4
@@ -7,6 +7,7 @@ from uuid import uuid4
7 import cloudpickle
8 import zmq
9 import zmq.asyncio
10 +from zmq import Frame # type: ignore[attr-defined]
11 from zmq.asyncio import Socket
12
13 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
@@ -214,6 +215,7 @@ class AsyncEngineRPCClient:
215
216 # Await the data from the Server.
217 frame = await socket.recv(copy=False)
218 + assert isinstance(frame, Frame)
219 data = pickle.loads(frame.buffer)
220
221 if isinstance(data, Exception):
@@ -247,6 +249,7 @@ class AsyncEngineRPCClient:
249 f"{self._data_timeout} ms")
250
251 frame = await socket.recv(copy=False)
252 + assert isinstance(frame, Frame)
253 return pickle.loads(frame.buffer)
254
255 # Make a new socket connection.
@@ -395,6 +398,7 @@ class AsyncEngineRPCClient:
398 # Stream back the results from the RPC Server.
399 while not finished:
400 message = await socket.recv(copy=False)
401 + assert isinstance(message, Frame)
402 request_output = pickle.loads(message.buffer)
403
404 if isinstance(request_output, Exception):