@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 45b6ef6513
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

feat(benchmarks): Add Prefix Caching Benchmark to Serving Benchmark (#3277)

Roger Wang committed Mar 27, 2024 at 13:39 UTC 45b6ef651387100c24d671fb485e7a6e208216f6
6 files changed +897 -155
.buildkite/run-benchmarks.sh
+3 -2
@@ -23,8 +23,9 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/r
23 # wait for server to start, timeout after 600 seconds
24 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
25 python3 benchmarks/benchmark_serving.py \
26 - --backend openai \
27 - --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
26 + --backend vllm \
27 + --dataset-name sharegpt \
28 + --dataset-path ./ShareGPT_V3_unfiltered_cleaned_split.json \
29 --model meta-llama/Llama-2-7b-chat-hf \
30 --num-prompts 20 \
31 --endpoint /v1/completions \
benchmarks/backend_request_func.py
+121 -99
@@ -1,8 +1,10 @@
1 import json
2 import os
3 +import sys
4 import time
4 -from dataclasses import dataclass
5 -from typing import Optional
5 +import traceback
6 +from dataclasses import dataclass, field
7 +from typing import List, Optional
8
9 import aiohttp
10 from tqdm.asyncio import tqdm
@@ -26,8 +28,11 @@ class RequestFuncOutput:
28 generated_text: str = ""
29 success: bool = False
30 latency: float = 0
29 - ttft: float = 0
31 + ttft: float = 0 # Time to first token
32 + itl: List[float] = field(
33 + default_factory=list) # List of inter-token latencies
34 prompt_len: int = 0
35 + error: str = ""
36
37
38 async def async_request_tgi(
@@ -55,71 +60,38 @@ async def async_request_tgi(
60
61 ttft = 0
62 st = time.perf_counter()
63 + most_recent_timestamp = st
64 try:
65 async with session.post(url=api_url, json=payload) as response:
66 if response.status == 200:
61 - async for data in response.content.iter_any():
62 - if ttft == 0:
63 - ttft = time.perf_counter() - st
64 - output.ttft = ttft
65 - output.latency = time.perf_counter() - st
66 -
67 - body = remove_prefix(data.decode("utf-8"), "data:")
68 - output.generated_text = json.loads(body)["generated_text"]
69 - output.success = True
70 - else:
71 - output.success = False
72 - except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
73 - output.success = False
74 -
75 - if pbar:
76 - pbar.update(1)
77 - return output
78 -
79 -
80 -async def async_request_vllm(
81 - request_func_input: RequestFuncInput,
82 - pbar: Optional[tqdm] = None,
83 -) -> RequestFuncOutput:
84 - api_url = request_func_input.api_url
85 - assert api_url.endswith("generate")
67 + async for chunk in response.content:
68 + chunk = chunk.strip()
69 + if not chunk:
70 + continue
71
87 - async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
88 - payload = {
89 - "prompt": request_func_input.prompt,
90 - "n": 1,
91 - "best_of": request_func_input.best_of,
92 - "use_beam_search": request_func_input.use_beam_search,
93 - "temperature": 0.0 if request_func_input.use_beam_search else 1.0,
94 - "top_p": 1.0,
95 - "max_tokens": request_func_input.output_len,
96 - "ignore_eos": True,
97 - "stream": True,
98 - }
99 - output = RequestFuncOutput()
100 - output.prompt_len = request_func_input.prompt_len
72 + chunk = remove_prefix(chunk.decode("utf-8"), "data:")
73
102 - ttft = 0
103 - st = time.perf_counter()
104 - try:
105 - async with session.post(url=api_url, json=payload) as response:
106 - if response.status == 200:
107 - async for data in response.content.iter_any():
74 + data = json.loads(chunk)
75 + timestamp = time.perf_counter()
76 + # First token
77 if ttft == 0:
78 ttft = time.perf_counter() - st
79 output.ttft = ttft
111 - output.latency = time.perf_counter() - st
80
113 - # When streaming, '\0' is appended to the end of response.
114 - body = data.decode("utf-8").strip("\0")
115 - output.generated_text = json.loads(
116 - body)["text"][0][len(request_func_input.prompt):]
117 - output.success = True
81 + # Decoding phase
82 + else:
83 + output.itl.append(timestamp -
84 + most_recent_timestamp)
85
119 - else:
120 - output.success = False
121 - except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
86 + most_recent_timestamp = timestamp
87 +
88 + output.latency = most_recent_timestamp - st
89 + output.success = True
90 + output.generated_text = data["generated_text"]
91 + except Exception:
92 output.success = False
93 + exc_info = sys.exc_info()
94 + output.error = "".join(traceback.format_exception(*exc_info))
95
96 if pbar:
97 pbar.update(1)
@@ -146,26 +118,45 @@ async def async_request_trt_llm(
118 }
119 output = RequestFuncOutput()
120 output.prompt_len = request_func_input.prompt_len
149 - ttft = 0
121
122 + ttft = 0
123 st = time.perf_counter()
124 + most_recent_timestamp = st
125 try:
153 - async with session.post(url=api_url, json=payload) as resp:
154 - if resp.status == 200:
155 - async for data in resp.content.iter_any():
126 + async with session.post(url=api_url, json=payload) as response:
127 + if response.status == 200:
128 + async for chunk in response.content:
129 + chunk = chunk.strip()
130 + if not chunk:
131 + continue
132 +
133 + chunk = remove_prefix(chunk.decode("utf-8"), "data:")
134 +
135 + data = json.loads(chunk)
136 + timestamp = time.perf_counter()
137 + # First token
138 if ttft == 0:
139 ttft = time.perf_counter() - st
140 output.ttft = ttft
159 - output.latency = time.perf_counter() - st
141
161 - body = remove_prefix(data.decode("utf-8"), "data:")
162 - output.generated_text = json.loads(body)["text_output"]
142 + # Decoding phase
143 + else:
144 + output.itl.append(timestamp -
145 + most_recent_timestamp)
146 +
147 + most_recent_timestamp = timestamp
148 +
149 + output.latency = most_recent_timestamp - st
150 + output.generated_text = json.loads(data)["text_output"]
151 output.success = True
152
153 else:
154 + output.error = response.reason
155 output.success = False
167 - except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
156 + except Exception:
157 output.success = False
158 + exc_info = sys.exc_info()
159 + output.error = "".join(traceback.format_exception(*exc_info))
160
161 if pbar:
162 pbar.update(1)
@@ -181,35 +172,35 @@ async def async_request_deepspeed_mii(
172 assert not request_func_input.use_beam_search
173
174 payload = {
184 - "prompts": request_func_input.prompt,
185 - "max_new_tokens": request_func_input.output_len,
186 - "ignore_eos": True,
187 - "do_sample": True,
188 - "temperature":
189 - 0.01, # deepspeed-mii does not accept 0.0 temperature.
175 + "prompt": request_func_input.prompt,
176 + "max_tokens": request_func_input.output_len,
177 + "temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.
178 "top_p": 1.0,
179 }
180 output = RequestFuncOutput()
181 output.prompt_len = request_func_input.prompt_len
182
195 - # DeepSpeed-MII doesn't support streaming as of Jan 28 2024,
183 + # NOTE: DeepSpeed-MII doesn't support streaming as of Jan 28 2024,
184 # will use 0 as placeholder.
197 - # https://github.com/microsoft/DeepSpeed-MII/pull/311
185 + # See https://github.com/microsoft/DeepSpeed-MII/pull/311
186 output.ttft = 0
187
188 st = time.perf_counter()
189 try:
190 async with session.post(url=request_func_input.api_url,
203 - json=payload) as resp:
204 - if resp.status == 200:
205 - parsed_resp = await resp.json()
191 + json=payload) as response:
192 + if response.status == 200:
193 + parsed_resp = await response.json()
194 output.latency = time.perf_counter() - st
207 - output.generated_text = parsed_resp[0]["generated_text"]
195 + output.generated_text = parsed_resp["text"][0]
196 output.success = True
197 else:
198 + output.error = response.reason
199 output.success = False
211 - except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
200 + except Exception:
201 output.success = False
202 + exc_info = sys.exc_info()
203 + output.error = "".join(traceback.format_exception(*exc_info))
204
205 if pbar:
206 pbar.update(1)
@@ -221,7 +212,9 @@ async def async_request_openai_completions(
212 pbar: Optional[tqdm] = None,
213 ) -> RequestFuncOutput:
214 api_url = request_func_input.api_url
224 - assert api_url.endswith("v1/completions")
215 + assert api_url.endswith(
216 + "v1/completions"
217 + ), "OpenAI Completions API URL must end with 'v1/completions'."
218
219 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
220 assert not request_func_input.use_beam_search
@@ -243,15 +236,12 @@ async def async_request_openai_completions(
236 generated_text = ""
237 ttft = 0
238 st = time.perf_counter()
239 + most_recent_timestamp = st
240 try:
241 async with session.post(url=api_url, json=payload,
242 headers=headers) as response:
243 if response.status == 200:
244 async for chunk in response.content:
251 - if ttft == 0:
252 - ttft = time.perf_counter() - st
253 - output.ttft = ttft
254 -
245 chunk = chunk.strip()
246 if not chunk:
247 continue
@@ -260,16 +250,33 @@ async def async_request_openai_completions(
250 if chunk == "[DONE]":
251 latency = time.perf_counter() - st
252 else:
263 - body = json.loads(chunk)
264 - generated_text += body["choices"][0]["text"]
253 + data = json.loads(chunk)
254 +
255 + if data["choices"][0]["text"]:
256 + timestamp = time.perf_counter()
257 + # First token
258 + if ttft == 0:
259 + ttft = time.perf_counter() - st
260 + output.ttft = ttft
261 +
262 + # Decoding phase
263 + # NOTE: Some completion API might have a last
264 + # usage summary response without a token so we
265 + # do not want to include as inter-token-latency
266 + elif data.get("usage", None) is None:
267 + output.itl.append(timestamp -
268 + most_recent_timestamp)
269 +
270 + most_recent_timestamp = timestamp
271 + generated_text += data["choices"][0]["text"]
272
273 output.generated_text = generated_text
274 output.success = True
275 output.latency = latency
269 - else:
270 - output.success = False
271 - except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
276 + except Exception:
277 output.success = False
278 + exc_info = sys.exc_info()
279 + output.error = "".join(traceback.format_exception(*exc_info))
280
281 if pbar:
282 pbar.update(1)
@@ -283,7 +290,7 @@ async def async_request_openai_chat_completions(
290 api_url = request_func_input.api_url
291 assert api_url.endswith(
292 "v1/chat/completions"
286 - ), "OpenAI Chat API URL must end with 'v1/chat/completions'."
293 + ), "OpenAI Chat Completions API URL must end with 'v1/chat/completions'."
294
295 async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
296 assert not request_func_input.use_beam_search
@@ -301,7 +308,7 @@ async def async_request_openai_chat_completions(
308 }
309 headers = {
310 "Content-Type": "application/json",
304 - "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
311 + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
312 }
313
314 output = RequestFuncOutput()
@@ -310,15 +317,12 @@ async def async_request_openai_chat_completions(
317 generated_text = ""
318 ttft = 0
319 st = time.perf_counter()
320 + most_recent_timestamp = st
321 try:
322 async with session.post(url=api_url, json=payload,
323 headers=headers) as response:
324 if response.status == 200:
325 async for chunk in response.content:
318 - if ttft == 0:
319 - ttft = time.perf_counter() - st
320 - output.ttft = ttft
321 -
326 chunk = chunk.strip()
327 if not chunk:
328 continue
@@ -327,18 +331,35 @@ async def async_request_openai_chat_completions(
331 if chunk == "[DONE]":
332 latency = time.perf_counter() - st
333 else:
330 - body = json.loads(chunk)
331 - if "content" in body["choices"][0]["delta"]:
332 - generated_text += body["choices"][0]["delta"][
334 + timestamp = time.perf_counter()
335 + data = json.loads(chunk)
336 +
337 + if "content" in data["choices"][0]["delta"]:
338 + # First token
339 + if ttft == 0:
340 + ttft = time.perf_counter() - st
341 + output.ttft = ttft
342 +
343 + # Decoding phase
344 + else:
345 + output.itl.append(timestamp -
346 + most_recent_timestamp)
347 +
348 + generated_text += data["choices"][0]["delta"][
349 "content"]
350
351 + most_recent_timestamp = timestamp
352 +
353 output.generated_text = generated_text
354 output.success = True
355 output.latency = latency
356 else:
357 + output.error = response.reason
358 output.success = False
340 - except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
359 + except Exception:
360 output.success = False
361 + exc_info = sys.exc_info()
362 + output.error = "".join(traceback.format_exception(*exc_info))
363
364 if pbar:
365 pbar.update(1)
@@ -355,7 +376,8 @@ def remove_prefix(text: str, prefix: str) -> str:
376
377 ASYNC_REQUEST_FUNCS = {
378 "tgi": async_request_tgi,
358 - "vllm": async_request_vllm,
379 + "vllm": async_request_openai_completions,
380 + "lmdeploy": async_request_openai_completions,
381 "deepspeed-mii": async_request_deepspeed_mii,
382 "openai": async_request_openai_completions,
383 "openai-chat": async_request_openai_chat_completions,
benchmarks/benchmark_serving.py
+252 -51
@@ -1,8 +1,8 @@
1 """Benchmark online serving throughput.
2
3 On the server side, run one of the following commands:
4 - (vLLM backend)
5 - python -m vllm.entrypoints.api_server \
4 + vLLM OpenAI API server
5 + python -m vllm.entrypoints.openai.api_server \
6 --model <your_model> --swap-space 16 \
7 --disable-log-requests
8
@@ -12,14 +12,19 @@ On the server side, run one of the following commands:
12 On the client side, run:
13 python benchmarks/benchmark_serving.py \
14 --backend <backend> \
15 - --model <your_model> --dataset <target_dataset> \
16 - --request-rate <request_rate>
15 + --model <your_model> \
16 + --dataset-name sharegpt \
17 + --dataset-path <path to dataset> \
18 + --request-rate <request_rate> \ # By default <request_rate> is inf
19 + --num-prompts <num_prompts> # By default <num_prompts> is 1000
20 """
21 import argparse
22 import asyncio
23 import json
24 +import os
25 import random
26 import time
27 +import warnings
28 from dataclasses import dataclass
29 from datetime import datetime
30 from typing import AsyncGenerator, List, Tuple
@@ -49,7 +54,7 @@ class BenchmarkMetrics:
54 p99_tpot_ms: float
55
56
52 -def sample_requests(
57 +def sample_sharegpt_requests(
58 dataset_path: str,
59 num_requests: int,
60 tokenizer: PreTrainedTokenizerBase,
@@ -97,6 +102,73 @@ def sample_requests(
102 return sampled_requests
103
104
105 +def sample_sonnet_requests(
106 + dataset_path: str,
107 + num_requests: int,
108 + input_len: int,
109 + output_len: int,
110 + prefix_len: int,
111 + tokenizer: PreTrainedTokenizerBase,
112 +) -> List[Tuple[str, str, int, int]]:
113 + assert input_len > prefix_len, "input_len must be greater than prefix_len."
114 +
115 + # Load the dataset.
116 + with open(dataset_path) as f:
117 + poem_lines = f.readlines()
118 +
119 + # Tokenize the poem lines.
120 + poem_token_ids = tokenizer(poem_lines).input_ids
121 + average_poem_len = sum(
122 + len(token_ids) for token_ids in poem_token_ids) / len(poem_token_ids)
123 +
124 + # Base prefix for all requests.
125 + base_prompt = "Pick as many lines as you can from these poem lines:\n"
126 + base_message = [{
127 + "role": "user",
128 + "content": base_prompt,
129 + }]
130 + base_prompt_formatted = tokenizer.apply_chat_template(
131 + base_message, add_generation_prompt=True, tokenize=False)
132 + base_prompt_offset = len(tokenizer(base_prompt_formatted).input_ids)
133 +
134 + assert (input_len > base_prompt_offset
135 + ), f"Please set 'args.input-len' higher than {base_prompt_offset}."
136 + num_input_lines = round(
137 + (input_len - base_prompt_offset) / average_poem_len)
138 +
139 + # First approximately `prefix_len` number of tokens in the
140 + # prompt are fixed poem lines.
141 + assert (
142 + prefix_len > base_prompt_offset
143 + ), f"Please set 'args.prefix-len' higher than {base_prompt_offset}."
144 +
145 + num_prefix_lines = round(
146 + (prefix_len - base_prompt_offset) / average_poem_len)
147 + prefix_lines = poem_lines[:num_prefix_lines]
148 +
149 + # Sample the rest of lines per request.
150 + sampled_requests: List[Tuple[str, int, int]] = []
151 + for _ in range(num_requests):
152 + sampled_lines = "".join(
153 + prefix_lines +
154 + random.sample(poem_lines, num_input_lines - num_prefix_lines))
155 +
156 + prompt = f"{base_prompt}{sampled_lines}"
157 + message = [
158 + {
159 + "role": "user",
160 + "content": prompt,
161 + },
162 + ]
163 + prompt_formatted = tokenizer.apply_chat_template(
164 + message, add_generation_prompt=True, tokenize=False)
165 + prompt_len = len(tokenizer(prompt_formatted).input_ids)
166 + sampled_requests.append(
167 + (prompt, prompt_formatted, prompt_len, output_len))
168 +
169 + return sampled_requests
170 +
171 +
172 async def get_request(
173 input_requests: List[Tuple[str, int, int]],
174 request_rate: float,
@@ -119,37 +191,42 @@ def calculate_metrics(
191 outputs: List[RequestFuncOutput],
192 dur_s: float,
193 tokenizer: PreTrainedTokenizerBase,
122 -) -> BenchmarkMetrics:
123 - total_output = 0
194 +) -> Tuple[BenchmarkMetrics, List[int]]:
195 + actual_output_lens = []
196 total_input = 0
197 completed = 0
126 - per_token_latencies = []
198 + tpots = []
199 ttfts = []
200 for i in range(len(outputs)):
201 if outputs[i].success:
130 - output_len = len(tokenizer.encode(outputs[i].generated_text))
131 - total_output += output_len
202 + output_len = len(tokenizer(outputs[i].generated_text).input_ids)
203 + actual_output_lens.append(output_len)
204 total_input += input_requests[i][1]
133 - per_token_latencies.append(outputs[i].latency / output_len)
205 + if output_len > 1:
206 + tpots.append(
207 + (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
208 ttfts.append(outputs[i].ttft)
209 completed += 1
210 + else:
211 + actual_output_lens.append(0)
212
213 metrics = BenchmarkMetrics(
214 completed=completed,
215 total_input=total_input,
140 - total_output=total_output,
216 + total_output=sum(actual_output_lens),
217 request_throughput=completed / dur_s,
218 input_throughput=total_input / dur_s,
143 - output_throughput=total_output / dur_s,
144 - mean_ttft_ms=np.mean(ttfts) * 1000,
145 - median_ttft_ms=np.median(ttfts) * 1000,
146 - p99_ttft_ms=np.percentile(ttfts, 99) * 1000,
147 - mean_tpot_ms=np.mean(per_token_latencies) * 1000,
148 - median_tpot_ms=np.median(per_token_latencies) * 1000,
149 - p99_tpot_ms=np.percentile(per_token_latencies, 99) * 1000,
219 + output_throughput=sum(actual_output_lens) / dur_s,
220 + mean_ttft_ms=np.mean(ttfts or 0) *
221 + 1000, # ttfts is empty if streaming is not supported by backend
222 + median_ttft_ms=np.median(ttfts or 0) * 1000,
223 + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000,
224 + mean_tpot_ms=np.mean(tpots) * 1000,
225 + median_tpot_ms=np.median(tpots) * 1000,
226 + p99_tpot_ms=np.percentile(tpots, 99) * 1000,
227 )
228
152 - return metrics
229 + return metrics, actual_output_lens
230
231
232 async def benchmark(
@@ -189,40 +266,53 @@ async def benchmark(
266 asyncio.create_task(
267 request_func(request_func_input=request_func_input,
268 pbar=pbar)))
192 - outputs = await asyncio.gather(*tasks)
269 + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
270
271 if not disable_tqdm:
272 pbar.close()
273
274 benchmark_duration = time.perf_counter() - benchmark_start_time
275
199 - metrics = calculate_metrics(
276 + metrics, actual_output_lens = calculate_metrics(
277 input_requests=input_requests,
278 outputs=outputs,
279 dur_s=benchmark_duration,
280 tokenizer=tokenizer,
281 )
282
206 - print(f"Successful requests: {metrics.completed}")
207 - print(f"Benchmark duration: {benchmark_duration:2f} s")
208 - print(f"Total input tokens: {metrics.total_input}")
209 - print(f"Total generated tokens: {metrics.total_output}")
210 - print(f"Request throughput: {metrics.request_throughput:.2f} requests/s")
211 - print(f"Input token throughput: {metrics.input_throughput:.2f} tokens/s")
212 - print(f"Output token throughput: {metrics.output_throughput:.2f} tokens/s")
213 - print(f"Mean TTFT: {metrics.mean_ttft_ms:.2f} ms")
214 - print(f"Median TTFT: {metrics.median_ttft_ms:.2f} ms")
215 - print(f"P99 TTFT: {metrics.p99_ttft_ms:.2f} ms")
216 - print(f"Mean TPOT: {metrics.mean_tpot_ms:.2f} ms")
217 - print(f"Median TPOT: {metrics.median_tpot_ms:.2f} ms")
218 - print(f"P99 TPOT: {metrics.p99_tpot_ms:.2f} ms")
283 + print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
284 + print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
285 + print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
286 + benchmark_duration))
287 + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
288 + print("{:<40} {:<10}".format("Total generated tokens:",
289 + metrics.total_output))
290 + print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
291 + metrics.request_throughput))
292 + print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):",
293 + metrics.input_throughput))
294 + print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
295 + metrics.output_throughput))
296 + print("{s:{c}^{n}}".format(s='Time to First Token', n=50, c='-'))
297 + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms))
298 + print("{:<40} {:<10.2f}".format("Median TTFT (ms):",
299 + metrics.median_ttft_ms))
300 + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms))
301 + print("{s:{c}^{n}}".format(s='Time per Output Token (excl. 1st token)',
302 + n=50,
303 + c='-'))
304 + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms))
305 + print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
306 + metrics.median_tpot_ms))
307 + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
308 + print("=" * 50)
309
310 result = {
311 "duration": benchmark_duration,
312 "completed": metrics.completed,
313 "total_input_tokens": metrics.total_input,
314 "total_output_tokens": metrics.total_output,
225 - "request_inthroughput": metrics.request_throughput,
315 + "request_throughput": metrics.request_throughput,
316 "input_throughput": metrics.input_throughput,
317 "output_throughput": metrics.output_throughput,
318 "mean_ttft_ms": metrics.mean_ttft_ms,
@@ -230,7 +320,13 @@ async def benchmark(
320 "p99_ttft_ms": metrics.p99_ttft_ms,
321 "mean_tpot_ms": metrics.mean_tpot_ms,
322 "median_tpot_ms": metrics.median_tpot_ms,
233 - "p99_tpot_ms": metrics.p99_tpot_ms
323 + "p99_tpot_ms": metrics.p99_tpot_ms,
324 + "input_lens": [output.prompt_len for output in outputs],
325 + "output_lens": actual_output_lens,
326 + "ttfts": [output.ttft for output in outputs],
327 + "itls": [output.itl for output in outputs],
328 + "generated_texts": [output.generated_text for output in outputs],
329 + "errors": [output.error for output in outputs],
330 }
331 return result
332
@@ -251,7 +347,58 @@ def main(args: argparse.Namespace):
347
348 tokenizer = get_tokenizer(tokenizer_id,
349 trust_remote_code=args.trust_remote_code)
254 - input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
350 +
351 + if args.dataset is not None:
352 + warnings.warn(
353 + "The '--dataset' argument will be deprecated in the next "
354 + "release. Please use '--dataset-name' and "
355 + "'--dataset-path' in the future runs.",
356 + stacklevel=2)
357 + input_requests = sample_sharegpt_requests(
358 + dataset_path=args.dataset,
359 + num_requests=args.num_prompts,
360 + tokenizer=tokenizer,
361 + )
362 +
363 + elif args.dataset_name == "sharegpt":
364 + input_requests = sample_sharegpt_requests(
365 + dataset_path=args.dataset_path,
366 + num_requests=args.num_prompts,
367 + tokenizer=tokenizer,
368 + )
369 +
370 + elif args.dataset_name == "sonnet":
371 + # Do not format the prompt, pass to message directly
372 + if args.backend == "openai-chat":
373 + input_requests = sample_sonnet_requests(
374 + dataset_path=args.dataset_path,
375 + num_requests=args.num_prompts,
376 + input_len=args.input_len,
377 + output_len=args.output_len,
378 + prefix_len=args.prefix_len,
379 + tokenizer=tokenizer,
380 + )
381 + input_requests = [(prompt, prompt_len, output_len)
382 + for prompt, prompt_formatted, prompt_len,
383 + output_len in input_requests]
384 + else:
385 + assert (
386 + tokenizer.chat_template or tokenizer.default_chat_template
387 + ), "Tokenizer/model must have chat template for sonnet dataset."
388 + input_requests = sample_sonnet_requests(
389 + dataset_path=args.dataset_path,
390 + num_requests=args.num_prompts,
391 + input_len=args.input_len,
392 + output_len=args.output_len,
393 + prefix_len=args.prefix_len,
394 + tokenizer=tokenizer,
395 + )
396 + input_requests = [(prompt_formatted, prompt_len, output_len)
397 + for prompt, prompt_formatted, prompt_len,
398 + output_len in input_requests]
399 +
400 + else:
401 + raise ValueError(f"Unknown dataset: {args.dataset_name}")
402
403 benchmark_result = asyncio.run(
404 benchmark(
@@ -274,13 +421,23 @@ def main(args: argparse.Namespace):
421 current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
422 result_json["date"] = current_dt
423 result_json["backend"] = backend
277 - result_json["version"] = args.version
424 result_json["model_id"] = model_id
425 result_json["tokenizer_id"] = tokenizer_id
426 result_json["best_of"] = args.best_of
427 result_json["use_beam_search"] = args.use_beam_search
428 result_json["num_prompts"] = args.num_prompts
429
430 + # Metadata
431 + if args.metadata:
432 + for item in args.metadata:
433 + if "=" in item:
434 + kvstring = item.split("=")
435 + result_json[kvstring[0].strip()] = kvstring[1].strip()
436 + else:
437 + raise ValueError(
438 + "Invalid metadata format. Please use KEY=VALUE format."
439 + )
440 +
441 # Traffic
442 result_json["request_rate"] = (
443 args.request_rate if args.request_rate < float("inf") else "inf")
@@ -290,9 +447,9 @@ def main(args: argparse.Namespace):
447
448 # Save to file
449 base_model_id = model_id.split("/")[-1]
293 - file_name = (
294 - f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
295 - )
450 + file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" #noqa
451 + if args.result_dir:
452 + file_name = os.path.join(args.result_dir, file_name)
453 with open(file_name, "w") as outfile:
454 json.dump(result_json, outfile)
455
@@ -306,12 +463,6 @@ if __name__ == "__main__":
463 default="vllm",
464 choices=list(ASYNC_REQUEST_FUNCS.keys()),
465 )
309 - parser.add_argument(
310 - "--version",
311 - type=str,
312 - default="N/A",
313 - help="Version of the serving backend/engine.",
314 - )
466 parser.add_argument(
467 "--base-url",
468 type=str,
@@ -323,12 +474,26 @@ if __name__ == "__main__":
474 parser.add_argument(
475 "--endpoint",
476 type=str,
326 - default="/generate",
477 + default="/v1/completions",
478 help="API endpoint.",
479 )
329 - parser.add_argument("--dataset",
480 + parser.add_argument(
481 + "--dataset",
482 + type=str,
483 + default=None,
484 + help="Path to the ShareGPT dataset, will be deprecated in the "
485 + "next release.",
486 + )
487 + parser.add_argument(
488 + "--dataset-name",
489 + type=str,
490 + default="sharegpt",
491 + choices=["sharegpt", "sonnet"],
492 + help="Name of the dataset to benchmark on.",
493 + )
494 + parser.add_argument("--dataset-path",
495 type=str,
331 - required=True,
496 + default=None,
497 help="Path to the dataset.")
498 parser.add_argument(
499 "--model",
@@ -356,6 +521,27 @@ if __name__ == "__main__":
521 default=1000,
522 help="Number of prompts to process.",
523 )
524 + parser.add_argument(
525 + "--sonnet-input-len",
526 + type=int,
527 + default=550,
528 + help=
529 + "Number of input tokens per request, used only for sonnet dataset.",
530 + )
531 + parser.add_argument(
532 + "--sonnet-output-len",
533 + type=int,
534 + default=150,
535 + help=
536 + "Number of output tokens per request, used only for sonnet dataset.",
537 + )
538 + parser.add_argument(
539 + "--sonnet-prefix-len",
540 + type=int,
541 + default=200,
542 + help=
543 + "Number of prefix tokens per request, used only for sonnet dataset.",
544 + )
545 parser.add_argument(
546 "--request-rate",
547 type=float,
@@ -381,6 +567,21 @@ if __name__ == "__main__":
567 action="store_true",
568 help="Specify to save benchmark results to a json file",
569 )
570 + parser.add_argument(
571 + "--metadata",
572 + metavar="KEY=VALUE",
573 + nargs="*",
574 + help="Key-value pairs (e.g, --metadata version=0.3.3 tp=1) "
575 + "for metadata of this run to be saved in the result JSON file "
576 + "for record keeping purposes.",
577 + )
578 + parser.add_argument(
579 + "--result-dir",
580 + type=str,
581 + default=None,
582 + help="Specify directory to save benchmark json results."
583 + "If not specified, results are saved in the current directory.",
584 + )
585
586 args = parser.parse_args()
587 main(args)
benchmarks/sonnet.txt new
+518
@@ -0,0 +1,518 @@
1 +FROM fairest creatures we desire increase,
2 +That thereby beauty's rose might never die,
3 +But as the riper should by time decease,
4 +His tender heir might bear his memory:
5 +But thou, contracted to thine own bright eyes,
6 +Feed'st thy light'st flame with self-substantial fuel,
7 +Making a famine where abundance lies,
8 +Thyself thy foe, to thy sweet self too cruel.
9 +Thou that art now the world's fresh ornament
10 +And only herald to the gaudy spring,
11 +Within thine own bud buriest thy content
12 +And, tender churl, makest waste in niggarding.
13 +Pity the world, or else this glutton be,
14 +To eat the world's due, by the grave and thee.
15 +When forty winters shall beseige thy brow,
16 +And dig deep trenches in thy beauty's field,
17 +Thy youth's proud livery, so gazed on now,
18 +Will be a tatter'd weed, of small worth held:
19 +Then being ask'd where all thy beauty lies,
20 +Where all the treasure of thy lusty days,
21 +To say, within thine own deep-sunken eyes,
22 +Were an all-eating shame and thriftless praise.
23 +How much more praise deserved thy beauty's use,
24 +If thou couldst answer 'This fair child of mine
25 +Shall sum my count and make my old excuse,'
26 +Proving his beauty by succession thine!
27 +This were to be new made when thou art old,
28 +And see thy blood warm when thou feel'st it cold.
29 +Look in thy glass, and tell the face thou viewest
30 +Now is the time that face should form another;
31 +Whose fresh repair if now thou not renewest,
32 +Thou dost beguile the world, unbless some mother.
33 +For where is she so fair whose unear'd womb
34 +Disdains the tillage of thy husbandry?
35 +Or who is he so fond will be the tomb
36 +Of his self-love, to stop posterity?
37 +Thou art thy mother's glass, and she in thee
38 +Calls back the lovely April of her prime:
39 +So thou through windows of thine age shall see
40 +Despite of wrinkles this thy golden time.
41 +But if thou live, remember'd not to be,
42 +Die single, and thine image dies with thee.
43 +Unthrifty loveliness, why dost thou spend
44 +Upon thyself thy beauty's legacy?
45 +Nature's bequest gives nothing but doth lend,
46 +And being frank she lends to those are free.
47 +Then, beauteous niggard, why dost thou abuse
48 +The bounteous largess given thee to give?
49 +Profitless usurer, why dost thou use
50 +So great a sum of sums, yet canst not live?
51 +For having traffic with thyself alone,
52 +Thou of thyself thy sweet self dost deceive.
53 +Then how, when nature calls thee to be gone,
54 +What acceptable audit canst thou leave?
55 +Thy unused beauty must be tomb'd with thee,
56 +Which, used, lives th' executor to be.
57 +Those hours, that with gentle work did frame
58 +The lovely gaze where every eye doth dwell,
59 +Will play the tyrants to the very same
60 +And that unfair which fairly doth excel:
61 +For never-resting time leads summer on
62 +To hideous winter and confounds him there;
63 +Sap cheque'd with frost and lusty leaves quite gone,
64 +Beauty o'ersnow'd and bareness every where:
65 +Then, were not summer's distillation left,
66 +A liquid prisoner pent in walls of glass,
67 +Beauty's effect with beauty were bereft,
68 +Nor it nor no remembrance what it was:
69 +But flowers distill'd though they with winter meet,
70 +Leese but their show; their substance still lives sweet.
71 +Then let not winter's ragged hand deface
72 +In thee thy summer, ere thou be distill'd:
73 +Make sweet some vial; treasure thou some place
74 +With beauty's treasure, ere it be self-kill'd.
75 +That use is not forbidden usury,
76 +Which happies those that pay the willing loan;
77 +That's for thyself to breed another thee,
78 +Or ten times happier, be it ten for one;
79 +Ten times thyself were happier than thou art,
80 +If ten of thine ten times refigured thee:
81 +Then what could death do, if thou shouldst depart,
82 +Leaving thee living in posterity?
83 +Be not self-will'd, for thou art much too fair
84 +To be death's conquest and make worms thine heir.
85 +Lo! in the orient when the gracious light
86 +Lifts up his burning head, each under eye
87 +Doth homage to his new-appearing sight,
88 +Serving with looks his sacred majesty;
89 +And having climb'd the steep-up heavenly hill,
90 +Resembling strong youth in his middle age,
91 +yet mortal looks adore his beauty still,
92 +Attending on his golden pilgrimage;
93 +But when from highmost pitch, with weary car,
94 +Like feeble age, he reeleth from the day,
95 +The eyes, 'fore duteous, now converted are
96 +From his low tract and look another way:
97 +So thou, thyself out-going in thy noon,
98 +Unlook'd on diest, unless thou get a son.
99 +Music to hear, why hear'st thou music sadly?
100 +Sweets with sweets war not, joy delights in joy.
101 +Why lovest thou that which thou receivest not gladly,
102 +Or else receivest with pleasure thine annoy?
103 +If the true concord of well-tuned sounds,
104 +By unions married, do offend thine ear,
105 +They do but sweetly chide thee, who confounds
106 +In singleness the parts that thou shouldst bear.
107 +Mark how one string, sweet husband to another,
108 +Strikes each in each by mutual ordering,
109 +Resembling sire and child and happy mother
110 +Who all in one, one pleasing note do sing:
111 +Whose speechless song, being many, seeming one,
112 +Sings this to thee: 'thou single wilt prove none.'
113 +Is it for fear to wet a widow's eye
114 +That thou consumest thyself in single life?
115 +Ah! if thou issueless shalt hap to die.
116 +The world will wail thee, like a makeless wife;
117 +The world will be thy widow and still weep
118 +That thou no form of thee hast left behind,
119 +When every private widow well may keep
120 +By children's eyes her husband's shape in mind.
121 +Look, what an unthrift in the world doth spend
122 +Shifts but his place, for still the world enjoys it;
123 +But beauty's waste hath in the world an end,
124 +And kept unused, the user so destroys it.
125 +No love toward others in that bosom sits
126 +That on himself such murderous shame commits.
127 +For shame! deny that thou bear'st love to any,
128 +Who for thyself art so unprovident.
129 +Grant, if thou wilt, thou art beloved of many,
130 +But that thou none lovest is most evident;
131 +For thou art so possess'd with murderous hate
132 +That 'gainst thyself thou stick'st not to conspire.
133 +Seeking that beauteous roof to ruinate
134 +Which to repair should be thy chief desire.
135 +O, change thy thought, that I may change my mind!
136 +Shall hate be fairer lodged than gentle love?
137 +Be, as thy presence is, gracious and kind,
138 +Or to thyself at least kind-hearted prove:
139 +Make thee another self, for love of me,
140 +That beauty still may live in thine or thee.
141 +As fast as thou shalt wane, so fast thou growest
142 +In one of thine, from that which thou departest;
143 +And that fresh blood which youngly thou bestowest
144 +Thou mayst call thine when thou from youth convertest.
145 +Herein lives wisdom, beauty and increase:
146 +Without this, folly, age and cold decay:
147 +If all were minded so, the times should cease
148 +And threescore year would make the world away.
149 +Let those whom Nature hath not made for store,
150 +Harsh featureless and rude, barrenly perish:
151 +Look, whom she best endow'd she gave the more;
152 +Which bounteous gift thou shouldst in bounty cherish:
153 +She carved thee for her seal, and meant thereby
154 +Thou shouldst print more, not let that copy die.
155 +When I do count the clock that tells the time,
156 +And see the brave day sunk in hideous night;
157 +When I behold the violet past prime,
158 +And sable curls all silver'd o'er with white;
159 +When lofty trees I see barren of leaves
160 +Which erst from heat did canopy the herd,
161 +And summer's green all girded up in sheaves
162 +Borne on the bier with white and bristly beard,
163 +Then of thy beauty do I question make,
164 +That thou among the wastes of time must go,
165 +Since sweets and beauties do themselves forsake
166 +And die as fast as they see others grow;
167 +And nothing 'gainst Time's scythe can make defence
168 +Save breed, to brave him when he takes thee hence.
169 +O, that you were yourself! but, love, you are
170 +No longer yours than you yourself here live:
171 +Against this coming end you should prepare,
172 +And your sweet semblance to some other give.
173 +So should that beauty which you hold in lease
174 +Find no determination: then you were
175 +Yourself again after yourself's decease,
176 +When your sweet issue your sweet form should bear.
177 +Who lets so fair a house fall to decay,
178 +Which husbandry in honour might uphold
179 +Against the stormy gusts of winter's day
180 +And barren rage of death's eternal cold?
181 +O, none but unthrifts! Dear my love, you know
182 +You had a father: let your son say so.
183 +Not from the stars do I my judgment pluck;
184 +And yet methinks I have astronomy,
185 +But not to tell of good or evil luck,
186 +Of plagues, of dearths, or seasons' quality;
187 +Nor can I fortune to brief minutes tell,
188 +Pointing to each his thunder, rain and wind,
189 +Or say with princes if it shall go well,
190 +By oft predict that I in heaven find:
191 +But from thine eyes my knowledge I derive,
192 +And, constant stars, in them I read such art
193 +As truth and beauty shall together thrive,
194 +If from thyself to store thou wouldst convert;
195 +Or else of thee this I prognosticate:
196 +Thy end is truth's and beauty's doom and date.
197 +When I consider every thing that grows
198 +Holds in perfection but a little moment,
199 +That this huge stage presenteth nought but shows
200 +Whereon the stars in secret influence comment;
201 +When I perceive that men as plants increase,
202 +Cheered and cheque'd even by the self-same sky,
203 +Vaunt in their youthful sap, at height decrease,
204 +And wear their brave state out of memory;
205 +Then the conceit of this inconstant stay
206 +Sets you most rich in youth before my sight,
207 +Where wasteful Time debateth with Decay,
208 +To change your day of youth to sullied night;
209 +And all in war with Time for love of you,
210 +As he takes from you, I engraft you new.
211 +But wherefore do not you a mightier way
212 +Make war upon this bloody tyrant, Time?
213 +And fortify yourself in your decay
214 +With means more blessed than my barren rhyme?
215 +Now stand you on the top of happy hours,
216 +And many maiden gardens yet unset
217 +With virtuous wish would bear your living flowers,
218 +Much liker than your painted counterfeit:
219 +So should the lines of life that life repair,
220 +Which this, Time's pencil, or my pupil pen,
221 +Neither in inward worth nor outward fair,
222 +Can make you live yourself in eyes of men.
223 +To give away yourself keeps yourself still,
224 +And you must live, drawn by your own sweet skill.
225 +Who will believe my verse in time to come,
226 +If it were fill'd with your most high deserts?
227 +Though yet, heaven knows, it is but as a tomb
228 +Which hides your life and shows not half your parts.
229 +If I could write the beauty of your eyes
230 +And in fresh numbers number all your graces,
231 +The age to come would say 'This poet lies:
232 +Such heavenly touches ne'er touch'd earthly faces.'
233 +So should my papers yellow'd with their age
234 +Be scorn'd like old men of less truth than tongue,
235 +And your true rights be term'd a poet's rage
236 +And stretched metre of an antique song:
237 +But were some child of yours alive that time,
238 +You should live twice; in it and in my rhyme.
239 +Shall I compare thee to a summer's day?
240 +Thou art more lovely and more temperate:
241 +Rough winds do shake the darling buds of May,
242 +And summer's lease hath all too short a date:
243 +Sometime too hot the eye of heaven shines,
244 +And often is his gold complexion dimm'd;
245 +And every fair from fair sometime declines,
246 +By chance or nature's changing course untrimm'd;
247 +But thy eternal summer shall not fade
248 +Nor lose possession of that fair thou owest;
249 +Nor shall Death brag thou wander'st in his shade,
250 +When in eternal lines to time thou growest:
251 +So long as men can breathe or eyes can see,
252 +So long lives this and this gives life to thee.
253 +Devouring Time, blunt thou the lion's paws,
254 +And make the earth devour her own sweet brood;
255 +Pluck the keen teeth from the fierce tiger's jaws,
256 +And burn the long-lived phoenix in her blood;
257 +Make glad and sorry seasons as thou fleets,
258 +And do whate'er thou wilt, swift-footed Time,
259 +To the wide world and all her fading sweets;
260 +But I forbid thee one most heinous crime:
261 +O, carve not with thy hours my love's fair brow,
262 +Nor draw no lines there with thine antique pen;
263 +Him in thy course untainted do allow
264 +For beauty's pattern to succeeding men.
265 +Yet, do thy worst, old Time: despite thy wrong,
266 +My love shall in my verse ever live young.
267 +A woman's face with Nature's own hand painted
268 +Hast thou, the master-mistress of my passion;
269 +A woman's gentle heart, but not acquainted
270 +With shifting change, as is false women's fashion;
271 +An eye more bright than theirs, less false in rolling,
272 +Gilding the object whereupon it gazeth;
273 +A man in hue, all 'hues' in his controlling,
274 +Much steals men's eyes and women's souls amazeth.
275 +And for a woman wert thou first created;
276 +Till Nature, as she wrought thee, fell a-doting,
277 +And by addition me of thee defeated,
278 +By adding one thing to my purpose nothing.
279 +But since she prick'd thee out for women's pleasure,
280 +Mine be thy love and thy love's use their treasure.
281 +So is it not with me as with that Muse
282 +Stirr'd by a painted beauty to his verse,
283 +Who heaven itself for ornament doth use
284 +And every fair with his fair doth rehearse
285 +Making a couplement of proud compare,
286 +With sun and moon, with earth and sea's rich gems,
287 +With April's first-born flowers, and all things rare
288 +That heaven's air in this huge rondure hems.
289 +O' let me, true in love, but truly write,
290 +And then believe me, my love is as fair
291 +As any mother's child, though not so bright
292 +As those gold candles fix'd in heaven's air:
293 +Let them say more than like of hearsay well;
294 +I will not praise that purpose not to sell.
295 +My glass shall not persuade me I am old,
296 +So long as youth and thou are of one date;
297 +But when in thee time's furrows I behold,
298 +Then look I death my days should expiate.
299 +For all that beauty that doth cover thee
300 +Is but the seemly raiment of my heart,
301 +Which in thy breast doth live, as thine in me:
302 +How can I then be elder than thou art?
303 +O, therefore, love, be of thyself so wary
304 +As I, not for myself, but for thee will;
305 +Bearing thy heart, which I will keep so chary
306 +As tender nurse her babe from faring ill.
307 +Presume not on thy heart when mine is slain;
308 +Thou gavest me thine, not to give back again.
309 +As an unperfect actor on the stage
310 +Who with his fear is put besides his part,
311 +Or some fierce thing replete with too much rage,
312 +Whose strength's abundance weakens his own heart.
313 +So I, for fear of trust, forget to say
314 +The perfect ceremony of love's rite,
315 +And in mine own love's strength seem to decay,
316 +O'ercharged with burden of mine own love's might.
317 +O, let my books be then the eloquence
318 +And dumb presagers of my speaking breast,
319 +Who plead for love and look for recompense
320 +More than that tongue that more hath more express'd.
321 +O, learn to read what silent love hath writ:
322 +To hear with eyes belongs to love's fine wit.
323 +Mine eye hath play'd the painter and hath stell'd
324 +Thy beauty's form in table of my heart;
325 +My body is the frame wherein 'tis held,
326 +And perspective it is the painter's art.
327 +For through the painter must you see his skill,
328 +To find where your true image pictured lies;
329 +Which in my bosom's shop is hanging still,
330 +That hath his windows glazed with thine eyes.
331 +Now see what good turns eyes for eyes have done:
332 +Mine eyes have drawn thy shape, and thine for me
333 +Are windows to my breast, where-through the sun
334 +Delights to peep, to gaze therein on thee;
335 +Yet eyes this cunning want to grace their art;
336 +They draw but what they see, know not the heart.
337 +Let those who are in favour with their stars
338 +Of public honour and proud titles boast,
339 +Whilst I, whom fortune of such triumph bars,
340 +Unlook'd for joy in that I honour most.
341 +Great princes' favourites their fair leaves spread
342 +But as the marigold at the sun's eye,
343 +And in themselves their pride lies buried,
344 +For at a frown they in their glory die.
345 +The painful warrior famoused for fight,
346 +After a thousand victories once foil'd,
347 +Is from the book of honour razed quite,
348 +And all the rest forgot for which he toil'd:
349 +Then happy I, that love and am beloved
350 +Where I may not remove nor be removed.
351 +Lord of my love, to whom in vassalage
352 +Thy merit hath my duty strongly knit,
353 +To thee I send this written embassage,
354 +To witness duty, not to show my wit:
355 +Duty so great, which wit so poor as mine
356 +May make seem bare, in wanting words to show it,
357 +But that I hope some good conceit of thine
358 +In thy soul's thought, all naked, will bestow it;
359 +Till whatsoever star that guides my moving
360 +Points on me graciously with fair aspect
361 +And puts apparel on my tatter'd loving,
362 +To show me worthy of thy sweet respect:
363 +Then may I dare to boast how I do love thee;
364 +Till then not show my head where thou mayst prove me.
365 +Weary with toil, I haste me to my bed,
366 +The dear repose for limbs with travel tired;
367 +But then begins a journey in my head,
368 +To work my mind, when body's work's expired:
369 +For then my thoughts, from far where I abide,
370 +Intend a zealous pilgrimage to thee,
371 +And keep my drooping eyelids open wide,
372 +Looking on darkness which the blind do see
373 +Save that my soul's imaginary sight
374 +Presents thy shadow to my sightless view,
375 +Which, like a jewel hung in ghastly night,
376 +Makes black night beauteous and her old face new.
377 +Lo! thus, by day my limbs, by night my mind,
378 +For thee and for myself no quiet find.
379 +How can I then return in happy plight,
380 +That am debarr'd the benefit of rest?
381 +When day's oppression is not eased by night,
382 +But day by night, and night by day, oppress'd?
383 +And each, though enemies to either's reign,
384 +Do in consent shake hands to torture me;
385 +The one by toil, the other to complain
386 +How far I toil, still farther off from thee.
387 +I tell the day, to please them thou art bright
388 +And dost him grace when clouds do blot the heaven:
389 +So flatter I the swart-complexion'd night,
390 +When sparkling stars twire not thou gild'st the even.
391 +But day doth daily draw my sorrows longer
392 +And night doth nightly make grief's strength seem stronger.
393 +When, in disgrace with fortune and men's eyes,
394 +I all alone beweep my outcast state
395 +And trouble deal heaven with my bootless cries
396 +And look upon myself and curse my fate,
397 +Wishing me like to one more rich in hope,
398 +Featured like him, like him with friends possess'd,
399 +Desiring this man's art and that man's scope,
400 +With what I most enjoy contented least;
401 +Yet in these thoughts myself almost despising,
402 +Haply I think on thee, and then my state,
403 +Like to the lark at break of day arising
404 +From sullen earth, sings hymns at heaven's gate;
405 +For thy sweet love remember'd such wealth brings
406 +That then I scorn to change my state with kings.
407 +When to the sessions of sweet silent thought
408 +I summon up remembrance of things past,
409 +I sigh the lack of many a thing I sought,
410 +And with old woes new wail my dear time's waste:
411 +Then can I drown an eye, unused to flow,
412 +For precious friends hid in death's dateless night,
413 +And weep afresh love's long since cancell'd woe,
414 +And moan the expense of many a vanish'd sight:
415 +Then can I grieve at grievances foregone,
416 +And heavily from woe to woe tell o'er
417 +The sad account of fore-bemoaned moan,
418 +Which I new pay as if not paid before.
419 +But if the while I think on thee, dear friend,
420 +All losses are restored and sorrows end.
421 +Thy bosom is endeared with all hearts,
422 +Which I by lacking have supposed dead,
423 +And there reigns love and all love's loving parts,
424 +And all those friends which I thought buried.
425 +How many a holy and obsequious tear
426 +Hath dear religious love stol'n from mine eye
427 +As interest of the dead, which now appear
428 +But things removed that hidden in thee lie!
429 +Thou art the grave where buried love doth live,
430 +Hung with the trophies of my lovers gone,
431 +Who all their parts of me to thee did give;
432 +That due of many now is thine alone:
433 +Their images I loved I view in thee,
434 +And thou, all they, hast all the all of me.
435 +If thou survive my well-contented day,
436 +When that churl Death my bones with dust shall cover,
437 +And shalt by fortune once more re-survey
438 +These poor rude lines of thy deceased lover,
439 +Compare them with the bettering of the time,
440 +And though they be outstripp'd by every pen,
441 +Reserve them for my love, not for their rhyme,
442 +Exceeded by the height of happier men.
443 +O, then vouchsafe me but this loving thought:
444 +'Had my friend's Muse grown with this growing age,
445 +A dearer birth than this his love had brought,
446 +To march in ranks of better equipage:
447 +But since he died and poets better prove,
448 +Theirs for their style I'll read, his for his love.'
449 +Full many a glorious morning have I seen
450 +Flatter the mountain-tops with sovereign eye,
451 +Kissing with golden face the meadows green,
452 +Gilding pale streams with heavenly alchemy;
453 +Anon permit the basest clouds to ride
454 +With ugly rack on his celestial face,
455 +And from the forlorn world his visage hide,
456 +Stealing unseen to west with this disgrace:
457 +Even so my sun one early morn did shine
458 +With all triumphant splendor on my brow;
459 +But out, alack! he was but one hour mine;
460 +The region cloud hath mask'd him from me now.
461 +Yet him for this my love no whit disdaineth;
462 +Suns of the world may stain when heaven's sun staineth.
463 +Why didst thou promise such a beauteous day,
464 +And make me travel forth without my cloak,
465 +To let base clouds o'ertake me in my way,
466 +Hiding thy bravery in their rotten smoke?
467 +'Tis not enough that through the cloud thou break,
468 +To dry the rain on my storm-beaten face,
469 +For no man well of such a salve can speak
470 +That heals the wound and cures not the disgrace:
471 +Nor can thy shame give physic to my grief;
472 +Though thou repent, yet I have still the loss:
473 +The offender's sorrow lends but weak relief
474 +To him that bears the strong offence's cross.
475 +Ah! but those tears are pearl which thy love sheds,
476 +And they are rich and ransom all ill deeds.
477 +No more be grieved at that which thou hast done:
478 +Roses have thorns, and silver fountains mud;
479 +Clouds and eclipses stain both moon and sun,
480 +And loathsome canker lives in sweetest bud.
481 +All men make faults, and even I in this,
482 +Authorizing thy trespass with compare,
483 +Myself corrupting, salving thy amiss,
484 +Excusing thy sins more than thy sins are;
485 +For to thy sensual fault I bring in sense--
486 +Thy adverse party is thy advocate--
487 +And 'gainst myself a lawful plea commence:
488 +Such civil war is in my love and hate
489 +That I an accessary needs must be
490 +To that sweet thief which sourly robs from me.
491 +Let me confess that we two must be twain,
492 +Although our undivided loves are one:
493 +So shall those blots that do with me remain
494 +Without thy help by me be borne alone.
495 +In our two loves there is but one respect,
496 +Though in our lives a separable spite,
497 +Which though it alter not love's sole effect,
498 +Yet doth it steal sweet hours from love's delight.
499 +I may not evermore acknowledge thee,
500 +Lest my bewailed guilt should do thee shame,
501 +Nor thou with public kindness honour me,
502 +Unless thou take that honour from thy name:
503 +But do not so; I love thee in such sort
504 +As, thou being mine, mine is thy good report.
505 +As a decrepit father takes delight
506 +To see his active child do deeds of youth,
507 +So I, made lame by fortune's dearest spite,
508 +Take all my comfort of thy worth and truth.
509 +For whether beauty, birth, or wealth, or wit,
510 +Or any of these all, or all, or more,
511 +Entitled in thy parts do crowned sit,
512 +I make my love engrafted to this store:
513 +So then I am not lame, poor, nor despised,
514 +Whilst that this shadow doth such substance give
515 +That I in thy abundance am sufficed
516 +And by a part of all thy glory live.
517 +Look, what is best, that best I wish in thee:
518 +This wish I have; then ten times happy me!
\ No newline at end of file
pyproject.toml
+1 -1
@@ -50,7 +50,7 @@ exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
50
51 [tool.codespell]
52 ignore-words-list = "dout, te, indicies"
53 -skip = "./tests/prompts"
53 +skip = "./tests/prompts,./benchmarks/sonnet.txt"
54
55 [tool.isort]
56 use_parentheses = true
tests/kernels/test_prefix_prefill.py
+2 -2
@@ -36,8 +36,8 @@ def test_contexted_kv_attention(
36 torch.cuda.manual_seed(0)
37 torch.set_default_device(device)
38
39 - # Need this, otherwise when we capture the graph the process for GPU 1 would
40 - # run on both GPU0 and GPU1 and things would hang
39 + # Need this, otherwise when we capture the graph the process
40 + # for GPU 1 would run on both GPU0 and GPU1 and things would hang
41 #
42 # see also similar issue: https://github.com/Dao-AILab/flash-attention/issues/523
43 torch.cuda.set_device(device)