A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
feat(benchmarks): Add Prefix Caching Benchmark to Serving Benchmark (#3277)
Roger Wang committed
Mar 27, 2024 at 13:39 UTC
45b6ef651387100c24d671fb485e7a6e208216f6
6 files changed
+897
-155
.buildkite/run-benchmarks.sh
+3
-2
@@ -23,8 +23,9 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/r
23
# wait for server to start, timeout after 600 seconds
24
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
25
python3 benchmarks/benchmark_serving.py \
26
- --backend openai \
27
- --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
26
+ --backend vllm \
27
+ --dataset-name sharegpt \
28
+ --dataset-path ./ShareGPT_V3_unfiltered_cleaned_split.json \
29
--model meta-llama/Llama-2-7b-chat-hf \
30
--num-prompts 20 \
31
--endpoint /v1/completions \
benchmarks/backend_request_func.py
+121
-99
@@ -1,8 +1,10 @@
1
import json
2
import os
3
+import sys
4
import time
4
-from dataclasses import dataclass
5
-from typing import Optional
5
+import traceback
6
+from dataclasses import dataclass, field
7
+from typing import List, Optional
8
9
import aiohttp
10
from tqdm.asyncio import tqdm
@@ -26,8 +28,11 @@ class RequestFuncOutput:
28
generated_text: str = ""
29
success: bool = False
30
latency: float = 0
29
- ttft: float = 0
31
+ ttft: float = 0 # Time to first token
32
+ itl: List[float] = field(
33
+ default_factory=list) # List of inter-token latencies
34
prompt_len: int = 0
35
+ error: str = ""
36
37
38
async def async_request_tgi(
@@ -55,71 +60,38 @@ async def async_request_tgi(
60
61
ttft = 0
62
st = time.perf_counter()
63
+ most_recent_timestamp = st
64
try:
65
async with session.post(url=api_url, json=payload) as response:
66
if response.status == 200:
61
- async for data in response.content.iter_any():
62
- if ttft == 0:
63
- ttft = time.perf_counter() - st
64
- output.ttft = ttft
65
- output.latency = time.perf_counter() - st
66
-
67
- body = remove_prefix(data.decode("utf-8"), "data:")
68
- output.generated_text = json.loads(body)["generated_text"]
69
- output.success = True
70
- else:
71
- output.success = False
72
- except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
73
- output.success = False
74
-
75
- if pbar:
76
- pbar.update(1)
77
- return output
78
-
79
-
80
-async def async_request_vllm(
81
- request_func_input: RequestFuncInput,
82
- pbar: Optional[tqdm] = None,
83
-) -> RequestFuncOutput:
84
- api_url = request_func_input.api_url
85
- assert api_url.endswith("generate")
67
+ async for chunk in response.content:
68
+ chunk = chunk.strip()
69
+ if not chunk:
70
+ continue
71
87
- async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
88
- payload = {
89
- "prompt": request_func_input.prompt,
90
- "n": 1,
91
- "best_of": request_func_input.best_of,
92
- "use_beam_search": request_func_input.use_beam_search,
93
- "temperature": 0.0 if request_func_input.use_beam_search else 1.0,
94
- "top_p": 1.0,
95
- "max_tokens": request_func_input.output_len,
96
- "ignore_eos": True,
97
- "stream": True,
98
- }
99
- output = RequestFuncOutput()
100
- output.prompt_len = request_func_input.prompt_len
72
+ chunk = remove_prefix(chunk.decode("utf-8"), "data:")
73
102
- ttft = 0
103
- st = time.perf_counter()
104
- try:
105
- async with session.post(url=api_url, json=payload) as response:
106
- if response.status == 200:
107
- async for data in response.content.iter_any():
74
+ data = json.loads(chunk)
75
+ timestamp = time.perf_counter()
76
+ # First token
77
if ttft == 0:
78
ttft = time.perf_counter() - st
79
output.ttft = ttft
111
- output.latency = time.perf_counter() - st
80
113
- # When streaming, '\0' is appended to the end of response.
114
- body = data.decode("utf-8").strip("\0")
115
- output.generated_text = json.loads(
116
- body)["text"][0][len(request_func_input.prompt):]
117
- output.success = True
81
+ # Decoding phase
82
+ else:
83
+ output.itl.append(timestamp -
84
+ most_recent_timestamp)
85
119
- else:
120
- output.success = False
121
- except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
86
+ most_recent_timestamp = timestamp
87
+
88
+ output.latency = most_recent_timestamp - st
89
+ output.success = True
90
+ output.generated_text = data["generated_text"]
91
+ except Exception:
92
output.success = False
93
+ exc_info = sys.exc_info()
94
+ output.error = "".join(traceback.format_exception(*exc_info))
95
96
if pbar:
97
pbar.update(1)
@@ -146,26 +118,45 @@ async def async_request_trt_llm(
118
}
119
output = RequestFuncOutput()
120
output.prompt_len = request_func_input.prompt_len
149
- ttft = 0
121
122
+ ttft = 0
123
st = time.perf_counter()
124
+ most_recent_timestamp = st
125
try:
153
- async with session.post(url=api_url, json=payload) as resp:
154
- if resp.status == 200:
155
- async for data in resp.content.iter_any():
126
+ async with session.post(url=api_url, json=payload) as response:
127
+ if response.status == 200:
128
+ async for chunk in response.content:
129
+ chunk = chunk.strip()
130
+ if not chunk:
131
+ continue
132
+
133
+ chunk = remove_prefix(chunk.decode("utf-8"), "data:")
134
+
135
+ data = json.loads(chunk)
136
+ timestamp = time.perf_counter()
137
+ # First token
138
if ttft == 0:
139
ttft = time.perf_counter() - st
140
output.ttft = ttft
159
- output.latency = time.perf_counter() - st
141
161
- body = remove_prefix(data.decode("utf-8"), "data:")
162
- output.generated_text = json.loads(body)["text_output"]
142
+ # Decoding phase
143
+ else:
144
+ output.itl.append(timestamp -
145
+ most_recent_timestamp)
146
+
147
+ most_recent_timestamp = timestamp
148
+
149
+ output.latency = most_recent_timestamp - st
150
+ output.generated_text = json.loads(data)["text_output"]
151
output.success = True
152
153
else:
154
+ output.error = response.reason
155
output.success = False
167
- except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
156
+ except Exception:
157
output.success = False
158
+ exc_info = sys.exc_info()
159
+ output.error = "".join(traceback.format_exception(*exc_info))
160
161
if pbar:
162
pbar.update(1)
@@ -181,35 +172,35 @@ async def async_request_deepspeed_mii(
172
assert not request_func_input.use_beam_search
173
174
payload = {
184
- "prompts": request_func_input.prompt,
185
- "max_new_tokens": request_func_input.output_len,
186
- "ignore_eos": True,
187
- "do_sample": True,
188
- "temperature":
189
- 0.01, # deepspeed-mii does not accept 0.0 temperature.
175
+ "prompt": request_func_input.prompt,
176
+ "max_tokens": request_func_input.output_len,
177
+ "temperature": 0.01, # deepspeed-mii does not accept 0.0 temp.
178
"top_p": 1.0,
179
}
180
output = RequestFuncOutput()
181
output.prompt_len = request_func_input.prompt_len
182
195
- # DeepSpeed-MII doesn't support streaming as of Jan 28 2024,
183
+ # NOTE: DeepSpeed-MII doesn't support streaming as of Jan 28 2024,
184
# will use 0 as placeholder.
197
- # https://github.com/microsoft/DeepSpeed-MII/pull/311
185
+ # See https://github.com/microsoft/DeepSpeed-MII/pull/311
186
output.ttft = 0
187
188
st = time.perf_counter()
189
try:
190
async with session.post(url=request_func_input.api_url,
203
- json=payload) as resp:
204
- if resp.status == 200:
205
- parsed_resp = await resp.json()
191
+ json=payload) as response:
192
+ if response.status == 200:
193
+ parsed_resp = await response.json()
194
output.latency = time.perf_counter() - st
207
- output.generated_text = parsed_resp[0]["generated_text"]
195
+ output.generated_text = parsed_resp["text"][0]
196
output.success = True
197
else:
198
+ output.error = response.reason
199
output.success = False
211
- except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
200
+ except Exception:
201
output.success = False
202
+ exc_info = sys.exc_info()
203
+ output.error = "".join(traceback.format_exception(*exc_info))
204
205
if pbar:
206
pbar.update(1)
@@ -221,7 +212,9 @@ async def async_request_openai_completions(
212
pbar: Optional[tqdm] = None,
213
) -> RequestFuncOutput:
214
api_url = request_func_input.api_url
224
- assert api_url.endswith("v1/completions")
215
+ assert api_url.endswith(
216
+ "v1/completions"
217
+ ), "OpenAI Completions API URL must end with 'v1/completions'."
218
219
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
220
assert not request_func_input.use_beam_search
@@ -243,15 +236,12 @@ async def async_request_openai_completions(
236
generated_text = ""
237
ttft = 0
238
st = time.perf_counter()
239
+ most_recent_timestamp = st
240
try:
241
async with session.post(url=api_url, json=payload,
242
headers=headers) as response:
243
if response.status == 200:
244
async for chunk in response.content:
251
- if ttft == 0:
252
- ttft = time.perf_counter() - st
253
- output.ttft = ttft
254
-
245
chunk = chunk.strip()
246
if not chunk:
247
continue
@@ -260,16 +250,33 @@ async def async_request_openai_completions(
250
if chunk == "[DONE]":
251
latency = time.perf_counter() - st
252
else:
263
- body = json.loads(chunk)
264
- generated_text += body["choices"][0]["text"]
253
+ data = json.loads(chunk)
254
+
255
+ if data["choices"][0]["text"]:
256
+ timestamp = time.perf_counter()
257
+ # First token
258
+ if ttft == 0:
259
+ ttft = time.perf_counter() - st
260
+ output.ttft = ttft
261
+
262
+ # Decoding phase
263
+ # NOTE: Some completion API might have a last
264
+ # usage summary response without a token so we
265
+ # do not want to include as inter-token-latency
266
+ elif data.get("usage", None) is None:
267
+ output.itl.append(timestamp -
268
+ most_recent_timestamp)
269
+
270
+ most_recent_timestamp = timestamp
271
+ generated_text += data["choices"][0]["text"]
272
273
output.generated_text = generated_text
274
output.success = True
275
output.latency = latency
269
- else:
270
- output.success = False
271
- except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
276
+ except Exception:
277
output.success = False
278
+ exc_info = sys.exc_info()
279
+ output.error = "".join(traceback.format_exception(*exc_info))
280
281
if pbar:
282
pbar.update(1)
@@ -283,7 +290,7 @@ async def async_request_openai_chat_completions(
290
api_url = request_func_input.api_url
291
assert api_url.endswith(
292
"v1/chat/completions"
286
- ), "OpenAI Chat API URL must end with 'v1/chat/completions'."
293
+ ), "OpenAI Chat Completions API URL must end with 'v1/chat/completions'."
294
295
async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
296
assert not request_func_input.use_beam_search
@@ -301,7 +308,7 @@ async def async_request_openai_chat_completions(
308
}
309
headers = {
310
"Content-Type": "application/json",
304
- "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
311
+ "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
312
}
313
314
output = RequestFuncOutput()
@@ -310,15 +317,12 @@ async def async_request_openai_chat_completions(
317
generated_text = ""
318
ttft = 0
319
st = time.perf_counter()
320
+ most_recent_timestamp = st
321
try:
322
async with session.post(url=api_url, json=payload,
323
headers=headers) as response:
324
if response.status == 200:
325
async for chunk in response.content:
318
- if ttft == 0:
319
- ttft = time.perf_counter() - st
320
- output.ttft = ttft
321
-
326
chunk = chunk.strip()
327
if not chunk:
328
continue
@@ -327,18 +331,35 @@ async def async_request_openai_chat_completions(
331
if chunk == "[DONE]":
332
latency = time.perf_counter() - st
333
else:
330
- body = json.loads(chunk)
331
- if "content" in body["choices"][0]["delta"]:
332
- generated_text += body["choices"][0]["delta"][
334
+ timestamp = time.perf_counter()
335
+ data = json.loads(chunk)
336
+
337
+ if "content" in data["choices"][0]["delta"]:
338
+ # First token
339
+ if ttft == 0:
340
+ ttft = time.perf_counter() - st
341
+ output.ttft = ttft
342
+
343
+ # Decoding phase
344
+ else:
345
+ output.itl.append(timestamp -
346
+ most_recent_timestamp)
347
+
348
+ generated_text += data["choices"][0]["delta"][
349
"content"]
350
351
+ most_recent_timestamp = timestamp
352
+
353
output.generated_text = generated_text
354
output.success = True
355
output.latency = latency
356
else:
357
+ output.error = response.reason
358
output.success = False
340
- except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
359
+ except Exception:
360
output.success = False
361
+ exc_info = sys.exc_info()
362
+ output.error = "".join(traceback.format_exception(*exc_info))
363
364
if pbar:
365
pbar.update(1)
@@ -355,7 +376,8 @@ def remove_prefix(text: str, prefix: str) -> str:
376
377
ASYNC_REQUEST_FUNCS = {
378
"tgi": async_request_tgi,
358
- "vllm": async_request_vllm,
379
+ "vllm": async_request_openai_completions,
380
+ "lmdeploy": async_request_openai_completions,
381
"deepspeed-mii": async_request_deepspeed_mii,
382
"openai": async_request_openai_completions,
383
"openai-chat": async_request_openai_chat_completions,
benchmarks/benchmark_serving.py
+252
-51
@@ -1,8 +1,8 @@
1
"""Benchmark online serving throughput.
2
3
On the server side, run one of the following commands:
4
- (vLLM backend)
5
- python -m vllm.entrypoints.api_server \
4
+ vLLM OpenAI API server
5
+ python -m vllm.entrypoints.openai.api_server \
6
--model <your_model> --swap-space 16 \
7
--disable-log-requests
8
@@ -12,14 +12,19 @@ On the server side, run one of the following commands:
12
On the client side, run:
13
python benchmarks/benchmark_serving.py \
14
--backend <backend> \
15
- --model <your_model> --dataset <target_dataset> \
16
- --request-rate <request_rate>
15
+ --model <your_model> \
16
+ --dataset-name sharegpt \
17
+ --dataset-path <path to dataset> \
18
+ --request-rate <request_rate> \ # By default <request_rate> is inf
19
+ --num-prompts <num_prompts> # By default <num_prompts> is 1000
20
"""
21
import argparse
22
import asyncio
23
import json
24
+import os
25
import random
26
import time
27
+import warnings
28
from dataclasses import dataclass
29
from datetime import datetime
30
from typing import AsyncGenerator, List, Tuple
@@ -49,7 +54,7 @@ class BenchmarkMetrics:
54
p99_tpot_ms: float
55
56
52
-def sample_requests(
57
+def sample_sharegpt_requests(
58
dataset_path: str,
59
num_requests: int,
60
tokenizer: PreTrainedTokenizerBase,
@@ -97,6 +102,73 @@ def sample_requests(
102
return sampled_requests
103
104
105
+def sample_sonnet_requests(
106
+ dataset_path: str,
107
+ num_requests: int,
108
+ input_len: int,
109
+ output_len: int,
110
+ prefix_len: int,
111
+ tokenizer: PreTrainedTokenizerBase,
112
+) -> List[Tuple[str, str, int, int]]:
113
+ assert input_len > prefix_len, "input_len must be greater than prefix_len."
114
+
115
+ # Load the dataset.
116
+ with open(dataset_path) as f:
117
+ poem_lines = f.readlines()
118
+
119
+ # Tokenize the poem lines.
120
+ poem_token_ids = tokenizer(poem_lines).input_ids
121
+ average_poem_len = sum(
122
+ len(token_ids) for token_ids in poem_token_ids) / len(poem_token_ids)
123
+
124
+ # Base prefix for all requests.
125
+ base_prompt = "Pick as many lines as you can from these poem lines:\n"
126
+ base_message = [{
127
+ "role": "user",
128
+ "content": base_prompt,
129
+ }]
130
+ base_prompt_formatted = tokenizer.apply_chat_template(
131
+ base_message, add_generation_prompt=True, tokenize=False)
132
+ base_prompt_offset = len(tokenizer(base_prompt_formatted).input_ids)
133
+
134
+ assert (input_len > base_prompt_offset
135
+ ), f"Please set 'args.input-len' higher than {base_prompt_offset}."
136
+ num_input_lines = round(
137
+ (input_len - base_prompt_offset) / average_poem_len)
138
+
139
+ # First approximately `prefix_len` number of tokens in the
140
+ # prompt are fixed poem lines.
141
+ assert (
142
+ prefix_len > base_prompt_offset
143
+ ), f"Please set 'args.prefix-len' higher than {base_prompt_offset}."
144
+
145
+ num_prefix_lines = round(
146
+ (prefix_len - base_prompt_offset) / average_poem_len)
147
+ prefix_lines = poem_lines[:num_prefix_lines]
148
+
149
+ # Sample the rest of lines per request.
150
+ sampled_requests: List[Tuple[str, int, int]] = []
151
+ for _ in range(num_requests):
152
+ sampled_lines = "".join(
153
+ prefix_lines +
154
+ random.sample(poem_lines, num_input_lines - num_prefix_lines))
155
+
156
+ prompt = f"{base_prompt}{sampled_lines}"
157
+ message = [
158
+ {
159
+ "role": "user",
160
+ "content": prompt,
161
+ },
162
+ ]
163
+ prompt_formatted = tokenizer.apply_chat_template(
164
+ message, add_generation_prompt=True, tokenize=False)
165
+ prompt_len = len(tokenizer(prompt_formatted).input_ids)
166
+ sampled_requests.append(
167
+ (prompt, prompt_formatted, prompt_len, output_len))
168
+
169
+ return sampled_requests
170
+
171
+
172
async def get_request(
173
input_requests: List[Tuple[str, int, int]],
174
request_rate: float,
@@ -119,37 +191,42 @@ def calculate_metrics(
191
outputs: List[RequestFuncOutput],
192
dur_s: float,
193
tokenizer: PreTrainedTokenizerBase,
122
-) -> BenchmarkMetrics:
123
- total_output = 0
194
+) -> Tuple[BenchmarkMetrics, List[int]]:
195
+ actual_output_lens = []
196
total_input = 0
197
completed = 0
126
- per_token_latencies = []
198
+ tpots = []
199
ttfts = []
200
for i in range(len(outputs)):
201
if outputs[i].success:
130
- output_len = len(tokenizer.encode(outputs[i].generated_text))
131
- total_output += output_len
202
+ output_len = len(tokenizer(outputs[i].generated_text).input_ids)
203
+ actual_output_lens.append(output_len)
204
total_input += input_requests[i][1]
133
- per_token_latencies.append(outputs[i].latency / output_len)
205
+ if output_len > 1:
206
+ tpots.append(
207
+ (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
208
ttfts.append(outputs[i].ttft)
209
completed += 1
210
+ else:
211
+ actual_output_lens.append(0)
212
213
metrics = BenchmarkMetrics(
214
completed=completed,
215
total_input=total_input,
140
- total_output=total_output,
216
+ total_output=sum(actual_output_lens),
217
request_throughput=completed / dur_s,
218
input_throughput=total_input / dur_s,
143
- output_throughput=total_output / dur_s,
144
- mean_ttft_ms=np.mean(ttfts) * 1000,
145
- median_ttft_ms=np.median(ttfts) * 1000,
146
- p99_ttft_ms=np.percentile(ttfts, 99) * 1000,
147
- mean_tpot_ms=np.mean(per_token_latencies) * 1000,
148
- median_tpot_ms=np.median(per_token_latencies) * 1000,
149
- p99_tpot_ms=np.percentile(per_token_latencies, 99) * 1000,
219
+ output_throughput=sum(actual_output_lens) / dur_s,
220
+ mean_ttft_ms=np.mean(ttfts or 0) *
221
+ 1000, # ttfts is empty if streaming is not supported by backend
222
+ median_ttft_ms=np.median(ttfts or 0) * 1000,
223
+ p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000,
224
+ mean_tpot_ms=np.mean(tpots) * 1000,
225
+ median_tpot_ms=np.median(tpots) * 1000,
226
+ p99_tpot_ms=np.percentile(tpots, 99) * 1000,
227
)
228
152
- return metrics
229
+ return metrics, actual_output_lens
230
231
232
async def benchmark(
@@ -189,40 +266,53 @@ async def benchmark(
266
asyncio.create_task(
267
request_func(request_func_input=request_func_input,
268
pbar=pbar)))
192
- outputs = await asyncio.gather(*tasks)
269
+ outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
270
271
if not disable_tqdm:
272
pbar.close()
273
274
benchmark_duration = time.perf_counter() - benchmark_start_time
275
199
- metrics = calculate_metrics(
276
+ metrics, actual_output_lens = calculate_metrics(
277
input_requests=input_requests,
278
outputs=outputs,
279
dur_s=benchmark_duration,
280
tokenizer=tokenizer,
281
)
282
206
- print(f"Successful requests: {metrics.completed}")
207
- print(f"Benchmark duration: {benchmark_duration:2f} s")
208
- print(f"Total input tokens: {metrics.total_input}")
209
- print(f"Total generated tokens: {metrics.total_output}")
210
- print(f"Request throughput: {metrics.request_throughput:.2f} requests/s")
211
- print(f"Input token throughput: {metrics.input_throughput:.2f} tokens/s")
212
- print(f"Output token throughput: {metrics.output_throughput:.2f} tokens/s")
213
- print(f"Mean TTFT: {metrics.mean_ttft_ms:.2f} ms")
214
- print(f"Median TTFT: {metrics.median_ttft_ms:.2f} ms")
215
- print(f"P99 TTFT: {metrics.p99_ttft_ms:.2f} ms")
216
- print(f"Mean TPOT: {metrics.mean_tpot_ms:.2f} ms")
217
- print(f"Median TPOT: {metrics.median_tpot_ms:.2f} ms")
218
- print(f"P99 TPOT: {metrics.p99_tpot_ms:.2f} ms")
283
+ print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
284
+ print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
285
+ print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
286
+ benchmark_duration))
287
+ print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
288
+ print("{:<40} {:<10}".format("Total generated tokens:",
289
+ metrics.total_output))
290
+ print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
291
+ metrics.request_throughput))
292
+ print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):",
293
+ metrics.input_throughput))
294
+ print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
295
+ metrics.output_throughput))
296
+ print("{s:{c}^{n}}".format(s='Time to First Token', n=50, c='-'))
297
+ print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms))
298
+ print("{:<40} {:<10.2f}".format("Median TTFT (ms):",
299
+ metrics.median_ttft_ms))
300
+ print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms))
301
+ print("{s:{c}^{n}}".format(s='Time per Output Token (excl. 1st token)',
302
+ n=50,
303
+ c='-'))
304
+ print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms))
305
+ print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
306
+ metrics.median_tpot_ms))
307
+ print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
308
+ print("=" * 50)
309
310
result = {
311
"duration": benchmark_duration,
312
"completed": metrics.completed,
313
"total_input_tokens": metrics.total_input,
314
"total_output_tokens": metrics.total_output,
225
- "request_inthroughput": metrics.request_throughput,
315
+ "request_throughput": metrics.request_throughput,
316
"input_throughput": metrics.input_throughput,
317
"output_throughput": metrics.output_throughput,
318
"mean_ttft_ms": metrics.mean_ttft_ms,
@@ -230,7 +320,13 @@ async def benchmark(
320
"p99_ttft_ms": metrics.p99_ttft_ms,
321
"mean_tpot_ms": metrics.mean_tpot_ms,
322
"median_tpot_ms": metrics.median_tpot_ms,
233
- "p99_tpot_ms": metrics.p99_tpot_ms
323
+ "p99_tpot_ms": metrics.p99_tpot_ms,
324
+ "input_lens": [output.prompt_len for output in outputs],
325
+ "output_lens": actual_output_lens,
326
+ "ttfts": [output.ttft for output in outputs],
327
+ "itls": [output.itl for output in outputs],
328
+ "generated_texts": [output.generated_text for output in outputs],
329
+ "errors": [output.error for output in outputs],
330
}
331
return result
332
@@ -251,7 +347,58 @@ def main(args: argparse.Namespace):
347
348
tokenizer = get_tokenizer(tokenizer_id,
349
trust_remote_code=args.trust_remote_code)
254
- input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
350
+
351
+ if args.dataset is not None:
352
+ warnings.warn(
353
+ "The '--dataset' argument will be deprecated in the next "
354
+ "release. Please use '--dataset-name' and "
355
+ "'--dataset-path' in the future runs.",
356
+ stacklevel=2)
357
+ input_requests = sample_sharegpt_requests(
358
+ dataset_path=args.dataset,
359
+ num_requests=args.num_prompts,
360
+ tokenizer=tokenizer,
361
+ )
362
+
363
+ elif args.dataset_name == "sharegpt":
364
+ input_requests = sample_sharegpt_requests(
365
+ dataset_path=args.dataset_path,
366
+ num_requests=args.num_prompts,
367
+ tokenizer=tokenizer,
368
+ )
369
+
370
+ elif args.dataset_name == "sonnet":
371
+ # Do not format the prompt, pass to message directly
372
+ if args.backend == "openai-chat":
373
+ input_requests = sample_sonnet_requests(
374
+ dataset_path=args.dataset_path,
375
+ num_requests=args.num_prompts,
376
+ input_len=args.input_len,
377
+ output_len=args.output_len,
378
+ prefix_len=args.prefix_len,
379
+ tokenizer=tokenizer,
380
+ )
381
+ input_requests = [(prompt, prompt_len, output_len)
382
+ for prompt, prompt_formatted, prompt_len,
383
+ output_len in input_requests]
384
+ else:
385
+ assert (
386
+ tokenizer.chat_template or tokenizer.default_chat_template
387
+ ), "Tokenizer/model must have chat template for sonnet dataset."
388
+ input_requests = sample_sonnet_requests(
389
+ dataset_path=args.dataset_path,
390
+ num_requests=args.num_prompts,
391
+ input_len=args.input_len,
392
+ output_len=args.output_len,
393
+ prefix_len=args.prefix_len,
394
+ tokenizer=tokenizer,
395
+ )
396
+ input_requests = [(prompt_formatted, prompt_len, output_len)
397
+ for prompt, prompt_formatted, prompt_len,
398
+ output_len in input_requests]
399
+
400
+ else:
401
+ raise ValueError(f"Unknown dataset: {args.dataset_name}")
402
403
benchmark_result = asyncio.run(
404
benchmark(
@@ -274,13 +421,23 @@ def main(args: argparse.Namespace):
421
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
422
result_json["date"] = current_dt
423
result_json["backend"] = backend
277
- result_json["version"] = args.version
424
result_json["model_id"] = model_id
425
result_json["tokenizer_id"] = tokenizer_id
426
result_json["best_of"] = args.best_of
427
result_json["use_beam_search"] = args.use_beam_search
428
result_json["num_prompts"] = args.num_prompts
429
430
+ # Metadata
431
+ if args.metadata:
432
+ for item in args.metadata:
433
+ if "=" in item:
434
+ kvstring = item.split("=")
435
+ result_json[kvstring[0].strip()] = kvstring[1].strip()
436
+ else:
437
+ raise ValueError(
438
+ "Invalid metadata format. Please use KEY=VALUE format."
439
+ )
440
+
441
# Traffic
442
result_json["request_rate"] = (
443
args.request_rate if args.request_rate < float("inf") else "inf")
@@ -290,9 +447,9 @@ def main(args: argparse.Namespace):
447
448
# Save to file
449
base_model_id = model_id.split("/")[-1]
293
- file_name = (
294
- f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
295
- )
450
+ file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" #noqa
451
+ if args.result_dir:
452
+ file_name = os.path.join(args.result_dir, file_name)
453
with open(file_name, "w") as outfile:
454
json.dump(result_json, outfile)
455
@@ -306,12 +463,6 @@ if __name__ == "__main__":
463
default="vllm",
464
choices=list(ASYNC_REQUEST_FUNCS.keys()),
465
)
309
- parser.add_argument(
310
- "--version",
311
- type=str,
312
- default="N/A",
313
- help="Version of the serving backend/engine.",
314
- )
466
parser.add_argument(
467
"--base-url",
468
type=str,
@@ -323,12 +474,26 @@ if __name__ == "__main__":
474
parser.add_argument(
475
"--endpoint",
476
type=str,
326
- default="/generate",
477
+ default="/v1/completions",
478
help="API endpoint.",
479
)
329
- parser.add_argument("--dataset",
480
+ parser.add_argument(
481
+ "--dataset",
482
+ type=str,
483
+ default=None,
484
+ help="Path to the ShareGPT dataset, will be deprecated in the "
485
+ "next release.",
486
+ )
487
+ parser.add_argument(
488
+ "--dataset-name",
489
+ type=str,
490
+ default="sharegpt",
491
+ choices=["sharegpt", "sonnet"],
492
+ help="Name of the dataset to benchmark on.",
493
+ )
494
+ parser.add_argument("--dataset-path",
495
type=str,
331
- required=True,
496
+ default=None,
497
help="Path to the dataset.")
498
parser.add_argument(
499
"--model",
@@ -356,6 +521,27 @@ if __name__ == "__main__":
521
default=1000,
522
help="Number of prompts to process.",
523
)
524
+ parser.add_argument(
525
+ "--sonnet-input-len",
526
+ type=int,
527
+ default=550,
528
+ help=
529
+ "Number of input tokens per request, used only for sonnet dataset.",
530
+ )
531
+ parser.add_argument(
532
+ "--sonnet-output-len",
533
+ type=int,
534
+ default=150,
535
+ help=
536
+ "Number of output tokens per request, used only for sonnet dataset.",
537
+ )
538
+ parser.add_argument(
539
+ "--sonnet-prefix-len",
540
+ type=int,
541
+ default=200,
542
+ help=
543
+ "Number of prefix tokens per request, used only for sonnet dataset.",
544
+ )
545
parser.add_argument(
546
"--request-rate",
547
type=float,
@@ -381,6 +567,21 @@ if __name__ == "__main__":
567
action="store_true",
568
help="Specify to save benchmark results to a json file",
569
)
570
+ parser.add_argument(
571
+ "--metadata",
572
+ metavar="KEY=VALUE",
573
+ nargs="*",
574
+ help="Key-value pairs (e.g, --metadata version=0.3.3 tp=1) "
575
+ "for metadata of this run to be saved in the result JSON file "
576
+ "for record keeping purposes.",
577
+ )
578
+ parser.add_argument(
579
+ "--result-dir",
580
+ type=str,
581
+ default=None,
582
+ help="Specify directory to save benchmark json results."
583
+ "If not specified, results are saved in the current directory.",
584
+ )
585
586
args = parser.parse_args()
587
main(args)
benchmarks/sonnet.txt
new
+518
@@ -0,0 +1,518 @@
1
+FROM fairest creatures we desire increase,
2
+That thereby beauty's rose might never die,
3
+But as the riper should by time decease,
4
+His tender heir might bear his memory:
5
+But thou, contracted to thine own bright eyes,
6
+Feed'st thy light'st flame with self-substantial fuel,
7
+Making a famine where abundance lies,
8
+Thyself thy foe, to thy sweet self too cruel.
9
+Thou that art now the world's fresh ornament
10
+And only herald to the gaudy spring,
11
+Within thine own bud buriest thy content
12
+And, tender churl, makest waste in niggarding.
13
+Pity the world, or else this glutton be,
14
+To eat the world's due, by the grave and thee.
15
+When forty winters shall beseige thy brow,
16
+And dig deep trenches in thy beauty's field,
17
+Thy youth's proud livery, so gazed on now,
18
+Will be a tatter'd weed, of small worth held:
19
+Then being ask'd where all thy beauty lies,
20
+Where all the treasure of thy lusty days,
21
+To say, within thine own deep-sunken eyes,
22
+Were an all-eating shame and thriftless praise.
23
+How much more praise deserved thy beauty's use,
24
+If thou couldst answer 'This fair child of mine
25
+Shall sum my count and make my old excuse,'
26
+Proving his beauty by succession thine!
27
+This were to be new made when thou art old,
28
+And see thy blood warm when thou feel'st it cold.
29
+Look in thy glass, and tell the face thou viewest
30
+Now is the time that face should form another;
31
+Whose fresh repair if now thou not renewest,
32
+Thou dost beguile the world, unbless some mother.
33
+For where is she so fair whose unear'd womb
34
+Disdains the tillage of thy husbandry?
35
+Or who is he so fond will be the tomb
36
+Of his self-love, to stop posterity?
37
+Thou art thy mother's glass, and she in thee
38
+Calls back the lovely April of her prime:
39
+So thou through windows of thine age shall see
40
+Despite of wrinkles this thy golden time.
41
+But if thou live, remember'd not to be,
42
+Die single, and thine image dies with thee.
43
+Unthrifty loveliness, why dost thou spend
44
+Upon thyself thy beauty's legacy?
45
+Nature's bequest gives nothing but doth lend,
46
+And being frank she lends to those are free.
47
+Then, beauteous niggard, why dost thou abuse
48
+The bounteous largess given thee to give?
49
+Profitless usurer, why dost thou use
50
+So great a sum of sums, yet canst not live?
51
+For having traffic with thyself alone,
52
+Thou of thyself thy sweet self dost deceive.
53
+Then how, when nature calls thee to be gone,
54
+What acceptable audit canst thou leave?
55
+Thy unused beauty must be tomb'd with thee,
56
+Which, used, lives th' executor to be.
57
+Those hours, that with gentle work did frame
58
+The lovely gaze where every eye doth dwell,
59
+Will play the tyrants to the very same
60
+And that unfair which fairly doth excel:
61
+For never-resting time leads summer on
62
+To hideous winter and confounds him there;
63
+Sap cheque'd with frost and lusty leaves quite gone,
64
+Beauty o'ersnow'd and bareness every where:
65
+Then, were not summer's distillation left,
66
+A liquid prisoner pent in walls of glass,
67
+Beauty's effect with beauty were bereft,
68
+Nor it nor no remembrance what it was:
69
+But flowers distill'd though they with winter meet,
70
+Leese but their show; their substance still lives sweet.
71
+Then let not winter's ragged hand deface
72
+In thee thy summer, ere thou be distill'd:
73
+Make sweet some vial; treasure thou some place
74
+With beauty's treasure, ere it be self-kill'd.
75
+That use is not forbidden usury,
76
+Which happies those that pay the willing loan;
77
+That's for thyself to breed another thee,
78
+Or ten times happier, be it ten for one;
79
+Ten times thyself were happier than thou art,
80
+If ten of thine ten times refigured thee:
81
+Then what could death do, if thou shouldst depart,
82
+Leaving thee living in posterity?
83
+Be not self-will'd, for thou art much too fair
84
+To be death's conquest and make worms thine heir.
85
+Lo! in the orient when the gracious light
86
+Lifts up his burning head, each under eye
87
+Doth homage to his new-appearing sight,
88
+Serving with looks his sacred majesty;
89
+And having climb'd the steep-up heavenly hill,
90
+Resembling strong youth in his middle age,
91
+yet mortal looks adore his beauty still,
92
+Attending on his golden pilgrimage;
93
+But when from highmost pitch, with weary car,
94
+Like feeble age, he reeleth from the day,
95
+The eyes, 'fore duteous, now converted are
96
+From his low tract and look another way:
97
+So thou, thyself out-going in thy noon,
98
+Unlook'd on diest, unless thou get a son.
99
+Music to hear, why hear'st thou music sadly?
100
+Sweets with sweets war not, joy delights in joy.
101
+Why lovest thou that which thou receivest not gladly,
102
+Or else receivest with pleasure thine annoy?
103
+If the true concord of well-tuned sounds,
104
+By unions married, do offend thine ear,
105
+They do but sweetly chide thee, who confounds
106
+In singleness the parts that thou shouldst bear.
107
+Mark how one string, sweet husband to another,
108
+Strikes each in each by mutual ordering,
109
+Resembling sire and child and happy mother
110
+Who all in one, one pleasing note do sing:
111
+Whose speechless song, being many, seeming one,
112
+Sings this to thee: 'thou single wilt prove none.'
113
+Is it for fear to wet a widow's eye
114
+That thou consumest thyself in single life?
115
+Ah! if thou issueless shalt hap to die.
116
+The world will wail thee, like a makeless wife;
117
+The world will be thy widow and still weep
118
+That thou no form of thee hast left behind,
119
+When every private widow well may keep
120
+By children's eyes her husband's shape in mind.
121
+Look, what an unthrift in the world doth spend
122
+Shifts but his place, for still the world enjoys it;
123
+But beauty's waste hath in the world an end,
124
+And kept unused, the user so destroys it.
125
+No love toward others in that bosom sits
126
+That on himself such murderous shame commits.
127
+For shame! deny that thou bear'st love to any,
128
+Who for thyself art so unprovident.
129
+Grant, if thou wilt, thou art beloved of many,
130
+But that thou none lovest is most evident;
131
+For thou art so possess'd with murderous hate
132
+That 'gainst thyself thou stick'st not to conspire.
133
+Seeking that beauteous roof to ruinate
134
+Which to repair should be thy chief desire.
135
+O, change thy thought, that I may change my mind!
136
+Shall hate be fairer lodged than gentle love?
137
+Be, as thy presence is, gracious and kind,
138
+Or to thyself at least kind-hearted prove:
139
+Make thee another self, for love of me,
140
+That beauty still may live in thine or thee.
141
+As fast as thou shalt wane, so fast thou growest
142
+In one of thine, from that which thou departest;
143
+And that fresh blood which youngly thou bestowest
144
+Thou mayst call thine when thou from youth convertest.
145
+Herein lives wisdom, beauty and increase:
146
+Without this, folly, age and cold decay:
147
+If all were minded so, the times should cease
148
+And threescore year would make the world away.
149
+Let those whom Nature hath not made for store,
150
+Harsh featureless and rude, barrenly perish:
151
+Look, whom she best endow'd she gave the more;
152
+Which bounteous gift thou shouldst in bounty cherish:
153
+She carved thee for her seal, and meant thereby
154
+Thou shouldst print more, not let that copy die.
155
+When I do count the clock that tells the time,
156
+And see the brave day sunk in hideous night;
157
+When I behold the violet past prime,
158
+And sable curls all silver'd o'er with white;
159
+When lofty trees I see barren of leaves
160
+Which erst from heat did canopy the herd,
161
+And summer's green all girded up in sheaves
162
+Borne on the bier with white and bristly beard,
163
+Then of thy beauty do I question make,
164
+That thou among the wastes of time must go,
165
+Since sweets and beauties do themselves forsake
166
+And die as fast as they see others grow;
167
+And nothing 'gainst Time's scythe can make defence
168
+Save breed, to brave him when he takes thee hence.
169
+O, that you were yourself! but, love, you are
170
+No longer yours than you yourself here live:
171
+Against this coming end you should prepare,
172
+And your sweet semblance to some other give.
173
+So should that beauty which you hold in lease
174
+Find no determination: then you were
175
+Yourself again after yourself's decease,
176
+When your sweet issue your sweet form should bear.
177
+Who lets so fair a house fall to decay,
178
+Which husbandry in honour might uphold
179
+Against the stormy gusts of winter's day
180
+And barren rage of death's eternal cold?
181
+O, none but unthrifts! Dear my love, you know
182
+You had a father: let your son say so.
183
+Not from the stars do I my judgment pluck;
184
+And yet methinks I have astronomy,
185
+But not to tell of good or evil luck,
186
+Of plagues, of dearths, or seasons' quality;
187
+Nor can I fortune to brief minutes tell,
188
+Pointing to each his thunder, rain and wind,
189
+Or say with princes if it shall go well,
190
+By oft predict that I in heaven find:
191
+But from thine eyes my knowledge I derive,
192
+And, constant stars, in them I read such art
193
+As truth and beauty shall together thrive,
194
+If from thyself to store thou wouldst convert;
195
+Or else of thee this I prognosticate:
196
+Thy end is truth's and beauty's doom and date.
197
+When I consider every thing that grows
198
+Holds in perfection but a little moment,
199
+That this huge stage presenteth nought but shows
200
+Whereon the stars in secret influence comment;
201
+When I perceive that men as plants increase,
202
+Cheered and cheque'd even by the self-same sky,
203
+Vaunt in their youthful sap, at height decrease,
204
+And wear their brave state out of memory;
205
+Then the conceit of this inconstant stay
206
+Sets you most rich in youth before my sight,
207
+Where wasteful Time debateth with Decay,
208
+To change your day of youth to sullied night;
209
+And all in war with Time for love of you,
210
+As he takes from you, I engraft you new.
211
+But wherefore do not you a mightier way
212
+Make war upon this bloody tyrant, Time?
213
+And fortify yourself in your decay
214
+With means more blessed than my barren rhyme?
215
+Now stand you on the top of happy hours,
216
+And many maiden gardens yet unset
217
+With virtuous wish would bear your living flowers,
218
+Much liker than your painted counterfeit:
219
+So should the lines of life that life repair,
220
+Which this, Time's pencil, or my pupil pen,
221
+Neither in inward worth nor outward fair,
222
+Can make you live yourself in eyes of men.
223
+To give away yourself keeps yourself still,
224
+And you must live, drawn by your own sweet skill.
225
+Who will believe my verse in time to come,
226
+If it were fill'd with your most high deserts?
227
+Though yet, heaven knows, it is but as a tomb
228
+Which hides your life and shows not half your parts.
229
+If I could write the beauty of your eyes
230
+And in fresh numbers number all your graces,
231
+The age to come would say 'This poet lies:
232
+Such heavenly touches ne'er touch'd earthly faces.'
233
+So should my papers yellow'd with their age
234
+Be scorn'd like old men of less truth than tongue,
235
+And your true rights be term'd a poet's rage
236
+And stretched metre of an antique song:
237
+But were some child of yours alive that time,
238
+You should live twice; in it and in my rhyme.
239
+Shall I compare thee to a summer's day?
240
+Thou art more lovely and more temperate:
241
+Rough winds do shake the darling buds of May,
242
+And summer's lease hath all too short a date:
243
+Sometime too hot the eye of heaven shines,
244
+And often is his gold complexion dimm'd;
245
+And every fair from fair sometime declines,
246
+By chance or nature's changing course untrimm'd;
247
+But thy eternal summer shall not fade
248
+Nor lose possession of that fair thou owest;
249
+Nor shall Death brag thou wander'st in his shade,
250
+When in eternal lines to time thou growest:
251
+So long as men can breathe or eyes can see,
252
+So long lives this and this gives life to thee.
253
+Devouring Time, blunt thou the lion's paws,
254
+And make the earth devour her own sweet brood;
255
+Pluck the keen teeth from the fierce tiger's jaws,
256
+And burn the long-lived phoenix in her blood;
257
+Make glad and sorry seasons as thou fleets,
258
+And do whate'er thou wilt, swift-footed Time,
259
+To the wide world and all her fading sweets;
260
+But I forbid thee one most heinous crime:
261
+O, carve not with thy hours my love's fair brow,
262
+Nor draw no lines there with thine antique pen;
263
+Him in thy course untainted do allow
264
+For beauty's pattern to succeeding men.
265
+Yet, do thy worst, old Time: despite thy wrong,
266
+My love shall in my verse ever live young.
267
+A woman's face with Nature's own hand painted
268
+Hast thou, the master-mistress of my passion;
269
+A woman's gentle heart, but not acquainted
270
+With shifting change, as is false women's fashion;
271
+An eye more bright than theirs, less false in rolling,
272
+Gilding the object whereupon it gazeth;
273
+A man in hue, all 'hues' in his controlling,
274
+Much steals men's eyes and women's souls amazeth.
275
+And for a woman wert thou first created;
276
+Till Nature, as she wrought thee, fell a-doting,
277
+And by addition me of thee defeated,
278
+By adding one thing to my purpose nothing.
279
+But since she prick'd thee out for women's pleasure,
280
+Mine be thy love and thy love's use their treasure.
281
+So is it not with me as with that Muse
282
+Stirr'd by a painted beauty to his verse,
283
+Who heaven itself for ornament doth use
284
+And every fair with his fair doth rehearse
285
+Making a couplement of proud compare,
286
+With sun and moon, with earth and sea's rich gems,
287
+With April's first-born flowers, and all things rare
288
+That heaven's air in this huge rondure hems.
289
+O' let me, true in love, but truly write,
290
+And then believe me, my love is as fair
291
+As any mother's child, though not so bright
292
+As those gold candles fix'd in heaven's air:
293
+Let them say more than like of hearsay well;
294
+I will not praise that purpose not to sell.
295
+My glass shall not persuade me I am old,
296
+So long as youth and thou are of one date;
297
+But when in thee time's furrows I behold,
298
+Then look I death my days should expiate.
299
+For all that beauty that doth cover thee
300
+Is but the seemly raiment of my heart,
301
+Which in thy breast doth live, as thine in me:
302
+How can I then be elder than thou art?
303
+O, therefore, love, be of thyself so wary
304
+As I, not for myself, but for thee will;
305
+Bearing thy heart, which I will keep so chary
306
+As tender nurse her babe from faring ill.
307
+Presume not on thy heart when mine is slain;
308
+Thou gavest me thine, not to give back again.
309
+As an unperfect actor on the stage
310
+Who with his fear is put besides his part,
311
+Or some fierce thing replete with too much rage,
312
+Whose strength's abundance weakens his own heart.
313
+So I, for fear of trust, forget to say
314
+The perfect ceremony of love's rite,
315
+And in mine own love's strength seem to decay,
316
+O'ercharged with burden of mine own love's might.
317
+O, let my books be then the eloquence
318
+And dumb presagers of my speaking breast,
319
+Who plead for love and look for recompense
320
+More than that tongue that more hath more express'd.
321
+O, learn to read what silent love hath writ:
322
+To hear with eyes belongs to love's fine wit.
323
+Mine eye hath play'd the painter and hath stell'd
324
+Thy beauty's form in table of my heart;
325
+My body is the frame wherein 'tis held,
326
+And perspective it is the painter's art.
327
+For through the painter must you see his skill,
328
+To find where your true image pictured lies;
329
+Which in my bosom's shop is hanging still,
330
+That hath his windows glazed with thine eyes.
331
+Now see what good turns eyes for eyes have done:
332
+Mine eyes have drawn thy shape, and thine for me
333
+Are windows to my breast, where-through the sun
334
+Delights to peep, to gaze therein on thee;
335
+Yet eyes this cunning want to grace their art;
336
+They draw but what they see, know not the heart.
337
+Let those who are in favour with their stars
338
+Of public honour and proud titles boast,
339
+Whilst I, whom fortune of such triumph bars,
340
+Unlook'd for joy in that I honour most.
341
+Great princes' favourites their fair leaves spread
342
+But as the marigold at the sun's eye,
343
+And in themselves their pride lies buried,
344
+For at a frown they in their glory die.
345
+The painful warrior famoused for fight,
346
+After a thousand victories once foil'd,
347
+Is from the book of honour razed quite,
348
+And all the rest forgot for which he toil'd:
349
+Then happy I, that love and am beloved
350
+Where I may not remove nor be removed.
351
+Lord of my love, to whom in vassalage
352
+Thy merit hath my duty strongly knit,
353
+To thee I send this written embassage,
354
+To witness duty, not to show my wit:
355
+Duty so great, which wit so poor as mine
356
+May make seem bare, in wanting words to show it,
357
+But that I hope some good conceit of thine
358
+In thy soul's thought, all naked, will bestow it;
359
+Till whatsoever star that guides my moving
360
+Points on me graciously with fair aspect
361
+And puts apparel on my tatter'd loving,
362
+To show me worthy of thy sweet respect:
363
+Then may I dare to boast how I do love thee;
364
+Till then not show my head where thou mayst prove me.
365
+Weary with toil, I haste me to my bed,
366
+The dear repose for limbs with travel tired;
367
+But then begins a journey in my head,
368
+To work my mind, when body's work's expired:
369
+For then my thoughts, from far where I abide,
370
+Intend a zealous pilgrimage to thee,
371
+And keep my drooping eyelids open wide,
372
+Looking on darkness which the blind do see
373
+Save that my soul's imaginary sight
374
+Presents thy shadow to my sightless view,
375
+Which, like a jewel hung in ghastly night,
376
+Makes black night beauteous and her old face new.
377
+Lo! thus, by day my limbs, by night my mind,
378
+For thee and for myself no quiet find.
379
+How can I then return in happy plight,
380
+That am debarr'd the benefit of rest?
381
+When day's oppression is not eased by night,
382
+But day by night, and night by day, oppress'd?
383
+And each, though enemies to either's reign,
384
+Do in consent shake hands to torture me;
385
+The one by toil, the other to complain
386
+How far I toil, still farther off from thee.
387
+I tell the day, to please them thou art bright
388
+And dost him grace when clouds do blot the heaven:
389
+So flatter I the swart-complexion'd night,
390
+When sparkling stars twire not thou gild'st the even.
391
+But day doth daily draw my sorrows longer
392
+And night doth nightly make grief's strength seem stronger.
393
+When, in disgrace with fortune and men's eyes,
394
+I all alone beweep my outcast state
395
+And trouble deal heaven with my bootless cries
396
+And look upon myself and curse my fate,
397
+Wishing me like to one more rich in hope,
398
+Featured like him, like him with friends possess'd,
399
+Desiring this man's art and that man's scope,
400
+With what I most enjoy contented least;
401
+Yet in these thoughts myself almost despising,
402
+Haply I think on thee, and then my state,
403
+Like to the lark at break of day arising
404
+From sullen earth, sings hymns at heaven's gate;
405
+For thy sweet love remember'd such wealth brings
406
+That then I scorn to change my state with kings.
407
+When to the sessions of sweet silent thought
408
+I summon up remembrance of things past,
409
+I sigh the lack of many a thing I sought,
410
+And with old woes new wail my dear time's waste:
411
+Then can I drown an eye, unused to flow,
412
+For precious friends hid in death's dateless night,
413
+And weep afresh love's long since cancell'd woe,
414
+And moan the expense of many a vanish'd sight:
415
+Then can I grieve at grievances foregone,
416
+And heavily from woe to woe tell o'er
417
+The sad account of fore-bemoaned moan,
418
+Which I new pay as if not paid before.
419
+But if the while I think on thee, dear friend,
420
+All losses are restored and sorrows end.
421
+Thy bosom is endeared with all hearts,
422
+Which I by lacking have supposed dead,
423
+And there reigns love and all love's loving parts,
424
+And all those friends which I thought buried.
425
+How many a holy and obsequious tear
426
+Hath dear religious love stol'n from mine eye
427
+As interest of the dead, which now appear
428
+But things removed that hidden in thee lie!
429
+Thou art the grave where buried love doth live,
430
+Hung with the trophies of my lovers gone,
431
+Who all their parts of me to thee did give;
432
+That due of many now is thine alone:
433
+Their images I loved I view in thee,
434
+And thou, all they, hast all the all of me.
435
+If thou survive my well-contented day,
436
+When that churl Death my bones with dust shall cover,
437
+And shalt by fortune once more re-survey
438
+These poor rude lines of thy deceased lover,
439
+Compare them with the bettering of the time,
440
+And though they be outstripp'd by every pen,
441
+Reserve them for my love, not for their rhyme,
442
+Exceeded by the height of happier men.
443
+O, then vouchsafe me but this loving thought:
444
+'Had my friend's Muse grown with this growing age,
445
+A dearer birth than this his love had brought,
446
+To march in ranks of better equipage:
447
+But since he died and poets better prove,
448
+Theirs for their style I'll read, his for his love.'
449
+Full many a glorious morning have I seen
450
+Flatter the mountain-tops with sovereign eye,
451
+Kissing with golden face the meadows green,
452
+Gilding pale streams with heavenly alchemy;
453
+Anon permit the basest clouds to ride
454
+With ugly rack on his celestial face,
455
+And from the forlorn world his visage hide,
456
+Stealing unseen to west with this disgrace:
457
+Even so my sun one early morn did shine
458
+With all triumphant splendor on my brow;
459
+But out, alack! he was but one hour mine;
460
+The region cloud hath mask'd him from me now.
461
+Yet him for this my love no whit disdaineth;
462
+Suns of the world may stain when heaven's sun staineth.
463
+Why didst thou promise such a beauteous day,
464
+And make me travel forth without my cloak,
465
+To let base clouds o'ertake me in my way,
466
+Hiding thy bravery in their rotten smoke?
467
+'Tis not enough that through the cloud thou break,
468
+To dry the rain on my storm-beaten face,
469
+For no man well of such a salve can speak
470
+That heals the wound and cures not the disgrace:
471
+Nor can thy shame give physic to my grief;
472
+Though thou repent, yet I have still the loss:
473
+The offender's sorrow lends but weak relief
474
+To him that bears the strong offence's cross.
475
+Ah! but those tears are pearl which thy love sheds,
476
+And they are rich and ransom all ill deeds.
477
+No more be grieved at that which thou hast done:
478
+Roses have thorns, and silver fountains mud;
479
+Clouds and eclipses stain both moon and sun,
480
+And loathsome canker lives in sweetest bud.
481
+All men make faults, and even I in this,
482
+Authorizing thy trespass with compare,
483
+Myself corrupting, salving thy amiss,
484
+Excusing thy sins more than thy sins are;
485
+For to thy sensual fault I bring in sense--
486
+Thy adverse party is thy advocate--
487
+And 'gainst myself a lawful plea commence:
488
+Such civil war is in my love and hate
489
+That I an accessary needs must be
490
+To that sweet thief which sourly robs from me.
491
+Let me confess that we two must be twain,
492
+Although our undivided loves are one:
493
+So shall those blots that do with me remain
494
+Without thy help by me be borne alone.
495
+In our two loves there is but one respect,
496
+Though in our lives a separable spite,
497
+Which though it alter not love's sole effect,
498
+Yet doth it steal sweet hours from love's delight.
499
+I may not evermore acknowledge thee,
500
+Lest my bewailed guilt should do thee shame,
501
+Nor thou with public kindness honour me,
502
+Unless thou take that honour from thy name:
503
+But do not so; I love thee in such sort
504
+As, thou being mine, mine is thy good report.
505
+As a decrepit father takes delight
506
+To see his active child do deeds of youth,
507
+So I, made lame by fortune's dearest spite,
508
+Take all my comfort of thy worth and truth.
509
+For whether beauty, birth, or wealth, or wit,
510
+Or any of these all, or all, or more,
511
+Entitled in thy parts do crowned sit,
512
+I make my love engrafted to this store:
513
+So then I am not lame, poor, nor despised,
514
+Whilst that this shadow doth such substance give
515
+That I in thy abundance am sufficed
516
+And by a part of all thy glory live.
517
+Look, what is best, that best I wish in thee:
518
+This wish I have; then ten times happy me!
\ No newline at end of file
pyproject.toml
+1
-1
@@ -50,7 +50,7 @@ exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
50
51
[tool.codespell]
52
ignore-words-list = "dout, te, indicies"
53
-skip = "./tests/prompts"
53
+skip = "./tests/prompts,./benchmarks/sonnet.txt"
54
55
[tool.isort]
56
use_parentheses = true
tests/kernels/test_prefix_prefill.py
+2
-2
@@ -36,8 +36,8 @@ def test_contexted_kv_attention(
36
torch.cuda.manual_seed(0)
37
torch.set_default_device(device)
38
39
- # Need this, otherwise when we capture the graph the process for GPU 1 would
40
- # run on both GPU0 and GPU1 and things would hang
39
+ # Need this, otherwise when we capture the graph the process
40
+ # for GPU 1 would run on both GPU0 and GPU1 and things would hang
41
#
42
# see also similar issue: https://github.com/Dao-AILab/flash-attention/issues/523
43
torch.cuda.set_device(device)