A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Use monotonic time where appropriate (#1249)
Antoni Baum committed
Oct 2, 2023 at 19:22 UTC
acbed3ef40f015fcf64460e629813922fab90380
7 files changed
+18
-17
benchmarks/benchmark_latency.py
+2
-2
@@ -40,13 +40,13 @@ def main(args: argparse.Namespace):
40
def run_to_completion(profile: bool = False):
41
if profile:
42
torch.cuda.cudart().cudaProfilerStart()
43
- start_time = time.time()
43
+ start_time = time.perf_counter()
44
45
llm.generate(prompt_token_ids=dummy_prompt_token_ids,
46
sampling_params=sampling_params,
47
use_tqdm=False)
48
49
- end_time = time.time()
49
+ end_time = time.perf_counter()
50
latency = end_time - start_time
51
if profile:
52
torch.cuda.cudart().cudaProfilerStop()
benchmarks/benchmark_serving.py
+4
-4
@@ -105,7 +105,7 @@ async def send_request(
105
best_of: int,
106
use_beam_search: bool,
107
) -> None:
108
- request_start_time = time.time()
108
+ request_start_time = time.perf_counter()
109
110
headers = {"User-Agent": "Benchmark Client"}
111
if backend == "vllm":
@@ -148,7 +148,7 @@ async def send_request(
148
if "error" not in output:
149
break
150
151
- request_end_time = time.time()
151
+ request_end_time = time.perf_counter()
152
request_latency = request_end_time - request_start_time
153
REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
154
@@ -180,10 +180,10 @@ def main(args: argparse.Namespace):
180
tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
181
input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
182
183
- benchmark_start_time = time.time()
183
+ benchmark_start_time = time.perf_counter()
184
asyncio.run(benchmark(args.backend, api_url, input_requests, args.best_of,
185
args.use_beam_search, args.request_rate))
186
- benchmark_end_time = time.time()
186
+ benchmark_end_time = time.perf_counter()
187
benchmark_time = benchmark_end_time - benchmark_start_time
188
print(f"Total time: {benchmark_time:.2f} s")
189
print(f"Throughput: {args.num_prompts / benchmark_time:.2f} requests/s")
benchmarks/benchmark_throughput.py
+4
-4
@@ -93,10 +93,10 @@ def run_vllm(
93
sampling_params=sampling_params,
94
)
95
96
- start = time.time()
96
+ start = time.perf_counter()
97
# FIXME(woosuk): Do use internal method.
98
llm._run_engine(use_tqdm=True)
99
- end = time.time()
99
+ end = time.perf_counter()
100
return end - start
101
102
@@ -118,7 +118,7 @@ def run_hf(
118
llm = llm.cuda()
119
120
pbar = tqdm(total=len(requests))
121
- start = time.time()
121
+ start = time.perf_counter()
122
batch: List[str] = []
123
max_prompt_len = 0
124
max_output_len = 0
@@ -156,7 +156,7 @@ def run_hf(
156
batch = []
157
max_prompt_len = 0
158
max_output_len = 0
159
- end = time.time()
159
+ end = time.perf_counter()
160
return end - start
161
162
vllm/core/scheduler.py
+1
-1
@@ -121,7 +121,7 @@ class Scheduler:
121
blocks_to_copy: Dict[int, List[int]] = {}
122
123
# Fix the current time.
124
- now = time.time()
124
+ now = time.monotonic()
125
126
# Join waiting sequences if possible.
127
if not self.swapped:
vllm/engine/async_llm_engine.py
+2
-1
@@ -417,7 +417,8 @@ class AsyncLLMEngine:
417
request.
418
"""
419
# Preprocess the request.
420
- arrival_time = time.time()
420
+ # This should not be used for logging, as it is monotonic time.
421
+ arrival_time = time.monotonic()
422
423
try:
424
stream = await self.add_request(request_id,
vllm/engine/llm_engine.py
+3
-3
@@ -256,10 +256,10 @@ class LLMEngine:
256
prompt_token_ids: The token IDs of the prompt. If None, we
257
use the tokenizer to convert the prompts to token IDs.
258
arrival_time: The arrival time of the request. If None, we use
259
- the current time.
259
+ the current monotonic time.
260
"""
261
if arrival_time is None:
262
- arrival_time = time.time()
262
+ arrival_time = time.monotonic()
263
if prompt_token_ids is None:
264
assert prompt is not None
265
prompt_token_ids = self.tokenizer.encode(prompt)
@@ -568,7 +568,7 @@ class LLMEngine:
568
prompt_run: bool,
569
num_batched_tokens: int,
570
) -> None:
571
- now = time.time()
571
+ now = time.monotonic()
572
# Log the number of batched input tokens.
573
if prompt_run:
574
self.num_prompt_tokens.append((now, num_batched_tokens))
vllm/entrypoints/openai/api_server.py
+2
-2
@@ -210,7 +210,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
210
211
model_name = request.model
212
request_id = f"cmpl-{random_uuid()}"
213
- created_time = int(time.time())
213
+ created_time = int(time.monotonic())
214
try:
215
sampling_params = SamplingParams(
216
n=request.n,
@@ -411,7 +411,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
411
if error_check_ret is not None:
412
return error_check_ret
413
414
- created_time = int(time.time())
414
+ created_time = int(time.monotonic())
415
try:
416
sampling_params = SamplingParams(
417
n=request.n,