@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / acbed3ef40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Use monotonic time where appropriate (#1249)

Antoni Baum committed Oct 2, 2023 at 19:22 UTC acbed3ef40f015fcf64460e629813922fab90380
7 files changed +18 -17
benchmarks/benchmark_latency.py
+2 -2
@@ -40,13 +40,13 @@ def main(args: argparse.Namespace):
40 def run_to_completion(profile: bool = False):
41 if profile:
42 torch.cuda.cudart().cudaProfilerStart()
43 - start_time = time.time()
43 + start_time = time.perf_counter()
44
45 llm.generate(prompt_token_ids=dummy_prompt_token_ids,
46 sampling_params=sampling_params,
47 use_tqdm=False)
48
49 - end_time = time.time()
49 + end_time = time.perf_counter()
50 latency = end_time - start_time
51 if profile:
52 torch.cuda.cudart().cudaProfilerStop()
benchmarks/benchmark_serving.py
+4 -4
@@ -105,7 +105,7 @@ async def send_request(
105 best_of: int,
106 use_beam_search: bool,
107 ) -> None:
108 - request_start_time = time.time()
108 + request_start_time = time.perf_counter()
109
110 headers = {"User-Agent": "Benchmark Client"}
111 if backend == "vllm":
@@ -148,7 +148,7 @@ async def send_request(
148 if "error" not in output:
149 break
150
151 - request_end_time = time.time()
151 + request_end_time = time.perf_counter()
152 request_latency = request_end_time - request_start_time
153 REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
154
@@ -180,10 +180,10 @@ def main(args: argparse.Namespace):
180 tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
181 input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
182
183 - benchmark_start_time = time.time()
183 + benchmark_start_time = time.perf_counter()
184 asyncio.run(benchmark(args.backend, api_url, input_requests, args.best_of,
185 args.use_beam_search, args.request_rate))
186 - benchmark_end_time = time.time()
186 + benchmark_end_time = time.perf_counter()
187 benchmark_time = benchmark_end_time - benchmark_start_time
188 print(f"Total time: {benchmark_time:.2f} s")
189 print(f"Throughput: {args.num_prompts / benchmark_time:.2f} requests/s")
benchmarks/benchmark_throughput.py
+4 -4
@@ -93,10 +93,10 @@ def run_vllm(
93 sampling_params=sampling_params,
94 )
95
96 - start = time.time()
96 + start = time.perf_counter()
97 # FIXME(woosuk): Do use internal method.
98 llm._run_engine(use_tqdm=True)
99 - end = time.time()
99 + end = time.perf_counter()
100 return end - start
101
102
@@ -118,7 +118,7 @@ def run_hf(
118 llm = llm.cuda()
119
120 pbar = tqdm(total=len(requests))
121 - start = time.time()
121 + start = time.perf_counter()
122 batch: List[str] = []
123 max_prompt_len = 0
124 max_output_len = 0
@@ -156,7 +156,7 @@ def run_hf(
156 batch = []
157 max_prompt_len = 0
158 max_output_len = 0
159 - end = time.time()
159 + end = time.perf_counter()
160 return end - start
161
162
vllm/core/scheduler.py
+1 -1
@@ -121,7 +121,7 @@ class Scheduler:
121 blocks_to_copy: Dict[int, List[int]] = {}
122
123 # Fix the current time.
124 - now = time.time()
124 + now = time.monotonic()
125
126 # Join waiting sequences if possible.
127 if not self.swapped:
vllm/engine/async_llm_engine.py
+2 -1
@@ -417,7 +417,8 @@ class AsyncLLMEngine:
417 request.
418 """
419 # Preprocess the request.
420 - arrival_time = time.time()
420 + # This should not be used for logging, as it is monotonic time.
421 + arrival_time = time.monotonic()
422
423 try:
424 stream = await self.add_request(request_id,
vllm/engine/llm_engine.py
+3 -3
@@ -256,10 +256,10 @@ class LLMEngine:
256 prompt_token_ids: The token IDs of the prompt. If None, we
257 use the tokenizer to convert the prompts to token IDs.
258 arrival_time: The arrival time of the request. If None, we use
259 - the current time.
259 + the current monotonic time.
260 """
261 if arrival_time is None:
262 - arrival_time = time.time()
262 + arrival_time = time.monotonic()
263 if prompt_token_ids is None:
264 assert prompt is not None
265 prompt_token_ids = self.tokenizer.encode(prompt)
@@ -568,7 +568,7 @@ class LLMEngine:
568 prompt_run: bool,
569 num_batched_tokens: int,
570 ) -> None:
571 - now = time.time()
571 + now = time.monotonic()
572 # Log the number of batched input tokens.
573 if prompt_run:
574 self.num_prompt_tokens.append((now, num_batched_tokens))
vllm/entrypoints/openai/api_server.py
+2 -2
@@ -210,7 +210,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
210
211 model_name = request.model
212 request_id = f"cmpl-{random_uuid()}"
213 - created_time = int(time.time())
213 + created_time = int(time.monotonic())
214 try:
215 sampling_params = SamplingParams(
216 n=request.n,
@@ -411,7 +411,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
411 if error_check_ret is not None:
412 return error_check_ret
413
414 - created_time = int(time.time())
414 + created_time = int(time.monotonic())
415 try:
416 sampling_params = SamplingParams(
417 n=request.n,