A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fixes the misuse/mixuse of time.time()/time.monotonic() (#3220)
Signed-off-by: Tao He <sighingnow@gmail.com> Co-authored-by: simon-mo <simon.mo@hey.com>
Tao He committed
Mar 16, 2024 at 02:25 UTC
14b8ae02e74aa7223a25cf914b61e0a76e3cad87
5 files changed
+7
-8
vllm/core/scheduler.py
+1
-1
@@ -160,7 +160,7 @@ class Scheduler:
160
blocks_to_copy: Dict[int, List[int]] = {}
161
162
# Fix the current time.
163
- now = time.monotonic()
163
+ now = time.time()
164
165
# Join waiting sequences if possible.
166
if not self.swapped:
vllm/engine/async_llm_engine.py
+1
-2
@@ -604,8 +604,7 @@ class AsyncLLMEngine:
604
>>> ...
605
"""
606
# Preprocess the request.
607
- # This should not be used for logging, as it is monotonic time.
608
- arrival_time = time.monotonic()
607
+ arrival_time = time.time()
608
609
try:
610
stream = await self.add_request(
vllm/engine/llm_engine.py
+2
-2
@@ -244,7 +244,7 @@ class LLMEngine:
244
raise ValueError(f"Cannot request more than "
245
f"{max_logprobs} logprobs.")
246
if arrival_time is None:
247
- arrival_time = time.monotonic()
247
+ arrival_time = time.time()
248
prompt_token_ids = self.encode_request(
249
request_id=request_id,
250
prompt=prompt,
@@ -628,7 +628,7 @@ class LLMEngine:
628
def _get_stats(self,
629
scheduler_outputs: Optional[SchedulerOutputs]) -> Stats:
630
"""Get Stats to be Logged to Prometheus."""
631
- now = time.monotonic()
631
+ now = time.time()
632
633
# KV Cache Usage in %.
634
num_total_gpu = self.cache_config.num_gpu_blocks
vllm/entrypoints/openai/serving_chat.py
+2
-2
@@ -103,7 +103,7 @@ class OpenAIServingChat(OpenAIServing):
103
) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
104
105
model_name = request.model
106
- created_time = int(time.monotonic())
106
+ created_time = int(time.time())
107
chunk_object_type = "chat.completion.chunk"
108
first_iteration = True
109
@@ -244,7 +244,7 @@ class OpenAIServingChat(OpenAIServing):
244
request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
245
246
model_name = request.model
247
- created_time = int(time.monotonic())
247
+ created_time = int(time.time())
248
final_res: RequestOutput = None
249
250
async for res in result_generator:
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -118,7 +118,7 @@ class OpenAIServingCompletion(OpenAIServing):
118
119
model_name = request.model
120
request_id = f"cmpl-{random_uuid()}"
121
- created_time = int(time.monotonic())
121
+ created_time = int(time.time())
122
123
# Schedule the request and get the result generator.
124
generators = []