@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14b8ae02e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fixes the misuse/mixuse of time.time()/time.monotonic() (#3220)

Signed-off-by: Tao He <sighingnow@gmail.com> Co-authored-by: simon-mo <simon.mo@hey.com>

Tao He committed Mar 16, 2024 at 02:25 UTC 14b8ae02e74aa7223a25cf914b61e0a76e3cad87
5 files changed +7 -8
vllm/core/scheduler.py
+1 -1
@@ -160,7 +160,7 @@ class Scheduler:
160 blocks_to_copy: Dict[int, List[int]] = {}
161
162 # Fix the current time.
163 - now = time.monotonic()
163 + now = time.time()
164
165 # Join waiting sequences if possible.
166 if not self.swapped:
vllm/engine/async_llm_engine.py
+1 -2
@@ -604,8 +604,7 @@ class AsyncLLMEngine:
604 >>> ...
605 """
606 # Preprocess the request.
607 - # This should not be used for logging, as it is monotonic time.
608 - arrival_time = time.monotonic()
607 + arrival_time = time.time()
608
609 try:
610 stream = await self.add_request(
vllm/engine/llm_engine.py
+2 -2
@@ -244,7 +244,7 @@ class LLMEngine:
244 raise ValueError(f"Cannot request more than "
245 f"{max_logprobs} logprobs.")
246 if arrival_time is None:
247 - arrival_time = time.monotonic()
247 + arrival_time = time.time()
248 prompt_token_ids = self.encode_request(
249 request_id=request_id,
250 prompt=prompt,
@@ -628,7 +628,7 @@ class LLMEngine:
628 def _get_stats(self,
629 scheduler_outputs: Optional[SchedulerOutputs]) -> Stats:
630 """Get Stats to be Logged to Prometheus."""
631 - now = time.monotonic()
631 + now = time.time()
632
633 # KV Cache Usage in %.
634 num_total_gpu = self.cache_config.num_gpu_blocks
vllm/entrypoints/openai/serving_chat.py
+2 -2
@@ -103,7 +103,7 @@ class OpenAIServingChat(OpenAIServing):
103 ) -> Union[ErrorResponse, AsyncGenerator[str, None]]:
104
105 model_name = request.model
106 - created_time = int(time.monotonic())
106 + created_time = int(time.time())
107 chunk_object_type = "chat.completion.chunk"
108 first_iteration = True
109
@@ -244,7 +244,7 @@ class OpenAIServingChat(OpenAIServing):
244 request_id: str) -> Union[ErrorResponse, ChatCompletionResponse]:
245
246 model_name = request.model
247 - created_time = int(time.monotonic())
247 + created_time = int(time.time())
248 final_res: RequestOutput = None
249
250 async for res in result_generator:
vllm/entrypoints/openai/serving_completion.py
+1 -1
@@ -118,7 +118,7 @@ class OpenAIServingCompletion(OpenAIServing):
118
119 model_name = request.model
120 request_id = f"cmpl-{random_uuid()}"
121 - created_time = int(time.monotonic())
121 + created_time = int(time.time())
122
123 # Schedule the request and get the result generator.
124 generators = []