A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Core] Use os.sched_yield in ShmRingBuffer instead of time.sleep (#9994)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Nov 4, 2024 at 20:08 UTC
04bbf38e05fe75539577184f6ca776df39e70dcd
1 file changed
+5
-10
vllm/distributed/device_communicators/shm_broadcast.py
+5
-10
@@ -1,3 +1,4 @@
1
+import os
2
import pickle
3
import time
4
from contextlib import contextmanager
@@ -18,12 +19,6 @@ from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address
19
20
VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
21
21
-# time to wait if the queue is full or empty
22
-# if we sleep for too short, it will consume too much CPU
23
-# if we sleep for too long, it will slow down the writer/reader
24
-# 0.1 us is a good balance
25
-RINGBUFFER_SLEEP_INTERVAL = 1e-7
26
-
22
logger = init_logger(__name__)
23
24
@@ -333,8 +328,8 @@ class MessageQueue:
328
# if this block is not ready to write,
329
# we need to wait until it is read by all readers
330
336
- # wait for a while
337
- time.sleep(RINGBUFFER_SLEEP_INTERVAL)
331
+ # Release the processor to other threads
332
+ os.sched_yield()
333
334
# if we wait for a long time, we should warn the user
335
if (time.monotonic() - start_time >
@@ -387,8 +382,8 @@ class MessageQueue:
382
# if this block is not ready,
383
# we need to wait until it is written
384
390
- # wait for a while
391
- time.sleep(RINGBUFFER_SLEEP_INTERVAL)
385
+ # Release the processor to other threads
386
+ os.sched_yield()
387
388
# if we wait for a long time, we should warn the user
389
if (time.monotonic() - start_time >