@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04bbf38e05
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Core] Use os.sched_yield in ShmRingBuffer instead of time.sleep (#9994)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Nov 4, 2024 at 20:08 UTC 04bbf38e05fe75539577184f6ca776df39e70dcd
1 file changed +5 -10
vllm/distributed/device_communicators/shm_broadcast.py
+5 -10
@@ -1,3 +1,4 @@
1 +import os
2 import pickle
3 import time
4 from contextlib import contextmanager
@@ -18,12 +19,6 @@ from vllm.utils import get_ip, get_open_port, is_valid_ipv6_address
19
20 VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
21
21 -# time to wait if the queue is full or empty
22 -# if we sleep for too short, it will consume too much CPU
23 -# if we sleep for too long, it will slow down the writer/reader
24 -# 0.1 us is a good balance
25 -RINGBUFFER_SLEEP_INTERVAL = 1e-7
26 -
22 logger = init_logger(__name__)
23
24
@@ -333,8 +328,8 @@ class MessageQueue:
328 # if this block is not ready to write,
329 # we need to wait until it is read by all readers
330
336 - # wait for a while
337 - time.sleep(RINGBUFFER_SLEEP_INTERVAL)
331 + # Release the processor to other threads
332 + os.sched_yield()
333
334 # if we wait for a long time, we should warn the user
335 if (time.monotonic() - start_time >
@@ -387,8 +382,8 @@ class MessageQueue:
382 # if this block is not ready,
383 # we need to wait until it is written
384
390 - # wait for a while
391 - time.sleep(RINGBUFFER_SLEEP_INTERVAL)
385 + # Release the processor to other threads
386 + os.sched_yield()
387
388 # if we wait for a long time, we should warn the user
389 if (time.monotonic() - start_time >