A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[MISC][Bugfix] Use less CPU when message queue has been empty for some time (#16226)
Signed-off-by: Povilas Kanapickas <povilas@radix.lt>
Povilas Kanapickas committed
Jun 5, 2025 at 19:53 UTC
85e2b7bb1380dd7096e0d6b64b0d7633b0a9db4a
3 files changed
+70
-11
tests/basic_correctness/test_basic_correctness.py
+19
-9
@@ -128,15 +128,21 @@ def test_models(
128
@multi_gpu_test(num_gpus=2)
129
@pytest.mark.parametrize(
130
"model, distributed_executor_backend, attention_backend, "
131
- "test_suite", [
132
- ("distilbert/distilgpt2", "ray", "", "L4"),
133
- ("distilbert/distilgpt2", "mp", "", "L4"),
134
- ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4"),
135
- ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4"),
136
- ("distilbert/distilgpt2", "ray", "", "A100"),
137
- ("distilbert/distilgpt2", "mp", "", "A100"),
138
- ("distilbert/distilgpt2", "mp", "FLASHINFER", "A100"),
139
- ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100"),
131
+ "test_suite, extra_env", [
132
+ ("distilbert/distilgpt2", "ray", "", "L4", {}),
133
+ ("distilbert/distilgpt2", "mp", "", "L4", {}),
134
+ ("distilbert/distilgpt2", "ray", "", "L4", {
135
+ "VLLM_SLEEP_WHEN_IDLE": "1"
136
+ }),
137
+ ("distilbert/distilgpt2", "mp", "", "L4", {
138
+ "VLLM_SLEEP_WHEN_IDLE": "1"
139
+ }),
140
+ ("meta-llama/Llama-3.2-1B-Instruct", "ray", "", "L4", {}),
141
+ ("meta-llama/Llama-3.2-1B-Instruct", "mp", "", "L4", {}),
142
+ ("distilbert/distilgpt2", "ray", "", "A100", {}),
143
+ ("distilbert/distilgpt2", "mp", "", "A100", {}),
144
+ ("distilbert/distilgpt2", "mp", "FLASHINFER", "A100", {}),
145
+ ("meta-llama/Meta-Llama-3-8B", "ray", "FLASHINFER", "A100", {}),
146
])
147
@pytest.mark.parametrize("enable_prompt_embeds", [True, False])
148
def test_models_distributed(
@@ -148,6 +154,7 @@ def test_models_distributed(
154
distributed_executor_backend: str,
155
attention_backend: str,
156
test_suite: str,
157
+ extra_env: dict[str, str],
158
enable_prompt_embeds: bool,
159
) -> None:
160
@@ -173,6 +180,9 @@ def test_models_distributed(
180
attention_backend,
181
)
182
183
+ for k, v in extra_env.items():
184
+ monkeypatch_context.setenv(k, v)
185
+
186
dtype = "half"
187
max_tokens = 5
188
vllm/distributed/device_communicators/shm_broadcast.py
+45
-2
@@ -28,6 +28,43 @@ VLLM_RINGBUFFER_WARNING_INTERVAL = envs.VLLM_RINGBUFFER_WARNING_INTERVAL
28
logger = init_logger(__name__)
29
30
31
+class SpinTimer:
32
+
33
+ def record_activity(self):
34
+ pass
35
+
36
+ def spin(self):
37
+ sched_yield()
38
+
39
+
40
+class SpinSleepTimer(SpinTimer):
41
+ """
42
+ In setups which have long inactivity periods it is desirable to reduce
43
+ system power consumption when vllm does nothing. This would lead to more
44
+ CPU thermal headroom when a request eventually comes, especially when
45
+ multiple GPUs are connected as each GPU would otherwise pin one thread at
46
+ 100% CPU usage.
47
+
48
+ The simplest solution is to reduce polling frequency when there is no
49
+ activity for a certain period of time.
50
+ """
51
+
52
+ def __init__(self, busy_loop_s: float = 3.0, wait_sleep_s: float = 0.1):
53
+ self.last_activity = time.monotonic()
54
+ self.busy_loop_s = busy_loop_s
55
+ self.wait_sleep_s = wait_sleep_s
56
+
57
+ def record_activity(self):
58
+ self.last_activity = time.monotonic()
59
+
60
+ def spin(self):
61
+ curr_time = time.monotonic()
62
+ if curr_time >= self.last_activity + self.busy_loop_s:
63
+ time.sleep(self.wait_sleep_s)
64
+ else:
65
+ sched_yield()
66
+
67
+
68
class ShmRingBuffer:
69
70
def __init__(self,
@@ -42,7 +79,7 @@ class ShmRingBuffer:
79
of items that can be stored in the buffer are known in advance.
80
In this case, we don't need to synchronize the access to
81
the buffer.
45
-
82
+
83
Buffer memory layout:
84
data metadata
85
| |
@@ -238,6 +275,7 @@ class MessageQueue:
275
self.local_reader_rank = -1
276
# rank does not matter for remote readers
277
self._is_remote_reader = False
278
+ self._read_spin_timer = SpinTimer()
279
280
self.handle = Handle(
281
local_reader_ranks=local_reader_ranks,
@@ -276,6 +314,9 @@ class MessageQueue:
314
self.local_socket.connect(socket_addr)
315
316
self.remote_socket = None
317
+
318
+ self._read_spin_timer = SpinSleepTimer(
319
+ ) if envs.VLLM_SLEEP_WHEN_IDLE else SpinTimer()
320
else:
321
self.buffer = None # type: ignore
322
self.current_idx = -1
@@ -407,7 +448,7 @@ class MessageQueue:
448
# we need to wait until it is written
449
450
# Release the processor to other threads
410
- sched_yield()
451
+ self._read_spin_timer.spin()
452
453
# if we wait for a long time, log a message
454
if (time.monotonic() - start_time
@@ -438,6 +479,8 @@ class MessageQueue:
479
metadata_buffer[self.local_reader_rank + 1] = 1
480
self.current_idx = (self.current_idx +
481
1) % self.buffer.max_chunks
482
+
483
+ self._read_spin_timer.record_activity()
484
break
485
486
def enqueue(self, obj, timeout: Optional[float] = None):
vllm/envs.py
+6
@@ -122,6 +122,7 @@ if TYPE_CHECKING:
122
VLLM_ALL2ALL_BACKEND: str = "naive"
123
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
124
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
125
+ VLLM_SLEEP_WHEN_IDLE: bool = False
126
127
128
def get_default_cache_root():
@@ -841,6 +842,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
842
# Regex timeout for use by the vLLM tool parsing plugins.
843
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS":
844
lambda: int(os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")),
845
+
846
+ # Reduce CPU usage when vLLM is idle. Enabling this will incur small
847
+ # latency penalty when a request eventually comes.
848
+ "VLLM_SLEEP_WHEN_IDLE":
849
+ lambda: bool(int(os.getenv("VLLM_SLEEP_WHEN_IDLE", "0"))),
850
}
851
852
# --8<-- [end:env-vars-definition]