A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Scheduler] Warning upon preemption and Swapping (#4647)
Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>
SangBin Cho committed
May 13, 2024 at 23:50 UTC
e7c46b9527c9a50253657fd0078a0b1f23560ce4
7 files changed
+108
-3
docs/source/models/performance.rst
+19
@@ -3,6 +3,25 @@
3
Performance and Tuning
4
======================
5
6
+Preemption
7
+----------
8
+Due to the auto-regressive nature of transformer architecture, there are times when KV cache space is insufficient to handle all batched requests.
9
+The vLLM can preempt requests to free up KV cache space for other requests. Preempted requests are recomputed when sufficient KV cache space becomes
10
+available again. When this occurs, the following warning is printed:
11
+
12
+```
13
+WARNING 05-09 00:49:33 scheduler.py:1057] Sequence group 0 is preempted by PreemptionMode.SWAP mode because there is not enough KV cache space. This can affect the end-to-end performance. Increase gpu_memory_utilization or tensor_parallel_size to provide more KV cache memory. total_cumulative_preemption_cnt=1
14
+```
15
+
16
+While this mechanism ensures system robustness, preemption and recomputation can adversely affect end-to-end latency.
17
+If you frequently encounter preemptions from the vLLM engine, consider the following actions:
18
+
19
+- Increase `gpu_memory_utilization`. The vLLM pre-allocates GPU cache by using gpu_memory_utilization% of memory. By increasing this utilization, you can provide more KV cache space.
20
+- Decrease `max_num_seqs` or `max_num_batched_tokens`. This can reduce the number of concurrent requests in a batch, thereby requiring less KV cache space.
21
+- Increase `tensor_parallel_size`. This approach shards model weights, so each GPU has more memory available for KV cache.
22
+
23
+You can also monitor the number of preemption requests through Prometheus metrics exposed by the vLLM. Additionally, you can log the cumulative number of preemption requests by setting disable_log_stats=False.
24
+
25
Chunked Prefill
26
---------------
27
vLLM supports an experimental feature chunked prefill. Chunked prefill allows to chunk large prefills into smaller chunks and batch them together with decode requests.
tests/basic_correctness/test_preemption.py
+43
-1
@@ -6,6 +6,7 @@ Run `VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1
6
pytest tests/basic_correctness/test_preemption.py`.
7
"""
8
import pytest
9
+from prometheus_client import REGISTRY
10
11
from vllm import SamplingParams
12
from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
@@ -71,6 +72,7 @@ def test_chunked_prefill_recompute(
72
@pytest.mark.parametrize("dtype", ["float"])
73
@pytest.mark.parametrize("max_tokens", [96])
74
def test_preemption(
75
+ caplog_vllm,
76
hf_runner,
77
vllm_runner,
78
example_prompts,
@@ -87,10 +89,13 @@ def test_preemption(
89
vllm_model = vllm_runner(
90
model,
91
dtype=dtype,
92
+ disable_log_stats=False,
93
)
94
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
95
assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
96
ARTIFICIAL_PREEMPTION_MAX_CNT)
97
+ total_preemption = (
98
+ vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
99
del vllm_model
100
101
for i in range(len(example_prompts)):
@@ -100,6 +105,20 @@ def test_preemption(
105
f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
106
assert hf_output_ids == vllm_output_ids, (
107
f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
108
+ assert ("is preempted by PreemptionMode.RECOMPUTE mode because there "
109
+ "is not enough KV cache space." in caplog_vllm.text)
110
+ # Ensure the count bucket of request-level histogram metrics matches
111
+ # the number of requests as a simple sanity check to ensure metrics are
112
+ # generated
113
+ preemption_metrics = None
114
+ for m in REGISTRY.collect():
115
+ if m.name == "vllm:num_preemptions":
116
+ preemption_metrics = m
117
+ assert preemption_metrics is not None
118
+ total_recorded_preemption = 0
119
+ for sample in preemption_metrics.samples:
120
+ total_recorded_preemption += sample.value
121
+ assert total_preemption == total_recorded_preemption
122
123
124
@pytest.mark.parametrize("model", MODELS)
@@ -107,6 +126,7 @@ def test_preemption(
126
@pytest.mark.parametrize("max_tokens", [96])
127
@pytest.mark.parametrize("beam_width", [4])
128
def test_swap(
129
+ caplog_vllm,
130
hf_runner,
131
vllm_runner,
132
example_prompts,
@@ -122,11 +142,18 @@ def test_swap(
142
max_tokens)
143
del hf_model
144
125
- vllm_model = vllm_runner(model, dtype=dtype, swap_space=10)
145
+ vllm_model = vllm_runner(
146
+ model,
147
+ dtype=dtype,
148
+ swap_space=10,
149
+ disable_log_stats=False,
150
+ )
151
vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
152
max_tokens)
153
assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
154
ARTIFICIAL_PREEMPTION_MAX_CNT)
155
+ total_preemption = (
156
+ vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
157
del vllm_model
158
159
for i in range(len(example_prompts)):
@@ -138,6 +165,21 @@ def test_swap(
165
f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
166
f"vLLM: {vllm_output_ids}")
167
168
+ assert ("is preempted by PreemptionMode.SWAP mode because there "
169
+ "is not enough KV cache space." in caplog_vllm.text)
170
+ # Ensure the count bucket of request-level histogram metrics matches
171
+ # the number of requests as a simple sanity check to ensure metrics are
172
+ # generated
173
+ preemption_metrics = None
174
+ for m in REGISTRY.collect():
175
+ if m.name == "vllm:num_preemptions":
176
+ preemption_metrics = m
177
+ assert preemption_metrics is not None
178
+ total_recorded_preemption = 0
179
+ for sample in preemption_metrics.samples:
180
+ total_recorded_preemption += sample.value
181
+ assert total_preemption == total_recorded_preemption
182
+
183
184
@pytest.mark.parametrize("model", MODELS)
185
@pytest.mark.parametrize("dtype", ["float"])
tests/conftest.py
+16
@@ -499,3 +499,19 @@ def get_tokenizer_pool_config(tokenizer_group_type):
499
pool_type="ray",
500
extra_config={})
501
raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
502
+
503
+
504
+@pytest.fixture()
505
+def temporary_enable_log_propagate():
506
+ import logging
507
+ logger = logging.getLogger("vllm")
508
+ logger.propagate = True
509
+ yield
510
+ logger.propagate = False
511
+
512
+
513
+@pytest.fixture()
514
+def caplog_vllm(temporary_enable_log_propagate, caplog):
515
+ # To capture vllm log, we should enable propagate=True temporarily
516
+ # because caplog depends on logs propagated to the root logger.
517
+ yield caplog
tests/core/test_scheduler.py
+1
@@ -180,6 +180,7 @@ def test_scheduler_schedule_preempt_abort():
180
and not out.blocks_to_swap_out)
181
assert len(seq_group_meta) == 1
182
assert scheduler.get_num_unfinished_seq_groups() == 2
183
+ assert out.preempted == 1
184
185
# Abort seq group a. Re-schedule seq group b prompt with recomputation.
186
scheduler.abort_seq_group("1")
vllm/core/scheduler.py
+18
@@ -129,6 +129,7 @@ class SchedulerOutputs:
129
num_lookahead_slots: int
130
# The number of requests in the running queue
131
running_queue_size: int
132
+ preempted: int
133
134
def __post_init__(self):
135
# Swap in and swap out should never happen at the same time.
@@ -310,6 +311,7 @@ class Scheduler:
311
self.artificial_preempt_cnt = (ARTIFICIAL_PREEMPTION_MAX_CNT
312
if self.enable_artificial_preemption
313
else 0)
314
+ self.num_cumulative_preemption: int = 0
315
316
@property
317
def lora_enabled(self) -> bool:
@@ -785,6 +787,8 @@ class Scheduler:
787
# Update swapped requests.
788
self.swapped = remaining_swapped
789
self.swapped.extend(running_scheduled.swapped_out)
790
+ preempted = (len(running_scheduled.preempted) +
791
+ len(running_scheduled.swapped_out))
792
793
# There should be no prefill from running queue because this policy
794
# doesn't allow chunked prefills.
@@ -804,6 +808,7 @@ class Scheduler:
808
swapped_in.infeasible_seq_groups,
809
num_lookahead_slots=running_scheduled.num_lookahead_slots,
810
running_queue_size=len(self.running),
811
+ preempted=preempted,
812
)
813
814
def _schedule_chunked_prefill(self):
@@ -891,6 +896,8 @@ class Scheduler:
896
ignored_seq_groups=prefills.ignored_seq_groups,
897
num_lookahead_slots=running_scheduled.num_lookahead_slots,
898
running_queue_size=len(self.running),
899
+ preempted=(len(running_scheduled.preempted) +
900
+ len(running_scheduled.swapped_out)),
901
)
902
903
def _schedule(self) -> SchedulerOutputs:
@@ -1057,6 +1064,17 @@ class Scheduler:
1064
preemption_mode = PreemptionMode.RECOMPUTE
1065
else:
1066
preemption_mode = PreemptionMode.SWAP
1067
+
1068
+ if self.num_cumulative_preemption % 50 == 0:
1069
+ logger.warning(
1070
+ "Sequence group %s is preempted by %s mode because there is "
1071
+ "not enough KV cache space. This can affect the end-to-end "
1072
+ "performance. Increase gpu_memory_utilization or "
1073
+ "tensor_parallel_size to provide more KV cache memory. "
1074
+ "total_num_cumulative_preemption=%d", seq_group.request_id,
1075
+ preemption_mode, self.num_cumulative_preemption + 1)
1076
+ self.num_cumulative_preemption += 1
1077
+
1078
if preemption_mode == PreemptionMode.RECOMPUTE:
1079
self._preempt_by_recompute(seq_group)
1080
elif preemption_mode == PreemptionMode.SWAP:
vllm/engine/llm_engine.py
+3
-1
@@ -737,6 +737,8 @@ class LLMEngine:
737
num_generation_tokens_iter = 0
738
time_to_first_tokens_iter: List[float] = []
739
time_per_output_tokens_iter: List[float] = []
740
+ num_preemption_iter = (0 if scheduler_outputs is None else
741
+ scheduler_outputs.preempted)
742
743
# Request stats
744
# Latency
@@ -830,7 +832,6 @@ class LLMEngine:
832
833
return Stats(
834
now=now,
833
-
835
# System stats
836
# Scheduler State
837
num_running_sys=num_running_sys,
@@ -846,6 +847,7 @@ class LLMEngine:
847
time_to_first_tokens_iter=time_to_first_tokens_iter,
848
time_per_output_tokens_iter=time_per_output_tokens_iter,
849
spec_decode_metrics=spec_decode_metrics,
850
+ num_preemption_iter=num_preemption_iter,
851
852
# Request stats
853
# Latency
vllm/engine/metrics.py
+8
-1
@@ -61,6 +61,10 @@ class Metrics:
61
labelnames=labelnames)
62
63
# Iteration stats
64
+ self.counter_num_preemption = Counter(
65
+ name="vllm:num_preemptions_total",
66
+ documentation="Cumulative number of preemption from the engine.",
67
+ labelnames=labelnames)
68
self.counter_prompt_tokens = Counter(
69
name="vllm:prompt_tokens_total",
70
documentation="Number of prefill tokens processed.",
@@ -181,6 +185,7 @@ class Stats:
185
num_generation_tokens_iter: int
186
time_to_first_tokens_iter: List[float]
187
time_per_output_tokens_iter: List[float]
188
+ num_preemption_iter: int
189
190
# Request stats (should have _requests suffix)
191
# Latency
@@ -244,6 +249,8 @@ class StatLogger:
249
stats.cpu_cache_usage_sys)
250
251
# Iteration level data
252
+ self._log_counter(self.metrics.counter_num_preemption,
253
+ stats.num_preemption_iter)
254
self._log_counter(self.metrics.counter_prompt_tokens,
255
stats.num_prompt_tokens_iter)
256
self._log_counter(self.metrics.counter_generation_tokens,
@@ -336,7 +343,7 @@ class StatLogger:
343
"Avg generation throughput: %.1f tokens/s, "
344
"Running: %d reqs, Swapped: %d reqs, "
345
"Pending: %d reqs, GPU KV cache usage: %.1f%%, "
339
- "CPU KV cache usage: %.1f%%",
346
+ "CPU KV cache usage: %.1f%%.",
347
prompt_throughput,
348
generation_throughput,
349
stats.num_running_sys,