@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7c46b9527
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Scheduler] Warning upon preemption and Swapping (#4647)

Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com>

SangBin Cho committed May 13, 2024 at 23:50 UTC e7c46b9527c9a50253657fd0078a0b1f23560ce4
7 files changed +108 -3
docs/source/models/performance.rst
+19
@@ -3,6 +3,25 @@
3 Performance and Tuning
4 ======================
5
6 +Preemption
7 +----------
8 +Due to the auto-regressive nature of transformer architecture, there are times when KV cache space is insufficient to handle all batched requests.
9 +The vLLM can preempt requests to free up KV cache space for other requests. Preempted requests are recomputed when sufficient KV cache space becomes
10 +available again. When this occurs, the following warning is printed:
11 +
12 +```
13 +WARNING 05-09 00:49:33 scheduler.py:1057] Sequence group 0 is preempted by PreemptionMode.SWAP mode because there is not enough KV cache space. This can affect the end-to-end performance. Increase gpu_memory_utilization or tensor_parallel_size to provide more KV cache memory. total_cumulative_preemption_cnt=1
14 +```
15 +
16 +While this mechanism ensures system robustness, preemption and recomputation can adversely affect end-to-end latency.
17 +If you frequently encounter preemptions from the vLLM engine, consider the following actions:
18 +
19 +- Increase `gpu_memory_utilization`. The vLLM pre-allocates GPU cache by using gpu_memory_utilization% of memory. By increasing this utilization, you can provide more KV cache space.
20 +- Decrease `max_num_seqs` or `max_num_batched_tokens`. This can reduce the number of concurrent requests in a batch, thereby requiring less KV cache space.
21 +- Increase `tensor_parallel_size`. This approach shards model weights, so each GPU has more memory available for KV cache.
22 +
23 +You can also monitor the number of preemption requests through Prometheus metrics exposed by the vLLM. Additionally, you can log the cumulative number of preemption requests by setting disable_log_stats=False.
24 +
25 Chunked Prefill
26 ---------------
27 vLLM supports an experimental feature chunked prefill. Chunked prefill allows to chunk large prefills into smaller chunks and batch them together with decode requests.
tests/basic_correctness/test_preemption.py
+43 -1
@@ -6,6 +6,7 @@ Run `VLLM_TEST_ENABLE_ARTIFICIAL_PREEMPT=1
6 pytest tests/basic_correctness/test_preemption.py`.
7 """
8 import pytest
9 +from prometheus_client import REGISTRY
10
11 from vllm import SamplingParams
12 from vllm.core.scheduler import (ARTIFICIAL_PREEMPTION_MAX_CNT,
@@ -71,6 +72,7 @@ def test_chunked_prefill_recompute(
72 @pytest.mark.parametrize("dtype", ["float"])
73 @pytest.mark.parametrize("max_tokens", [96])
74 def test_preemption(
75 + caplog_vllm,
76 hf_runner,
77 vllm_runner,
78 example_prompts,
@@ -87,10 +89,13 @@ def test_preemption(
89 vllm_model = vllm_runner(
90 model,
91 dtype=dtype,
92 + disable_log_stats=False,
93 )
94 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
95 assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
96 ARTIFICIAL_PREEMPTION_MAX_CNT)
97 + total_preemption = (
98 + vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
99 del vllm_model
100
101 for i in range(len(example_prompts)):
@@ -100,6 +105,20 @@ def test_preemption(
105 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
106 assert hf_output_ids == vllm_output_ids, (
107 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
108 + assert ("is preempted by PreemptionMode.RECOMPUTE mode because there "
109 + "is not enough KV cache space." in caplog_vllm.text)
110 + # Ensure the count bucket of request-level histogram metrics matches
111 + # the number of requests as a simple sanity check to ensure metrics are
112 + # generated
113 + preemption_metrics = None
114 + for m in REGISTRY.collect():
115 + if m.name == "vllm:num_preemptions":
116 + preemption_metrics = m
117 + assert preemption_metrics is not None
118 + total_recorded_preemption = 0
119 + for sample in preemption_metrics.samples:
120 + total_recorded_preemption += sample.value
121 + assert total_preemption == total_recorded_preemption
122
123
124 @pytest.mark.parametrize("model", MODELS)
@@ -107,6 +126,7 @@ def test_preemption(
126 @pytest.mark.parametrize("max_tokens", [96])
127 @pytest.mark.parametrize("beam_width", [4])
128 def test_swap(
129 + caplog_vllm,
130 hf_runner,
131 vllm_runner,
132 example_prompts,
@@ -122,11 +142,18 @@ def test_swap(
142 max_tokens)
143 del hf_model
144
125 - vllm_model = vllm_runner(model, dtype=dtype, swap_space=10)
145 + vllm_model = vllm_runner(
146 + model,
147 + dtype=dtype,
148 + swap_space=10,
149 + disable_log_stats=False,
150 + )
151 vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
152 max_tokens)
153 assert (vllm_model.model.llm_engine.scheduler.artificial_preempt_cnt <
154 ARTIFICIAL_PREEMPTION_MAX_CNT)
155 + total_preemption = (
156 + vllm_model.model.llm_engine.scheduler.num_cumulative_preemption)
157 del vllm_model
158
159 for i in range(len(example_prompts)):
@@ -138,6 +165,21 @@ def test_swap(
165 f"Test{i} output{j}:\nHF: {hf_output_ids}\n"
166 f"vLLM: {vllm_output_ids}")
167
168 + assert ("is preempted by PreemptionMode.SWAP mode because there "
169 + "is not enough KV cache space." in caplog_vllm.text)
170 + # Ensure the count bucket of request-level histogram metrics matches
171 + # the number of requests as a simple sanity check to ensure metrics are
172 + # generated
173 + preemption_metrics = None
174 + for m in REGISTRY.collect():
175 + if m.name == "vllm:num_preemptions":
176 + preemption_metrics = m
177 + assert preemption_metrics is not None
178 + total_recorded_preemption = 0
179 + for sample in preemption_metrics.samples:
180 + total_recorded_preemption += sample.value
181 + assert total_preemption == total_recorded_preemption
182 +
183
184 @pytest.mark.parametrize("model", MODELS)
185 @pytest.mark.parametrize("dtype", ["float"])
tests/conftest.py
+16
@@ -499,3 +499,19 @@ def get_tokenizer_pool_config(tokenizer_group_type):
499 pool_type="ray",
500 extra_config={})
501 raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
502 +
503 +
504 +@pytest.fixture()
505 +def temporary_enable_log_propagate():
506 + import logging
507 + logger = logging.getLogger("vllm")
508 + logger.propagate = True
509 + yield
510 + logger.propagate = False
511 +
512 +
513 +@pytest.fixture()
514 +def caplog_vllm(temporary_enable_log_propagate, caplog):
515 + # To capture vllm log, we should enable propagate=True temporarily
516 + # because caplog depends on logs propagated to the root logger.
517 + yield caplog
tests/core/test_scheduler.py
+1
@@ -180,6 +180,7 @@ def test_scheduler_schedule_preempt_abort():
180 and not out.blocks_to_swap_out)
181 assert len(seq_group_meta) == 1
182 assert scheduler.get_num_unfinished_seq_groups() == 2
183 + assert out.preempted == 1
184
185 # Abort seq group a. Re-schedule seq group b prompt with recomputation.
186 scheduler.abort_seq_group("1")
vllm/core/scheduler.py
+18
@@ -129,6 +129,7 @@ class SchedulerOutputs:
129 num_lookahead_slots: int
130 # The number of requests in the running queue
131 running_queue_size: int
132 + preempted: int
133
134 def __post_init__(self):
135 # Swap in and swap out should never happen at the same time.
@@ -310,6 +311,7 @@ class Scheduler:
311 self.artificial_preempt_cnt = (ARTIFICIAL_PREEMPTION_MAX_CNT
312 if self.enable_artificial_preemption
313 else 0)
314 + self.num_cumulative_preemption: int = 0
315
316 @property
317 def lora_enabled(self) -> bool:
@@ -785,6 +787,8 @@ class Scheduler:
787 # Update swapped requests.
788 self.swapped = remaining_swapped
789 self.swapped.extend(running_scheduled.swapped_out)
790 + preempted = (len(running_scheduled.preempted) +
791 + len(running_scheduled.swapped_out))
792
793 # There should be no prefill from running queue because this policy
794 # doesn't allow chunked prefills.
@@ -804,6 +808,7 @@ class Scheduler:
808 swapped_in.infeasible_seq_groups,
809 num_lookahead_slots=running_scheduled.num_lookahead_slots,
810 running_queue_size=len(self.running),
811 + preempted=preempted,
812 )
813
814 def _schedule_chunked_prefill(self):
@@ -891,6 +896,8 @@ class Scheduler:
896 ignored_seq_groups=prefills.ignored_seq_groups,
897 num_lookahead_slots=running_scheduled.num_lookahead_slots,
898 running_queue_size=len(self.running),
899 + preempted=(len(running_scheduled.preempted) +
900 + len(running_scheduled.swapped_out)),
901 )
902
903 def _schedule(self) -> SchedulerOutputs:
@@ -1057,6 +1064,17 @@ class Scheduler:
1064 preemption_mode = PreemptionMode.RECOMPUTE
1065 else:
1066 preemption_mode = PreemptionMode.SWAP
1067 +
1068 + if self.num_cumulative_preemption % 50 == 0:
1069 + logger.warning(
1070 + "Sequence group %s is preempted by %s mode because there is "
1071 + "not enough KV cache space. This can affect the end-to-end "
1072 + "performance. Increase gpu_memory_utilization or "
1073 + "tensor_parallel_size to provide more KV cache memory. "
1074 + "total_num_cumulative_preemption=%d", seq_group.request_id,
1075 + preemption_mode, self.num_cumulative_preemption + 1)
1076 + self.num_cumulative_preemption += 1
1077 +
1078 if preemption_mode == PreemptionMode.RECOMPUTE:
1079 self._preempt_by_recompute(seq_group)
1080 elif preemption_mode == PreemptionMode.SWAP:
vllm/engine/llm_engine.py
+3 -1
@@ -737,6 +737,8 @@ class LLMEngine:
737 num_generation_tokens_iter = 0
738 time_to_first_tokens_iter: List[float] = []
739 time_per_output_tokens_iter: List[float] = []
740 + num_preemption_iter = (0 if scheduler_outputs is None else
741 + scheduler_outputs.preempted)
742
743 # Request stats
744 # Latency
@@ -830,7 +832,6 @@ class LLMEngine:
832
833 return Stats(
834 now=now,
833 -
835 # System stats
836 # Scheduler State
837 num_running_sys=num_running_sys,
@@ -846,6 +847,7 @@ class LLMEngine:
847 time_to_first_tokens_iter=time_to_first_tokens_iter,
848 time_per_output_tokens_iter=time_per_output_tokens_iter,
849 spec_decode_metrics=spec_decode_metrics,
850 + num_preemption_iter=num_preemption_iter,
851
852 # Request stats
853 # Latency
vllm/engine/metrics.py
+8 -1
@@ -61,6 +61,10 @@ class Metrics:
61 labelnames=labelnames)
62
63 # Iteration stats
64 + self.counter_num_preemption = Counter(
65 + name="vllm:num_preemptions_total",
66 + documentation="Cumulative number of preemption from the engine.",
67 + labelnames=labelnames)
68 self.counter_prompt_tokens = Counter(
69 name="vllm:prompt_tokens_total",
70 documentation="Number of prefill tokens processed.",
@@ -181,6 +185,7 @@ class Stats:
185 num_generation_tokens_iter: int
186 time_to_first_tokens_iter: List[float]
187 time_per_output_tokens_iter: List[float]
188 + num_preemption_iter: int
189
190 # Request stats (should have _requests suffix)
191 # Latency
@@ -244,6 +249,8 @@ class StatLogger:
249 stats.cpu_cache_usage_sys)
250
251 # Iteration level data
252 + self._log_counter(self.metrics.counter_num_preemption,
253 + stats.num_preemption_iter)
254 self._log_counter(self.metrics.counter_prompt_tokens,
255 stats.num_prompt_tokens_iter)
256 self._log_counter(self.metrics.counter_generation_tokens,
@@ -336,7 +343,7 @@ class StatLogger:
343 "Avg generation throughput: %.1f tokens/s, "
344 "Running: %d reqs, Swapped: %d reqs, "
345 "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
339 - "CPU KV cache usage: %.1f%%",
346 + "CPU KV cache usage: %.1f%%.",
347 prompt_throughput,
348 generation_throughput,
349 stats.num_running_sys,