A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 02:45 UTC
[V0 Deprecation] Remove unused swap_space parameter (#36216)
Signed-off-by: majiayu000 <1835304752@qq.com> Co-authored-by: mcelrath
lif committed
Mar 7, 2026 at 22:09 UTC
00b814ba5a4139910c0824619a8dc6af547e178a
22 files changed
+19
-79
.buildkite/performance-benchmarks/README.md
-1
@@ -83,7 +83,6 @@ We test the throughput by using `vllm bench serve` with request rate = inf to co
83
"server_parameters": {
84
"model": "meta-llama/Meta-Llama-3-8B",
85
"tensor_parallel_size": 1,
86
- "swap_space": 16,
86
"disable_log_stats": "",
87
"load_format": "dummy"
88
},
.buildkite/performance-benchmarks/tests/serving-tests-hpu.json
-4
@@ -10,7 +10,6 @@
10
"server_parameters": {
11
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
12
"tensor_parallel_size": 1,
13
- "swap_space": 16,
13
"disable_log_stats": "",
14
"load_format": "dummy",
15
"max-model-len": 2048,
@@ -37,7 +36,6 @@
36
"server_parameters": {
37
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
38
"tensor_parallel_size": 4,
40
- "swap_space": 16,
39
"disable_log_stats": "",
40
"load_format": "dummy",
41
"max-model-len": 2048,
@@ -64,7 +62,6 @@
62
"server_parameters": {
63
"model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
64
"tensor_parallel_size": 2,
67
- "swap_space": 16,
65
"disable_log_stats": "",
66
"load_format": "dummy",
67
"max-model-len": 2048,
@@ -91,7 +88,6 @@
88
"server_parameters": {
89
"model": "deepseek-ai/DeepSeek-R1",
90
"tensor_parallel_size": 8,
94
- "swap_space": 16,
91
"disable_log_stats": "",
92
"load_format": "dummy",
93
"max-model-len": 2048,
.buildkite/performance-benchmarks/tests/serving-tests.json
-4
@@ -5,7 +5,6 @@
5
"server_parameters": {
6
"model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
7
"tensor_parallel_size": 1,
8
- "swap_space": 16,
8
"disable_log_stats": "",
9
"load_format": "dummy"
10
},
@@ -23,7 +22,6 @@
22
"server_parameters": {
23
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
24
"tensor_parallel_size": 4,
26
- "swap_space": 16,
25
"disable_log_stats": "",
26
"load_format": "dummy"
27
},
@@ -41,7 +39,6 @@
39
"server_parameters": {
40
"model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
41
"tensor_parallel_size": 2,
44
- "swap_space": 16,
42
"disable_log_stats": "",
43
"load_format": "dummy"
44
},
@@ -59,7 +56,6 @@
56
"server_parameters": {
57
"model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
58
"tensor_parallel_size": 4,
62
- "swap_space": 16,
59
"speculative_config": {
60
"model": "turboderp/Qwama-0.5B-Instruct",
61
"num_speculative_tokens": 4,
benchmarks/attention_benchmarks/mla_runner.py
-1
@@ -145,7 +145,6 @@ def create_minimal_vllm_config(
145
cache_config = CacheConfig(
146
block_size=block_size,
147
gpu_memory_utilization=0.9,
148
- swap_space=0,
148
cache_dtype="auto",
149
enable_prefix_caching=False,
150
)
benchmarks/attention_benchmarks/runner.py
-1
@@ -141,7 +141,6 @@ def _create_vllm_config(
141
cache_config = CacheConfig(
142
block_size=config.block_size,
143
cache_dtype="auto",
144
- swap_space=0,
144
)
145
cache_config.num_gpu_blocks = max_num_blocks
146
cache_config.num_cpu_blocks = 0
docs/design/metrics.md
+4
-4
@@ -507,10 +507,10 @@ longer relevant in v1:
507
- `vllm:num_requests_swapped`
508
- `vllm:cpu_cache_usage_perc`
509
510
-In this mode, when a request is preempted (e.g. to make room in KV
511
-cache to complete other requests), we swap kv cache blocks out to CPU
512
-memory. This is also known as "KV cache offloading" and is configured
513
-with `--swap-space` and `--preemption-mode`.
510
+In this mode, when a request was preempted (e.g. to make room in KV
511
+cache to complete other requests), kv cache blocks were swapped out to
512
+CPU memory. The `--swap-space` flag has been removed as this feature
513
+is no longer used in V1.
514
515
Historically, [vLLM has long supported beam search](https://github.com/vllm-project/vllm/issues/6226). The
516
SequenceGroup encapsulated the idea of N Sequences which
docs/serving/integrations/llamaindex.md
+1
-1
@@ -17,7 +17,7 @@ llm = Vllm(
17
model="microsoft/Orca-2-7b",
18
tensor_parallel_size=4,
19
max_new_tokens=100,
20
- vllm_kwargs={"swap_space": 1, "gpu_memory_utilization": 0.5},
20
+ vllm_kwargs={"gpu_memory_utilization": 0.5},
21
)
22
```
23
tests/conftest.py
-2
@@ -794,7 +794,6 @@ class VllmRunner:
794
tensor_parallel_size: int = 1,
795
block_size: int = 16 if not torch.xpu.is_available() else 64,
796
enable_chunked_prefill: bool | None = False,
797
- swap_space: int = 4,
797
enforce_eager: bool | None = False,
798
# Set this to avoid hanging issue
799
default_torch_num_threads: int | None = None,
@@ -831,7 +830,6 @@ class VllmRunner:
830
trust_remote_code=trust_remote_code,
831
dtype=dtype,
832
seed=seed,
834
- swap_space=swap_space,
833
enforce_eager=enforce_eager,
834
disable_log_stats=disable_log_stats,
835
tensor_parallel_size=tensor_parallel_size,
tests/distributed/test_torchrun_example.py
+1
-2
@@ -22,7 +22,7 @@ prompts = [
22
23
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
24
25
-# set different `gpu_memory_utilization` and `swap_space` for different ranks,
25
+# set different `gpu_memory_utilization` for different ranks,
26
# to test if all ranks agree on the same kv cache configuration.
27
llm = LLM(
28
model="facebook/opt-125m",
@@ -30,7 +30,6 @@ llm = LLM(
30
pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)),
31
distributed_executor_backend="external_launcher",
32
gpu_memory_utilization=random.uniform(0.7, 0.9),
33
- swap_space=random.randint(1, 4),
33
seed=0,
34
)
35
tests/distributed/test_torchrun_example_moe.py
+1
-2
@@ -28,7 +28,7 @@ if dp_size > 1:
28
29
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
30
31
-# set different `gpu_memory_utilization` and `swap_space` for different ranks,
31
+# set different `gpu_memory_utilization` for different ranks,
32
# to test if all ranks agree on the same kv cache configuration.
33
llm = LLM(
34
model="microsoft/Phi-mini-MoE-instruct",
@@ -37,7 +37,6 @@ llm = LLM(
37
enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1,
38
distributed_executor_backend="external_launcher",
39
gpu_memory_utilization=random.uniform(0.7, 0.9),
40
- swap_space=random.randint(1, 4),
40
seed=0,
41
)
42
tests/lora/test_worker.py
-1
@@ -64,7 +64,6 @@ def test_worker_apply_lora(qwen3_lora_files):
64
device_config=DeviceConfig("cuda"),
65
cache_config=CacheConfig(
66
block_size=16,
67
- swap_space=0,
67
cache_dtype="auto",
68
),
69
lora_config=LoRAConfig(
tests/v1/attention/utils.py
-1
@@ -182,7 +182,6 @@ def create_vllm_config(
182
cache_config = CacheConfig(
183
block_size=block_size,
184
cache_dtype="auto",
185
- swap_space=0,
185
)
186
# Set cache blocks for testing
187
# (these may be set during initialization normally)
tests/v1/core/test_scheduler.py
-2
@@ -1776,7 +1776,6 @@ def create_scheduler_with_priority(
1776
cache_config = CacheConfig(
1777
block_size=block_size,
1778
gpu_memory_utilization=0.9,
1779
- swap_space=0,
1779
cache_dtype="auto",
1780
enable_prefix_caching=enable_prefix_caching,
1781
)
@@ -3726,7 +3725,6 @@ def _create_encoder_decoder_scheduler(
3725
cache_config = CacheConfig(
3726
block_size=block_size,
3727
gpu_memory_utilization=0.9,
3729
- swap_space=0,
3728
cache_dtype="auto",
3729
enable_prefix_caching=False,
3730
)
tests/v1/core/utils.py
-1
@@ -94,7 +94,6 @@ def create_scheduler(
94
cache_config = CacheConfig(
95
block_size=block_size,
96
gpu_memory_utilization=0.9,
97
- swap_space=0,
97
cache_dtype="auto",
98
enable_prefix_caching=enable_prefix_caching,
99
)
tests/v1/engine/test_engine_core.py
-1
@@ -506,7 +506,6 @@ def test_encoder_instance_zero_kv_cache(
506
cache_config = CacheConfig(
507
block_size=16,
508
gpu_memory_utilization=gpu_memory_utilization,
509
- swap_space=0,
509
cache_dtype="auto",
510
enable_prefix_caching=enable_prefix_caching,
511
)
tests/v1/kv_connector/unit/test_moriio_connector.py
-1
@@ -206,7 +206,6 @@ def create_vllm_config(
206
cache_config = CacheConfig(
207
block_size=block_size,
208
gpu_memory_utilization=0.9,
209
- swap_space=0,
209
cache_dtype="auto",
210
enable_prefix_caching=True,
211
)
tests/v1/kv_connector/unit/utils.py
-1
@@ -118,7 +118,6 @@ def create_vllm_config(
118
cache_config = CacheConfig(
119
block_size=block_size,
120
gpu_memory_utilization=0.9,
121
- swap_space=0,
121
cache_dtype=cache_dtype,
122
enable_prefix_caching=True,
123
)
tests/v1/worker/test_gpu_model_runner.py
-3
@@ -96,7 +96,6 @@ def get_vllm_config():
96
cache_config = CacheConfig(
97
block_size=BLOCK_SIZE,
98
gpu_memory_utilization=0.9,
99
- swap_space=0,
99
cache_dtype="auto",
100
)
101
parallel_config = ParallelConfig()
@@ -809,7 +808,6 @@ def test_hybrid_attention_mamba_tensor_shapes():
808
cache_config = CacheConfig(
809
block_size=BLOCK_SIZE,
810
gpu_memory_utilization=0.9,
812
- swap_space=0,
811
cache_dtype="auto",
812
)
813
parallel_config = ParallelConfig()
@@ -1242,7 +1240,6 @@ def test_cudagraph_sizes_capped_for_mamba_cache():
1240
cache_config = CacheConfig(
1241
block_size=BLOCK_SIZE,
1242
gpu_memory_utilization=0.9,
1245
- swap_space=0,
1243
cache_dtype="auto",
1244
)
1245
parallel_config = ParallelConfig()
vllm/config/cache.py
+1
-33
@@ -1,21 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import math
4
from dataclasses import field
6
-from typing import TYPE_CHECKING, Any, Literal
5
+from typing import Literal
6
7
from pydantic import Field, SkipValidation, field_validator
8
9
from vllm.config.utils import config
10
from vllm.logger import init_logger
12
-from vllm.utils.mem_constants import GiB_bytes
13
-from vllm.utils.mem_utils import format_gib, get_cpu_memory
14
-
15
-if TYPE_CHECKING:
16
- from vllm.config.parallel import ParallelConfig
17
-else:
18
- ParallelConfig = Any
11
12
logger = init_logger(__name__)
13
@@ -53,8 +45,6 @@ class CacheConfig:
45
not matter if you have another vLLM instance running on the same GPU. For
46
example, if you have two vLLM instances running on the same GPU, you can
47
set the GPU memory utilization to 0.5 for each instance."""
56
- swap_space: float = Field(default=4, ge=0)
57
- """Size of the CPU swap space per GPU (in GiB)."""
48
cache_dtype: CacheDType = "auto"
49
"""Data type for kv cache storage. If "auto", will use model data type.
50
CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
@@ -173,7 +163,6 @@ class CacheConfig:
163
ignored_factors = {
164
# Runtime/derived knobs that don't affect compiled graph shape
165
"gpu_memory_utilization",
176
- "swap_space",
166
"is_attention_free",
167
"num_gpu_blocks_override",
168
"enable_prefix_caching",
@@ -208,24 +197,3 @@ class CacheConfig:
197
"scaling factor."
198
)
199
return cache_dtype
211
-
212
- def verify_with_parallel_config(
213
- self,
214
- parallel_config: ParallelConfig,
215
- ) -> None:
216
- swap_space_bytes = math.ceil(self.swap_space * GiB_bytes)
217
- total_cpu_memory = get_cpu_memory()
218
- # FIXME(woosuk): Here, it is assumed that the GPUs in a tensor parallel
219
- # group are in the same node. However, the GPUs may span multiple nodes.
220
- num_gpus_per_node = parallel_config.tensor_parallel_size
221
- cpu_memory_usage = swap_space_bytes * num_gpus_per_node
222
-
223
- msg = (
224
- f"{format_gib(cpu_memory_usage)} GiB out of the "
225
- f"{format_gib(total_cpu_memory)} GiB total CPU memory "
226
- "is allocated for the swap space."
227
- )
228
- if cpu_memory_usage > 0.7 * total_cpu_memory:
229
- raise ValueError("Too large swap space. " + msg)
230
- elif cpu_memory_usage > 0.4 * total_cpu_memory:
231
- logger.warning("Possibly too large swap space. %s", msg)
vllm/config/vllm.py
-2
@@ -674,8 +674,6 @@ class VllmConfig:
674
675
self.parallel_config.is_moe_model = self.model_config.is_moe
676
677
- self.cache_config.verify_with_parallel_config(self.parallel_config)
678
-
677
if self.lora_config is not None:
678
self.lora_config.verify_with_model_config(self.model_config)
679
vllm/engine/arg_utils.py
-3
@@ -447,7 +447,6 @@ class EngineArgs:
447
)
448
disable_sliding_window: bool = ModelConfig.disable_sliding_window
449
disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
450
- swap_space: float = CacheConfig.swap_space
450
offload_backend: str = OffloadConfig.offload_backend
451
cpu_offload_gb: float = UVAOffloadConfig.cpu_offload_gb
452
cpu_offload_params: set[str] = get_field(UVAOffloadConfig, "cpu_offload_params")
@@ -961,7 +960,6 @@ class EngineArgs:
960
cache_group.add_argument(
961
"--kv-cache-memory-bytes", **cache_kwargs["kv_cache_memory_bytes"]
962
)
964
- cache_group.add_argument("--swap-space", **cache_kwargs["swap_space"])
963
cache_group.add_argument("--kv-cache-dtype", **cache_kwargs["cache_dtype"])
964
cache_group.add_argument(
965
"--num-gpu-blocks-override", **cache_kwargs["num_gpu_blocks_override"]
@@ -1526,7 +1524,6 @@ class EngineArgs:
1524
block_size=self.block_size,
1525
gpu_memory_utilization=self.gpu_memory_utilization,
1526
kv_cache_memory_bytes=self.kv_cache_memory_bytes,
1529
- swap_space=self.swap_space,
1527
cache_dtype=resolved_cache_dtype, # type: ignore[arg-type]
1528
is_attention_free=model_config.is_attention_free,
1529
num_gpu_blocks_override=self.num_gpu_blocks_override,
vllm/entrypoints/llm.py
+11
-8
@@ -164,12 +164,6 @@ class LLM:
164
compared with using gpu_memory_utilization. Note that
165
kv_cache_memory_bytes (when not-None) ignores
166
gpu_memory_utilization
167
- swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
168
- This can be used for temporarily storing the states of the requests
169
- when their `best_of` sampling parameters are larger than 1. If all
170
- requests will have `best_of=1`, you can safely set this to 0.
171
- Noting that `best_of` is only supported in V0. Otherwise, too small
172
- values may cause out-of-memory (OOM) errors.
167
cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
168
the model weights. This virtually increases the GPU memory space
169
you can use to hold the model weights, at the cost of CPU-GPU data
@@ -240,7 +234,6 @@ class LLM:
234
chat_template: Path | str | None = None,
235
seed: int = 0,
236
gpu_memory_utilization: float = 0.9,
243
- swap_space: float = 4,
237
cpu_offload_gb: float = 0,
238
offload_group_size: int = 0,
239
offload_num_in_group: int = 1,
@@ -265,6 +258,17 @@ class LLM:
258
) -> None:
259
"""LLM constructor."""
260
261
+ if "swap_space" in kwargs:
262
+ kwargs.pop("swap_space")
263
+ import warnings
264
+
265
+ warnings.warn(
266
+ "The 'swap_space' parameter is deprecated and ignored. "
267
+ "It will be removed in a future version.",
268
+ DeprecationWarning,
269
+ stacklevel=2,
270
+ )
271
+
272
if "disable_log_stats" not in kwargs:
273
kwargs["disable_log_stats"] = True
274
@@ -353,7 +357,6 @@ class LLM:
357
seed=seed,
358
gpu_memory_utilization=gpu_memory_utilization,
359
kv_cache_memory_bytes=kv_cache_memory_bytes,
356
- swap_space=swap_space,
360
cpu_offload_gb=cpu_offload_gb,
361
offload_group_size=offload_group_size,
362
offload_num_in_group=offload_num_in_group,