A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Misc] [Core] Implement RFC "Augment BaseExecutor interfaces to enable hardware-agnostic speculative decoding" (#3837)
Cade Daniel committed
Apr 9, 2024 at 11:44 UTC
e7c7067b45c9f604e0c68015ee6e0fe345288111
20 files changed
+453
-277
tests/core/block/e2e/test_correctness.py
+3
-3
@@ -16,7 +16,7 @@ from vllm import SamplingParams
16
17
# Allow only 5 sequences of ~1024 tokens in worst case.
18
"block_size": 16,
19
- "forced_num_gpu_blocks": 5 * (64 + 1),
19
+ "num_gpu_blocks_override": 5 * (64 + 1),
20
}])
21
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
22
@pytest.mark.parametrize("baseline_llm_kwargs", [{
@@ -162,14 +162,14 @@ def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
162
163
# Allow only 2 sequences of ~128 tokens in worst case.
164
# Note 8 = 128/block_size
165
- "forced_num_gpu_blocks": 2 * (8 + 1),
165
+ "num_gpu_blocks_override": 2 * (8 + 1),
166
},
167
{
168
"block_size": 8,
169
170
# Allow only 2 sequences of ~128 tokens in worst case.
171
# Note 16 = 128/block_size
172
- "forced_num_gpu_blocks": 2 * (16 + 1),
172
+ "num_gpu_blocks_override": 2 * (16 + 1),
173
}
174
])
175
@pytest.mark.parametrize("baseline_llm_kwargs", [{
tests/lora/test_worker.py
+6
-2
@@ -3,8 +3,8 @@ import random
3
import tempfile
4
from unittest.mock import patch
5
6
-from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7
- SchedulerConfig)
6
+from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
+ ParallelConfig, SchedulerConfig)
8
from vllm.lora.models import LoRAMapping
9
from vllm.lora.request import LoRARequest
10
from vllm.worker.worker import Worker
@@ -27,6 +27,10 @@ def test_worker_apply_lora(sql_lora_files):
27
parallel_config=ParallelConfig(1, 1, False),
28
scheduler_config=SchedulerConfig(32, 32, 32),
29
device_config=DeviceConfig("cuda"),
30
+ cache_config=CacheConfig(block_size=16,
31
+ gpu_memory_utilization=1.,
32
+ swap_space=0,
33
+ cache_dtype="auto"),
34
local_rank=0,
35
rank=0,
36
lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
tests/spec_decode/test_spec_decode_worker.py
+13
-22
@@ -512,8 +512,8 @@ def test_init_device():
512
513
514
@torch.inference_mode()
515
-def test_init_cache_engine():
516
- """Verify SpecDecodeWorker invokes init_cache_engine on proposer/scorer
515
+def test_initialize_cache():
516
+ """Verify SpecDecodeWorker invokes initialize_cache on proposer/scorer
517
workers.
518
"""
519
draft_worker = mock_worker(cls=MultiStepWorker)
@@ -525,12 +525,11 @@ def test_init_cache_engine():
525
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
526
metrics_collector)
527
528
- cache_config = MagicMock()
528
+ kwargs = {"num_gpu_blocks": 1024, "num_cpu_blocks": 1023}
529
+ worker.initialize_cache(**kwargs)
530
530
- worker.init_cache_engine(cache_config)
531
-
532
- draft_worker.init_cache_engine.assert_called_once_with(cache_config)
533
- target_worker.init_cache_engine.assert_called_once_with(cache_config)
531
+ draft_worker.initialize_cache.assert_called_once_with(**kwargs)
532
+ target_worker.initialize_cache.assert_called_once_with(**kwargs)
533
534
535
@pytest.mark.parametrize('available_gpu_blocks', [1, 1024])
@@ -538,10 +537,10 @@ def test_init_cache_engine():
537
@pytest.mark.parametrize('target_cache_block_size_bytes', [2 * 2 * 4096])
538
@pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
539
@pytest.mark.skip_global_cleanup
541
-def test_profile_num_available_blocks(available_gpu_blocks: int,
542
- available_cpu_blocks: int,
543
- target_cache_block_size_bytes: int,
544
- draft_kv_size_bytes: int):
540
+def test_determine_num_available_blocks(available_gpu_blocks: int,
541
+ available_cpu_blocks: int,
542
+ target_cache_block_size_bytes: int,
543
+ draft_kv_size_bytes: int):
544
"""Verify SpecDecodeWorker correctly profiles num available GPU blocks.
545
Specifically, it should run profiling in the scorer worker, and then evenly
546
split the blocks between proposer and scorer worker.
@@ -552,7 +551,7 @@ def test_profile_num_available_blocks(available_gpu_blocks: int,
551
rejection_sampler.token_id_dtype = torch.int64
552
metrics_collector = MagicMock(spec=AsyncMetricsCollector)
553
555
- target_worker.profile_num_available_blocks.return_value = (
554
+ target_worker.determine_num_available_blocks.return_value = (
555
available_gpu_blocks, available_cpu_blocks)
556
target_worker.get_cache_block_size_bytes.return_value = (
557
target_cache_block_size_bytes)
@@ -561,17 +560,9 @@ def test_profile_num_available_blocks(available_gpu_blocks: int,
560
worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
561
metrics_collector)
562
564
- # These values do not directly impact the adjusted block size calculation,
565
- # so they can be fixed.
566
- gpu_memory_utilization = 0.9
567
- cpu_swap_space = 100
568
- block_size = 16
569
-
570
- num_gpu_blocks, num_cpu_blocks = worker.profile_num_available_blocks(
571
- block_size, gpu_memory_utilization, cpu_swap_space, cache_dtype="auto")
563
+ num_gpu_blocks, num_cpu_blocks = worker.determine_num_available_blocks()
564
573
- target_worker.profile_num_available_blocks.assert_called_once_with(
574
- block_size, gpu_memory_utilization, cpu_swap_space, "auto")
565
+ target_worker.determine_num_available_blocks.assert_called_once()
566
assert num_cpu_blocks == available_cpu_blocks
567
568
assert num_gpu_blocks == split_num_cache_blocks_evenly(
tests/spec_decode/utils.py
+4
-2
@@ -117,6 +117,7 @@ def create_worker(cls: type,
117
parallel_config=engine_config.parallel_config,
118
scheduler_config=engine_config.scheduler_config,
119
device_config=engine_config.device_config,
120
+ cache_config=engine_config.cache_config,
121
local_rank=0,
122
rank=0,
123
distributed_init_method=distributed_init_method,
@@ -128,8 +129,9 @@ def create_worker(cls: type,
129
130
engine_config.cache_config.num_gpu_blocks = num_gpu_blocks
131
engine_config.cache_config.num_cpu_blocks = 0
131
- worker.init_cache_engine(engine_config.cache_config)
132
- worker.warm_up_model()
132
+ worker.initialize_cache(
133
+ num_gpu_blocks=engine_config.cache_config.num_gpu_blocks,
134
+ num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
135
136
return worker
137
tests/worker/test_swap.py
+6
-4
@@ -11,8 +11,8 @@ def test_swap() -> None:
11
dtype="half",
12
load_format="dummy")
13
engine_config = engine_args.create_engine_config()
14
- engine_config.cache_config.num_gpu_blocks = 100
15
- engine_config.cache_config.num_cpu_blocks = 100
14
+ engine_config.cache_config.num_gpu_blocks = 1000
15
+ engine_config.cache_config.num_cpu_blocks = 1000
16
17
# Create the worker.
18
distributed_init_method = get_distributed_init_method(
@@ -22,6 +22,7 @@ def test_swap() -> None:
22
parallel_config=engine_config.parallel_config,
23
scheduler_config=engine_config.scheduler_config,
24
device_config=engine_config.device_config,
25
+ cache_config=engine_config.cache_config,
26
local_rank=0,
27
rank=0,
28
distributed_init_method=distributed_init_method,
@@ -31,8 +32,9 @@ def test_swap() -> None:
32
# Initialize the worker.
33
worker.init_device()
34
worker.load_model()
34
- worker.init_cache_engine(engine_config.cache_config)
35
- worker.warm_up_model()
35
+ worker.initialize_cache(
36
+ num_gpu_blocks=engine_config.cache_config.num_gpu_blocks,
37
+ num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
38
39
# Randomly initialize the cache.
40
gpu_cache = worker.cache_engine.gpu_cache
vllm/config.py
+3
-3
@@ -334,7 +334,7 @@ class CacheConfig:
334
vLLM execution.
335
swap_space: Size of the CPU swap space per GPU (in GiB).
336
cache_dtype: Data type for kv cache storage.
337
- forced_num_gpu_blocks: Number of GPU blocks to use. This overrides the
337
+ num_gpu_blocks_override: Number of GPU blocks to use. This overrides the
338
profiled num_gpu_blocks if specified. Does nothing if None.
339
"""
340
@@ -344,14 +344,14 @@ class CacheConfig:
344
gpu_memory_utilization: float,
345
swap_space: int,
346
cache_dtype: str,
347
- forced_num_gpu_blocks: Optional[int] = None,
347
+ num_gpu_blocks_override: Optional[int] = None,
348
sliding_window: Optional[int] = None,
349
enable_prefix_caching: bool = False,
350
) -> None:
351
self.block_size = block_size
352
self.gpu_memory_utilization = gpu_memory_utilization
353
self.swap_space_bytes = swap_space * _GB
354
- self.forced_num_gpu_blocks = forced_num_gpu_blocks
354
+ self.num_gpu_blocks_override = num_gpu_blocks_override
355
self.cache_dtype = cache_dtype
356
self.sliding_window = sliding_window
357
self.enable_prefix_caching = enable_prefix_caching
vllm/engine/arg_utils.py
+3
-3
@@ -55,7 +55,7 @@ class EngineArgs:
55
max_cpu_loras: Optional[int] = None
56
device: str = 'auto'
57
ray_workers_use_nsight: bool = False
58
- forced_num_gpu_blocks: Optional[int] = None
58
+ num_gpu_blocks_override: Optional[int] = None
59
num_lookahead_slots: int = 0
60
61
# Related to Vision-language models such as llava
@@ -246,7 +246,7 @@ class EngineArgs:
246
'the model executor, which can range from 0 to 1.'
247
'If unspecified, will use the default value of 0.9.')
248
parser.add_argument(
249
- '--forced-num-gpu-blocks',
249
+ '--num-gpu-blocks-override',
250
type=int,
251
default=None,
252
help='If specified, ignore GPU profiling result and use this number'
@@ -426,7 +426,7 @@ class EngineArgs:
426
cache_config = CacheConfig(self.block_size,
427
self.gpu_memory_utilization,
428
self.swap_space, self.kv_cache_dtype,
429
- self.forced_num_gpu_blocks,
429
+ self.num_gpu_blocks_override,
430
model_config.get_sliding_window(),
431
self.enable_prefix_caching)
432
parallel_config = ParallelConfig(
vllm/engine/llm_engine.py
+22
@@ -127,6 +127,8 @@ class LLMEngine:
127
speculative_config=speculative_config,
128
)
129
130
+ self._initialize_kv_caches()
131
+
132
# If usage stat is enabled, collect relevant info.
133
if is_usage_stats_enabled():
134
from vllm.model_executor.model_loader import (
@@ -178,6 +180,26 @@ class LLMEngine:
180
labels=dict(model_name=model_config.model))
181
self.stat_logger.info("cache_config", self.cache_config)
182
183
+ def _initialize_kv_caches(self) -> None:
184
+ """Initialize the KV cache in the worker(s).
185
+
186
+ The workers will determine the number of blocks in both the GPU cache
187
+ and the swap CPU cache.
188
+ """
189
+ num_gpu_blocks, num_cpu_blocks = (
190
+ self.model_executor.determine_num_available_blocks())
191
+
192
+ if self.cache_config.num_gpu_blocks_override is not None:
193
+ num_gpu_blocks_override = self.cache_config.num_gpu_blocks_override
194
+ logger.info(f"Overriding {num_gpu_blocks=} with "
195
+ f"{num_gpu_blocks_override=}")
196
+ num_gpu_blocks = num_gpu_blocks_override
197
+
198
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
199
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
200
+
201
+ self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
202
+
203
@classmethod
204
def from_engine_args(
205
cls,
vllm/executor/cpu_executor.py
+22
-36
@@ -35,7 +35,6 @@ class CPUExecutor(ExecutorBase):
35
36
# Instantiate the worker and load the model to CPU.
37
self._init_worker()
38
- self._init_cache()
38
39
def _init_worker(self):
40
from vllm.worker.cpu_worker import CPUWorker
@@ -46,10 +45,11 @@ class CPUExecutor(ExecutorBase):
45
distributed_init_method = get_distributed_init_method(
46
get_ip(), get_open_port())
47
self.driver_worker = CPUWorker(
49
- self.model_config,
50
- self.parallel_config,
51
- self.scheduler_config,
52
- self.device_config,
48
+ model_config=self.model_config,
49
+ parallel_config=self.parallel_config,
50
+ scheduler_config=self.scheduler_config,
51
+ device_config=self.device_config,
52
+ cache_config=self.cache_config,
53
local_rank=0,
54
rank=0,
55
distributed_init_method=distributed_init_method,
@@ -60,35 +60,21 @@ class CPUExecutor(ExecutorBase):
60
self.driver_worker.init_device()
61
self.driver_worker.load_model()
62
63
- def _init_cache(self) -> None:
64
- num_cpu_blocks = self.driver_worker.get_cpu_cache_block_num(
65
- block_size=self.cache_config.block_size,
66
- cache_space=self.cache_config.cpu_kvcache_space_bytes,
67
- cache_dtype=self.cache_config.cache_dtype,
68
- )
69
-
63
+ def determine_num_available_blocks(self) -> tuple[int, int]:
64
+ """Determine the number of available KV blocks by invoking the
65
+ underlying worker.
66
+ """
67
+ return self.driver_worker.determine_num_available_blocks()
68
+
69
+ def initialize_cache(self, num_gpu_blocks: int,
70
+ num_cpu_blocks: int) -> None:
71
+ """Initialize the KV cache by invoking the underlying worker.
72
+ """
73
+ # NOTE: We log here to avoid multiple logs when number of workers is
74
+ # greater than one. We could log in the engine, but not all executors
75
+ # have GPUs.
76
logger.info(f"# CPU blocks: {num_cpu_blocks}")
71
- if num_cpu_blocks <= 0:
72
- raise ValueError("No available memory for the cache blocks. "
73
- "Try increasing `VLLM_CPU_KVCACHE_SPACE` when "
74
- "initializing the engine.")
75
-
76
- max_seq_len = self.cache_config.block_size * num_cpu_blocks
77
- if self.model_config.max_model_len > max_seq_len:
78
- raise ValueError(
79
- f"The model's max seq len ({self.model_config.max_model_len}) "
80
- "is larger than the maximum number of tokens that can be "
81
- f"stored in KV cache ({max_seq_len}). Try increasing "
82
- "`VLLM_CPU_KVCACHE_SPACE` or decreasing `max_model_len` when "
83
- "initializing the engine.")
84
-
85
- # Note: To reuse the cache management procedure,
86
- # use cpu cache as 'gpu cache'.
87
- self.cache_config.num_gpu_blocks = num_cpu_blocks # type: ignore
88
- self.cache_config.num_cpu_blocks = 0 # type: ignore
89
-
90
- # Initialize the cache.
91
- self.driver_worker.init_cache_engine(cache_config=self.cache_config)
77
+ self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
78
79
def execute_model(self,
80
seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -104,13 +90,13 @@ class CPUExecutor(ExecutorBase):
90
return output
91
92
def add_lora(self, lora_request: LoRARequest) -> bool:
107
- raise NotImplementedError("LoRA is not implemented for cpu backend.")
93
+ return self.driver_worker.add_lora(lora_request)
94
95
def remove_lora(self, lora_id: int) -> bool:
110
- raise NotImplementedError("LoRA is not implemented for cpu backend.")
96
+ return self.driver_worker.remove_lora(lora_id)
97
98
def list_loras(self) -> List[int]:
113
- raise NotImplementedError("LoRA is not implemented for cpu backend.")
99
+ return self.driver_worker.list_loras()
100
101
def check_health(self) -> None:
102
# CPUExecutor will always be healthy as long as
vllm/executor/executor_base.py
+23
@@ -30,6 +30,29 @@ class ExecutorBase(ABC):
30
) -> None:
31
raise NotImplementedError
32
33
+ @abstractmethod
34
+ def determine_num_available_blocks(self) -> tuple[int, int]:
35
+ """Determine the number of available blocks for the GPU KV cache and
36
+ swappable CPU KV cache.
37
+
38
+ Normally, this should simply delegate to the underlying Worker. Some
39
+ ExecutorBase may require modification of the result, e.g. to ensure the
40
+ selected cache sizes are compatible with all workers.
41
+
42
+ Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
43
+ are blocks that are "active" on the device and can be appended to.
44
+ num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
45
+ appended to.
46
+ """
47
+ raise NotImplementedError
48
+
49
+ @abstractmethod
50
+ def initialize_cache(self, num_gpu_blocks: int,
51
+ num_cpu_blocks: int) -> None:
52
+ """Initialize the KV cache with the given size in blocks.
53
+ """
54
+ raise NotImplementedError
55
+
56
@abstractmethod
57
def execute_model(self,
58
seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/executor/gpu_executor.py
+16
-44
@@ -4,7 +4,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
ParallelConfig, SchedulerConfig, SpeculativeConfig,
5
VisionLanguageConfig)
6
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7
-from vllm.executor.utils import check_block_size_valid
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
9
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -41,9 +40,6 @@ class GPUExecutor(ExecutorBase):
40
# Instantiate the worker and load the model to GPU.
41
self._init_worker()
42
44
- # Profile the memory usage and initialize the cache.
45
- self._init_cache()
46
-
43
def _init_worker(self):
44
# Lazy import the Worker to avoid importing torch.cuda/xformers
45
# before CUDA_VISIBLE_DEVICES is set in the Worker
@@ -55,61 +51,37 @@ class GPUExecutor(ExecutorBase):
51
distributed_init_method = get_distributed_init_method(
52
get_ip(), get_open_port())
53
self.driver_worker = Worker(
58
- self.model_config,
59
- self.parallel_config,
60
- self.scheduler_config,
61
- self.device_config,
54
+ model_config=self.model_config,
55
+ parallel_config=self.parallel_config,
56
+ scheduler_config=self.scheduler_config,
57
+ device_config=self.device_config,
58
+ cache_config=self.cache_config,
59
local_rank=0,
60
rank=0,
61
distributed_init_method=distributed_init_method,
62
lora_config=self.lora_config,
63
vision_language_config=self.vision_language_config,
67
- kv_cache_dtype=self.cache_config.cache_dtype,
64
is_driver_worker=True,
65
)
66
self.driver_worker.init_device()
67
self.driver_worker.load_model()
68
73
- def _init_cache(self) -> None:
74
- """Profiles the memory usage and initializes the KV cache.
75
-
76
- The engine first profiles the existing memory usage.
77
- Then, it allocates the remaining memory for KV blocks.
78
-
79
- .. tip::
80
- You may limit the usage of GPU memory
81
- by adjusting the `gpu_memory_utilization` parameter.
69
+ def determine_num_available_blocks(self) -> tuple[int, int]:
70
+ """Determine the number of available KV blocks by invoking the
71
+ underlying worker.
72
"""
83
- # Get the maximum number of blocks that can be allocated on GPU and CPU.
84
- num_gpu_blocks, num_cpu_blocks = (
85
- self.driver_worker.profile_num_available_blocks(
86
- block_size=self.cache_config.block_size,
87
- gpu_memory_utilization=self.cache_config.
88
- gpu_memory_utilization,
89
- cpu_swap_space=self.cache_config.swap_space_bytes,
90
- cache_dtype=self.cache_config.cache_dtype,
91
- ))
92
-
93
- if self.cache_config.forced_num_gpu_blocks is not None:
94
- forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
95
- logger.info(f"Replacing profiled {num_gpu_blocks=} with "
96
- f"{forced_num_gpu_blocks=}")
97
- num_gpu_blocks = forced_num_gpu_blocks
73
+ return self.driver_worker.determine_num_available_blocks()
74
75
+ def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
76
+ """Initialize the KV cache by invoking the underlying worker.
77
+ """
78
+ # NOTE: This is logged in the executor because there can be >1 worker
79
+ # with other executors. We could log in the engine level, but work
80
+ # remains to abstract away the device for non-GPU configurations.
81
logger.info(f"# GPU blocks: {num_gpu_blocks}, "
82
f"# CPU blocks: {num_cpu_blocks}")
83
102
- check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
103
- self.model_config.max_model_len)
104
-
105
- self.cache_config.num_gpu_blocks = num_gpu_blocks
106
- self.cache_config.num_cpu_blocks = num_cpu_blocks
107
-
108
- # Initialize the cache.
109
- self.driver_worker.init_cache_engine(cache_config=self.cache_config)
110
- # Warm up the model. This includes capturing the model into CUDA graph
111
- # if enforce_eager is False.
112
- self.driver_worker.warm_up_model()
84
+ self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
85
86
def execute_model(self,
87
seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/executor/neuron_executor.py
+15
-13
@@ -25,7 +25,6 @@ class NeuronExecutor(ExecutorBase):
25
speculative_config: Optional[SpeculativeConfig],
26
) -> None:
27
self.model_config = model_config
28
- self.cache_config = cache_config
28
assert lora_config is None, "LoRA is not supported for Neuron backend."
29
self.parallel_config = parallel_config
30
self.scheduler_config = scheduler_config
@@ -33,12 +32,6 @@ class NeuronExecutor(ExecutorBase):
32
assert (not speculative_config
33
), "Speculative decoding not yet supported for Neuron backend."
34
36
- # Set the number of GPU blocks to be the same as the maximum number of
37
- # sequences that can be processed in a single batch. This is equivalent
38
- # to schedule without PagedAttention.
39
- self.cache_config.num_gpu_blocks = self.scheduler_config.max_num_seqs
40
- self.cache_config.num_cpu_blocks = 0
41
-
35
# Instantiate the worker and load the model to the device.
36
self._init_worker()
37
@@ -54,6 +47,18 @@ class NeuronExecutor(ExecutorBase):
47
self.driver_worker.init_device()
48
self.driver_worker.load_model()
49
50
+ def determine_num_available_blocks(self) -> tuple[int, int]:
51
+ """Determine the number of available KV blocks by invoking the
52
+ underlying worker.
53
+ """
54
+ return self.driver_worker.determine_num_available_blocks()
55
+
56
+ def initialize_cache(self, num_gpu_blocks: int,
57
+ num_cpu_blocks: int) -> None:
58
+ """Initialize the KV cache by invoking the underlying worker.
59
+ """
60
+ self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
61
+
62
def execute_model(self,
63
seq_group_metadata_list: List[SequenceGroupMetadata],
64
blocks_to_swap_in: Dict[int, int],
@@ -68,16 +73,13 @@ class NeuronExecutor(ExecutorBase):
73
return output
74
75
def add_lora(self, lora_request: LoRARequest) -> bool:
71
- raise NotImplementedError(
72
- "LoRA is not implemented for neuron backend.")
76
+ return self.driver_worker.add_lora(lora_request)
77
78
def remove_lora(self, lora_id: int) -> bool:
75
- raise NotImplementedError(
76
- "LoRA is not implemented for neuron backend.")
79
+ return self.driver_worker.remove_lora(lora_id)
80
81
def list_loras(self) -> List[int]:
79
- raise NotImplementedError(
80
- "LoRA is not implemented for neuron backend.")
82
+ return self.driver_worker.list_loras()
83
84
def check_health(self) -> None:
85
# NeuronExecutor will always be healthy as long as
vllm/executor/ray_gpu_executor.py
+37
-59
@@ -10,7 +10,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10
VisionLanguageConfig)
11
from vllm.engine.ray_utils import RayWorkerVllm, ray
12
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13
-from vllm.executor.utils import check_block_size_valid
13
from vllm.logger import init_logger
14
from vllm.lora.request import LoRARequest
15
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -65,9 +64,6 @@ class RayGPUExecutor(ExecutorBase):
64
# Create the parallel GPU workers.
65
self._init_workers_ray(placement_group)
66
68
- # Profile the memory usage and initialize the cache.
69
- self._init_cache()
70
-
67
self.forward_dag = None
68
if USE_RAY_COMPILED_DAG:
69
self.forward_dag = self._compiled_ray_dag()
@@ -154,8 +150,8 @@ class RayGPUExecutor(ExecutorBase):
150
scheduler_config = copy.deepcopy(self.scheduler_config)
151
device_config = copy.deepcopy(self.device_config)
152
lora_config = copy.deepcopy(self.lora_config)
153
+ cache_config = copy.deepcopy(self.cache_config)
154
vision_language_config = copy.deepcopy(self.vision_language_config)
158
- kv_cache_dtype = self.cache_config.cache_dtype
155
156
# Initialize the actual workers with the Worker class.
157
for rank, (worker, (node_id, _)) in enumerate(
@@ -165,32 +161,32 @@ class RayGPUExecutor(ExecutorBase):
161
local_rank = node_workers[node_id].index(rank)
162
worker.init_worker.remote(
163
lambda rank=rank, local_rank=local_rank: Worker(
168
- model_config,
169
- parallel_config,
170
- scheduler_config,
171
- device_config,
172
- local_rank,
173
- rank,
174
- distributed_init_method,
164
+ model_config=model_config,
165
+ parallel_config=parallel_config,
166
+ scheduler_config=scheduler_config,
167
+ device_config=device_config,
168
+ cache_config=cache_config,
169
+ local_rank=local_rank,
170
+ rank=rank,
171
+ distributed_init_method=distributed_init_method,
172
lora_config=lora_config,
173
vision_language_config=vision_language_config,
177
- kv_cache_dtype=kv_cache_dtype,
174
))
175
176
# Initialize the driver worker with the Worker class.
177
driver_rank = 0
178
driver_local_rank = node_workers[driver_node_id].index(driver_rank)
179
self.driver_worker = Worker(
184
- self.model_config,
185
- self.parallel_config,
186
- self.scheduler_config,
187
- self.device_config,
188
- driver_local_rank,
189
- driver_rank,
190
- distributed_init_method,
180
+ model_config=self.model_config,
181
+ parallel_config=self.parallel_config,
182
+ scheduler_config=self.scheduler_config,
183
+ device_config=self.device_config,
184
+ cache_config=self.cache_config,
185
+ local_rank=driver_local_rank,
186
+ rank=driver_rank,
187
+ distributed_init_method=distributed_init_method,
188
lora_config=self.lora_config,
189
vision_language_config=self.vision_language_config,
193
- kv_cache_dtype=kv_cache_dtype,
190
is_driver_worker=True,
191
)
192
@@ -201,35 +197,18 @@ class RayGPUExecutor(ExecutorBase):
197
max_parallel_loading_workers,
198
)
199
204
- def _init_cache(self) -> None:
205
- """Profiles the memory usage and initializes the KV cache.
206
-
207
- The engine will first conduct a profiling of the existing memory usage.
208
- Then, it calculate the maximum possible number of GPU and CPU blocks
209
- that can be allocated with the remaining free memory.
210
- More details can be found in the
211
- :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
212
- from class :class:`~vllm.worker.Worker`.
200
+ def determine_num_available_blocks(self) -> tuple[int, int]:
201
+ """Determine the number of available KV blocks.
202
214
- Afterwards, as there may be multiple workers,
215
- we take the minimum number of blocks across all workers
216
- to ensure this can be applied to all of them.
203
+ This invokes `determine_num_available_blocks` on each worker and takes
204
+ the min of the results, guaranteeing that the selected cache sizes are
205
+ compatible with all workers.
206
218
- Finally, the engine will initialize the KV cache
219
- with the calculated number of blocks.
220
-
221
- .. tip::
222
- You may limit the usage of GPU memory
223
- by adjusting the `gpu_memory_utilization` parameter.
207
+ Returns:
208
+ - tuple[num_gpu_blocks, num_cpu_blocks]
209
"""
210
# Get the maximum number of blocks that can be allocated on GPU and CPU.
226
- num_blocks = self._run_workers(
227
- "profile_num_available_blocks",
228
- block_size=self.cache_config.block_size,
229
- gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
230
- cpu_swap_space=self.cache_config.swap_space_bytes,
231
- cache_dtype=self.cache_config.cache_dtype,
232
- )
211
+ num_blocks = self._run_workers("determine_num_available_blocks", )
212
213
# Since we use a shared centralized controller, we take the minimum
214
# number of blocks across all workers to make sure all the memory
@@ -237,26 +216,25 @@ class RayGPUExecutor(ExecutorBase):
216
num_gpu_blocks = min(b[0] for b in num_blocks)
217
num_cpu_blocks = min(b[1] for b in num_blocks)
218
240
- if self.cache_config.forced_num_gpu_blocks is not None:
241
- forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
242
- logger.info(f"Replacing profiled {num_gpu_blocks=} with "
243
- f"{forced_num_gpu_blocks=}")
244
- num_gpu_blocks = forced_num_gpu_blocks
219
+ return num_gpu_blocks, num_cpu_blocks
220
221
+ def initialize_cache(self, num_gpu_blocks: int,
222
+ num_cpu_blocks: int) -> None:
223
+ """Initialize the KV cache in all workers.
224
+ """
225
+
226
+ # NOTE: We log here to avoid multiple logs when number of workers is
227
+ # greater than one. We could log in the engine, but not all executors
228
+ # have GPUs.
229
logger.info(f"# GPU blocks: {num_gpu_blocks}, "
230
f"# CPU blocks: {num_cpu_blocks}")
231
249
- check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
250
- self.model_config.max_model_len)
251
-
232
self.cache_config.num_gpu_blocks = num_gpu_blocks
233
self.cache_config.num_cpu_blocks = num_cpu_blocks
234
255
- # Initialize the cache.
256
- self._run_workers("init_cache_engine", cache_config=self.cache_config)
257
- # Warm up the model. This includes capturing the model into CUDA graph
258
- # if enforce_eager is False.
259
- self._run_workers("warm_up_model")
235
+ self._run_workers("initialize_cache",
236
+ num_gpu_blocks=num_gpu_blocks,
237
+ num_cpu_blocks=num_cpu_blocks)
238
239
def execute_model(self,
240
seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/executor/utils.py
deleted
-13
@@ -1,13 +0,0 @@
1
-def check_block_size_valid(num_gpu_blocks, block_size, max_model_len) -> None:
2
- if num_gpu_blocks <= 0:
3
- raise ValueError("No available memory for the cache blocks. "
4
- "Try increasing `gpu_memory_utilization` when "
5
- "initializing the engine.")
6
- max_seq_len = block_size * num_gpu_blocks
7
- if max_model_len > max_seq_len:
8
- raise ValueError(
9
- f"The model's max seq len ({max_model_len}) "
10
- "is larger than the maximum number of tokens that can be "
11
- f"stored in KV cache ({max_seq_len}). Try increasing "
12
- "`gpu_memory_utilization` or decreasing `max_model_len` when "
13
- "initializing the engine.")
vllm/spec_decode/spec_decode_worker.py
+22
-16
@@ -3,7 +3,6 @@ from typing import Dict, List, Optional, Tuple
3
4
import torch
5
6
-from vllm.config import CacheConfig
6
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7
from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
8
SequenceGroupOutput, SequenceOutput)
@@ -15,9 +14,10 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
14
from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
15
split_batch_by_proposal_len)
16
from vllm.worker.worker import Worker
17
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
18
19
20
-class SpecDecodeWorker:
20
+class SpecDecodeWorker(LoraNotSupportedWorkerBase):
21
"""Worker which implements speculative decoding.
22
23
Speculative decoding reduces decoding per-token latency by using a proposal
@@ -94,10 +94,7 @@ class SpecDecodeWorker:
94
device=self.device,
95
vocab_size=self._vocab_size)
96
97
- def profile_num_available_blocks(self, block_size: int,
98
- gpu_memory_utilization: float,
99
- cpu_swap_space: int,
100
- cache_dtype: str) -> Tuple[int, int]:
97
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
98
"""Determine the number of cache blocks to use.
99
100
This is done by profiling the scorer model (which is typically the
@@ -106,27 +103,26 @@ class SpecDecodeWorker:
103
such that the number of blocks is equal in both KV caches.
104
"""
105
num_gpu_blocks, num_cpu_blocks = (
109
- self.scorer_worker.profile_num_available_blocks(
110
- block_size, gpu_memory_utilization, cpu_swap_space,
111
- cache_dtype))
106
+ self.scorer_worker.determine_num_available_blocks())
107
108
scorer_cache_block_size_bytes = (
114
- self.scorer_worker.get_cache_block_size_bytes(
115
- block_size, cache_dtype))
109
+ self.scorer_worker.get_cache_block_size_bytes())
110
proposer_cache_block_size_bytes = (
117
- self.proposer_worker.get_cache_block_size_bytes(
118
- block_size, cache_dtype))
111
+ self.proposer_worker.get_cache_block_size_bytes())
112
113
new_num_gpu_blocks = split_num_cache_blocks_evenly(
114
scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
115
num_gpu_blocks)
116
return new_num_gpu_blocks, num_cpu_blocks
117
125
- def init_cache_engine(self, cache_config: CacheConfig):
118
+ def initialize_cache(self, num_gpu_blocks: int,
119
+ num_cpu_blocks: int) -> None:
120
"""Initialize the cache engine of the scorer and proposer workers.
121
"""
128
- self.scorer_worker.init_cache_engine(cache_config)
129
- self.proposer_worker.init_cache_engine(cache_config)
122
+ self.scorer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
123
+ num_cpu_blocks=num_cpu_blocks)
124
+ self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
125
+ num_cpu_blocks=num_cpu_blocks)
126
127
@torch.inference_mode()
128
def execute_model(
@@ -351,6 +347,16 @@ class SpecDecodeWorker:
347
def device(self):
348
return self.scorer_worker.device
349
350
+ def get_cache_block_size_bytes(self):
351
+ """Return the size of a cache block in bytes.
352
+
353
+ This function is only used to compose workers within a SpecDecodeWorker.
354
+ We leave composing a SpecDecodeWorker within a SpecDecodeWorker
355
+ undefined for now, although it could be implemented in the future.
356
+ See https://arxiv.org/abs/2308.04623.
357
+ """
358
+ raise NotImplementedError
359
+
360
361
def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
362
proposer_cache_block_size_bytes: int,
vllm/worker/cache_engine.py
+4
-5
@@ -82,8 +82,7 @@ class CacheEngine:
82
83
@staticmethod
84
def get_cache_block_size(
85
- block_size: int,
86
- cache_dtype: str,
85
+ cache_config: CacheConfig,
86
model_config: ModelConfig,
87
parallel_config: ParallelConfig,
88
) -> int:
@@ -91,13 +90,13 @@ class CacheEngine:
90
num_heads = model_config.get_num_kv_heads(parallel_config)
91
num_layers = model_config.get_num_layers(parallel_config)
92
94
- key_cache_block = block_size * num_heads * head_size
93
+ key_cache_block = cache_config.block_size * num_heads * head_size
94
value_cache_block = key_cache_block
95
total = num_layers * (key_cache_block + value_cache_block)
97
- if cache_dtype == "auto":
96
+ if cache_config.cache_dtype == "auto":
97
dtype = model_config.dtype
98
else:
100
- dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
99
+ dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
100
dtype_size = _get_dtype_size(dtype)
101
return dtype_size * total
102
vllm/worker/cpu_worker.py
+70
-19
@@ -17,6 +17,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
17
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19
from vllm.worker.model_runner import ModelRunner
20
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
21
22
logger = init_logger(__name__)
23
@@ -112,7 +113,7 @@ class CPUCacheEngine:
113
return dtype_size * total
114
115
115
-class CPUWorker:
116
+class CPUWorker(LoraNotSupportedWorkerBase):
117
"""A worker class that executes (a partition of) the model on a CPU socket.
118
119
Each worker is associated with a single CPU socket. The worker is
@@ -127,6 +128,7 @@ class CPUWorker:
128
parallel_config: ParallelConfig,
129
scheduler_config: SchedulerConfig,
130
device_config: DeviceConfig,
131
+ cache_config: CacheConfig,
132
local_rank: int,
133
rank: int,
134
distributed_init_method: str,
@@ -138,6 +140,7 @@ class CPUWorker:
140
self.parallel_config = parallel_config
141
self.scheduler_config = scheduler_config
142
self.device_config = device_config
143
+ self.cache_config = cache_config
144
self.local_rank = local_rank
145
self.rank = rank
146
self.distributed_init_method = distributed_init_method
@@ -154,8 +157,7 @@ class CPUWorker:
157
kv_cache_dtype=kv_cache_dtype,
158
is_driver_worker=is_driver_worker)
159
# Uninitialized cache engine. Will be initialized by
157
- # self.init_cache_engine().
158
- self.cache_config = None
160
+ # initialize_cache.
161
self.cache_engine = None
162
self.cpu_cache = None
163
@@ -167,28 +169,70 @@ class CPUWorker:
169
def load_model(self):
170
self.model_runner.load_model()
171
170
- def get_cpu_cache_block_num(
171
- self,
172
- block_size: int,
173
- cache_space: int,
174
- cache_dtype: str,
175
- ) -> int:
176
- """
177
- Args:
178
- block_size: The size of the cache block.
179
- cache_space: The size of the CPU KV cache space in bytes.
172
+ def determine_num_available_blocks(self) -> tuple[int, int]:
173
+ """Determine the number of blocks available for the KV cache.
174
+
175
+ This determines how many KV blocks can fit into the configured CPU
176
+ KV cache space.
177
+
178
+ Note that since vLLM assumes a block resides on GPU if it can be
179
+ modified, we return num_gpu_blocks=num_cpu_blocks and num_cpu_blocks=0.
180
+ This allows us to reuse the scheduler of vLLM without generalizing it
181
+ to different devices.
182
"""
183
# For CPU device, the block number will be calculated based on the
184
# cpu_kvcache_space.
183
- cache_block_size = CPUCacheEngine.get_cache_block_size(
184
- block_size, cache_dtype, self.model_config, self.parallel_config)
185
- num_cpu_blocks = int(cache_space // cache_block_size)
185
+ cache_block_size = self.get_cache_block_size_bytes()
186
+ num_cpu_blocks = int(self.cache_config.cpu_kvcache_space_bytes //
187
+ cache_block_size)
188
num_cpu_blocks = max(num_cpu_blocks, 0)
189
188
- return num_cpu_blocks
190
+ # Note: To reuse the cache management procedure,
191
+ # use cpu cache as 'gpu cache'.
192
+ num_gpu_blocks = num_cpu_blocks
193
+ num_cpu_blocks = 0
194
+ return num_gpu_blocks, num_cpu_blocks
195
190
- def init_cache_engine(self, cache_config: CacheConfig) -> None:
191
- self.cache_config = cache_config
196
+ def initialize_cache(self, num_gpu_blocks: int,
197
+ num_cpu_blocks: int) -> None:
198
+ """Initialize the KV cache. Currently, swappable CPU memory is not
199
+ supported.
200
+
201
+ Since this worker does not support GPUs, we use the num_gpu_blocks to
202
+ determine how many non-swappable CPU blocks to allocate.
203
+ """
204
+ assert (num_cpu_blocks == 0
205
+ ), f"{type(self)} does not support swappable cache"
206
+
207
+ # Note: To reuse the cache management procedure,
208
+ # use cpu cache as 'gpu cache'.
209
+ num_cpu_blocks = num_gpu_blocks
210
+
211
+ self._validate_num_cpu_blocks(num_cpu_blocks)
212
+ self.cache_config.num_gpu_blocks = num_cpu_blocks
213
+ self.cache_config.num_cpu_blocks = 0
214
+
215
+ # Initialize the cache.
216
+ self._init_cache_engine()
217
+
218
+ def _validate_num_cpu_blocks(self, num_cpu_blocks: int) -> None:
219
+ """Raise errors if the num_cpu_blocks is invalid.
220
+ """
221
+ if num_cpu_blocks <= 0:
222
+ raise ValueError("No available memory for the cache blocks. "
223
+ "Try increasing `VLLM_CPU_KVCACHE_SPACE` when "
224
+ "initializing the engine.")
225
+
226
+ max_seq_len = self.cache_config.block_size * num_cpu_blocks
227
+ if self.model_config.max_model_len > max_seq_len:
228
+ raise ValueError(
229
+ f"The model's max seq len ({self.model_config.max_model_len}) "
230
+ "is larger than the maximum number of tokens that can be "
231
+ f"stored in KV cache ({max_seq_len}). Try increasing "
232
+ "`VLLM_CPU_KVCACHE_SPACE` or decreasing `max_model_len` when "
233
+ "initializing the engine.")
234
+
235
+ def _init_cache_engine(self) -> None:
236
self.cache_engine = CPUCacheEngine(self.cache_config,
237
self.model_config,
238
self.parallel_config,
@@ -264,3 +308,10 @@ class CPUWorker:
308
ensure_model_parallel_initialized(
309
parallel_config.tensor_parallel_size,
310
parallel_config.pipeline_parallel_size)
311
+
312
+ def get_cache_block_size_bytes(self) -> int:
313
+ """Return the size in bytes of a single KV cache block.
314
+ """
315
+ return CPUCacheEngine.get_cache_block_size(
316
+ self.cache_config.block_size, self.cache_config.cache_dtype,
317
+ self.model_config, self.parallel_config)
vllm/worker/neuron_worker.py
+42
-3
@@ -4,14 +4,15 @@ from typing import List, Optional
4
import torch
5
import torch.distributed
6
7
-from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8
- SchedulerConfig)
7
+from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8
+ ParallelConfig, SchedulerConfig)
9
from vllm.model_executor import set_random_seed
10
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
11
from vllm.worker.neuron_model_runner import NeuronModelRunner
12
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
13
14
14
-class NeuronWorker:
15
+class NeuronWorker(LoraNotSupportedWorkerBase):
16
"""A worker class that executes the model on a group of neuron cores.
17
"""
18
@@ -21,11 +22,13 @@ class NeuronWorker:
22
parallel_config: ParallelConfig,
23
scheduler_config: SchedulerConfig,
24
device_config: DeviceConfig,
25
+ cache_config: CacheConfig,
26
) -> None:
27
self.model_config = model_config
28
self.parallel_config = parallel_config
29
self.scheduler_config = scheduler_config
30
self.device_config = device_config
31
+ self.cache_config = cache_config
32
33
self.model_runner = NeuronModelRunner(model_config, parallel_config,
34
scheduler_config, device_config)
@@ -37,6 +40,35 @@ class NeuronWorker:
40
def load_model(self):
41
self.model_runner.load_model()
42
43
+ def determine_num_available_blocks(self) -> tuple[int, int]:
44
+ """Determine the number of available KV blocks.
45
+
46
+ Swapping is not yet supported, so always return num_cpu_blocks=0.
47
+
48
+ We configure num_gpu_blocks to be equal to max_num_seqs.
49
+ """
50
+ # Set the number of GPU blocks to be the same as the maximum number of
51
+ # sequences that can be processed in a single batch. This is equivalent
52
+ # to schedule without PagedAttention.
53
+ num_gpu_blocks = self.scheduler_config.max_num_seqs
54
+
55
+ # Swap not yet supported with Neuron backend.
56
+ num_cpu_blocks = 0
57
+
58
+ return num_gpu_blocks, num_cpu_blocks
59
+
60
+ def initialize_cache(self, num_gpu_blocks: int,
61
+ num_cpu_blocks: int) -> None:
62
+ """Initialize the KV cache.
63
+ """
64
+
65
+ # Different values are not tested.
66
+ assert num_cpu_blocks == 0
67
+ assert num_gpu_blocks == self.scheduler_config.max_num_seqs
68
+
69
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
70
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
71
+
72
@torch.inference_mode()
73
def execute_model(
74
self,
@@ -50,3 +82,10 @@ class NeuronWorker:
82
83
output = self.model_runner.execute_model(seq_group_metadata_list)
84
return output
85
+
86
+ def get_cache_block_size_bytes(self) -> int:
87
+ """Determine the size in bytes of a cache block.
88
+
89
+ This is required for speculative decoding; it is not yet implemented.
90
+ """
91
+ raise NotImplementedError
vllm/worker/worker.py
+59
-30
@@ -19,9 +19,10 @@ from vllm.model_executor.parallel_utils.parallel_state import (
19
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20
from vllm.worker.cache_engine import CacheEngine
21
from vllm.worker.model_runner import ModelRunner
22
+from vllm.worker.worker_base import WorkerBase
23
24
24
-class Worker:
25
+class Worker(WorkerBase):
26
"""A worker class that executes (a partition of) the model on a GPU.
27
28
Each worker is associated with a single GPU. The worker is responsible for
@@ -35,18 +36,19 @@ class Worker:
36
parallel_config: ParallelConfig,
37
scheduler_config: SchedulerConfig,
38
device_config: DeviceConfig,
39
+ cache_config: CacheConfig,
40
local_rank: int,
41
rank: int,
42
distributed_init_method: str,
43
lora_config: Optional[LoRAConfig] = None,
44
vision_language_config: Optional[VisionLanguageConfig] = None,
43
- kv_cache_dtype: Optional[str] = "auto",
45
is_driver_worker: bool = False,
46
) -> None:
47
self.model_config = model_config
48
self.parallel_config = parallel_config
49
self.scheduler_config = scheduler_config
50
self.device_config = device_config
51
+ self.cache_config = cache_config
52
self.local_rank = local_rank
53
self.rank = rank
54
self.distributed_init_method = distributed_init_method
@@ -66,12 +68,11 @@ class Worker:
68
scheduler_config,
69
device_config,
70
lora_config=self.lora_config,
69
- kv_cache_dtype=kv_cache_dtype,
71
+ kv_cache_dtype=self.cache_config.cache_dtype,
72
is_driver_worker=is_driver_worker,
73
vision_language_config=vision_language_config)
74
# Uninitialized cache engine. Will be initialized by
73
- # self.init_cache_engine().
74
- self.cache_config = None
75
+ # initialize_cache.
76
self.cache_engine = None
77
self.gpu_cache = None
78
@@ -107,20 +108,17 @@ class Worker:
108
self.model_runner.load_model()
109
110
@torch.inference_mode()
110
- def profile_num_available_blocks(
111
- self,
112
- block_size: int,
113
- gpu_memory_utilization: float,
114
- cpu_swap_space: int,
115
- cache_dtype: str,
116
- ) -> Tuple[int, int]:
117
- """Profiles the peak memory usage of the model and returns the maximum
118
- number of GPU and CPU cache blocks that can be allocated.
119
-
120
- Args:
121
- block_size: The size of the cache block.
122
- gpu_memory_utilization: The fraction of the total GPU memory to use.
123
- cpu_swap_space: The size of the CPU swap space in bytes.
111
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
112
+ """Profiles the peak memory usage of the model to determine how many
113
+ KV blocks may be allocated without OOMs.
114
+
115
+ The engine will first conduct a profiling of the existing memory usage.
116
+ Then, it calculate the maximum possible number of GPU and CPU blocks
117
+ that can be allocated with the remaining free memory.
118
+
119
+ .. tip::
120
+ You may limit the usage of GPU memory
121
+ by adjusting the `gpu_memory_utilization` parameter.
122
"""
123
# Profile the memory usage of the model and get the maximum number of
124
# cache blocks that can be allocated with the remaining free memory.
@@ -141,12 +139,12 @@ class Worker:
139
"Error in memory profiling. This happens when the GPU memory was "
140
"not properly cleaned up before initializing the vLLM instance.")
141
144
- cache_block_size = self.get_cache_block_size_bytes(
145
- block_size, cache_dtype)
142
+ cache_block_size = self.get_cache_block_size_bytes()
143
num_gpu_blocks = int(
147
- (total_gpu_memory * gpu_memory_utilization - peak_memory) //
148
- cache_block_size)
149
- num_cpu_blocks = int(cpu_swap_space // cache_block_size)
144
+ (total_gpu_memory * self.cache_config.gpu_memory_utilization -
145
+ peak_memory) // cache_block_size)
146
+ num_cpu_blocks = int(self.cache_config.swap_space_bytes //
147
+ cache_block_size)
148
num_gpu_blocks = max(num_gpu_blocks, 0)
149
num_cpu_blocks = max(num_cpu_blocks, 0)
150
if self.model_runner.lora_manager:
@@ -155,14 +153,30 @@ class Worker:
153
torch.cuda.empty_cache()
154
return num_gpu_blocks, num_cpu_blocks
155
158
- def init_cache_engine(self, cache_config: CacheConfig) -> None:
159
- self.cache_config = cache_config
156
+ def initialize_cache(self, num_gpu_blocks: int,
157
+ num_cpu_blocks: int) -> None:
158
+ """Allocate GPU and CPU KV cache with the specified number of blocks.
159
+
160
+ This also warms up the model, which may record CUDA graphs.
161
+ """
162
+ raise_if_cache_size_invalid(num_gpu_blocks,
163
+ self.cache_config.block_size,
164
+ self.model_config.max_model_len)
165
+
166
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
167
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
168
+
169
+ self._init_cache_engine()
170
+ self._warm_up_model()
171
+
172
+ def _init_cache_engine(self):
173
+ assert self.cache_config.num_gpu_blocks is not None
174
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
175
self.parallel_config)
176
self.gpu_cache = self.cache_engine.gpu_cache
177
self.model_runner.set_block_size(self.cache_engine.block_size)
178
165
- def warm_up_model(self) -> None:
179
+ def _warm_up_model(self) -> None:
180
if not self.model_config.enforce_eager:
181
self.model_runner.capture_model(self.gpu_cache)
182
# Reset the seed to ensure that the random state is not affected by
@@ -239,11 +253,10 @@ class Worker:
253
def vocab_size(self) -> int:
254
return self.model_runner.vocab_size
255
242
- def get_cache_block_size_bytes(self, block_size: int,
243
- cache_dtype: str) -> int:
256
+ def get_cache_block_size_bytes(self) -> int:
257
"""Get the size of the KV cache block size in bytes.
258
"""
246
- return CacheEngine.get_cache_block_size(block_size, cache_dtype,
259
+ return CacheEngine.get_cache_block_size(self.cache_config,
260
self.model_config,
261
self.parallel_config)
262
@@ -300,3 +313,19 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
313
f"{compute_capability[0]}.{compute_capability[1]}. "
314
"You can use float16 instead by explicitly setting the"
315
"`dtype` flag in CLI, for example: --dtype=half.")
316
+
317
+
318
+def raise_if_cache_size_invalid(num_gpu_blocks, block_size,
319
+ max_model_len) -> None:
320
+ if num_gpu_blocks <= 0:
321
+ raise ValueError("No available memory for the cache blocks. "
322
+ "Try increasing `gpu_memory_utilization` when "
323
+ "initializing the engine.")
324
+ max_seq_len = block_size * num_gpu_blocks
325
+ if max_model_len > max_seq_len:
326
+ raise ValueError(
327
+ f"The model's max seq len ({max_model_len}) "
328
+ "is larger than the maximum number of tokens that can be "
329
+ f"stored in KV cache ({max_seq_len}). Try increasing "
330
+ "`gpu_memory_utilization` or decreasing `max_model_len` when "
331
+ "initializing the engine.")
vllm/worker/worker_base.py
new
+83
@@ -0,0 +1,83 @@
1
+from abc import ABC, abstractmethod
2
+from typing import Dict, List
3
+
4
+from vllm.lora.request import LoRARequest
5
+from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6
+
7
+
8
+class WorkerBase(ABC):
9
+ """Worker interface that allows vLLM to cleanly separate implementations for
10
+ different hardware.
11
+ """
12
+
13
+ @abstractmethod
14
+ def init_device(self) -> None:
15
+ """Initialize device state, such as loading the model or other on-device
16
+ memory allocations.
17
+ """
18
+ raise NotImplementedError
19
+
20
+ @abstractmethod
21
+ def determine_num_available_blocks(self) -> tuple[int, int]:
22
+ """Determine the number of available blocks for the GPU KV cache and
23
+ swappable CPU KV cache.
24
+
25
+ The implementation may run profiling or other heuristics to determine
26
+ the size of caches.
27
+
28
+ Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
29
+ are blocks that are "active" on the device and can be appended to.
30
+ num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
31
+ appended to.
32
+ """
33
+ raise NotImplementedError
34
+
35
+ @abstractmethod
36
+ def initialize_cache(self, num_gpu_blocks: int,
37
+ num_cpu_blocks: int) -> None:
38
+ """Initialize the KV cache with the given size in blocks.
39
+ """
40
+ raise NotImplementedError
41
+
42
+ @abstractmethod
43
+ def execute_model(self,
44
+ seq_group_metadata_list: List[SequenceGroupMetadata],
45
+ blocks_to_swap_in: Dict[int, int],
46
+ blocks_to_swap_out: Dict[int, int],
47
+ blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
48
+ """Executes one model step on the given sequences."""
49
+ raise NotImplementedError
50
+
51
+ @abstractmethod
52
+ def get_cache_block_size_bytes() -> int:
53
+ """Return the size of a single cache block, in bytes. Used in
54
+ speculative decoding.
55
+ """
56
+ raise NotImplementedError
57
+
58
+ @abstractmethod
59
+ def add_lora(self, lora_request: LoRARequest) -> bool:
60
+ raise NotImplementedError
61
+
62
+ @abstractmethod
63
+ def remove_lora(self, lora_id: int) -> bool:
64
+ raise NotImplementedError
65
+
66
+ @abstractmethod
67
+ def list_loras(self) -> List[int]:
68
+ raise NotImplementedError
69
+
70
+
71
+class LoraNotSupportedWorkerBase(WorkerBase):
72
+ """Partial implementation of WorkerBase that raises exceptions when LoRA
73
+ methods are invoked.
74
+ """
75
+
76
+ def add_lora(self, lora_request: LoRARequest) -> bool:
77
+ raise ValueError(f"{type(self)} does not support LoRA")
78
+
79
+ def remove_lora(self, lora_id: int) -> bool:
80
+ raise ValueError(f"{type(self)} does not support LoRA")
81
+
82
+ def list_loras(self) -> List[int]:
83
+ raise ValueError(f"{type(self)} does not support LoRA")