A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1] Move more control of kv cache initialization from model_executor to EngineCore (#11960)
Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Chen Zhang committed
Jan 17, 2025 at 15:39 UTC
69d765f5a5bbbe1ea9843be19b9480660fc5bc8b
12 files changed
+515
-104
tests/v1/test_utils.py
new
+62
@@ -0,0 +1,62 @@
1
+from typing import List
2
+
3
+import torch
4
+
5
+from vllm.v1.utils import bind_kv_cache
6
+
7
+
8
+def test_bind_kv_cache():
9
+ from vllm.attention import Attention
10
+
11
+ ctx = {
12
+ 'layers.0.self_attn': Attention(32, 128, 0.1),
13
+ 'layers.1.self_attn': Attention(32, 128, 0.1),
14
+ 'layers.2.self_attn': Attention(32, 128, 0.1),
15
+ 'layers.3.self_attn': Attention(32, 128, 0.1),
16
+ }
17
+ kv_cache = {
18
+ 'layers.0.self_attn': torch.zeros((1, )),
19
+ 'layers.1.self_attn': torch.zeros((1, )),
20
+ 'layers.2.self_attn': torch.zeros((1, )),
21
+ 'layers.3.self_attn': torch.zeros((1, )),
22
+ }
23
+ runner_kv_caches: List[torch.Tensor] = []
24
+ bind_kv_cache(kv_cache, ctx, runner_kv_caches)
25
+ assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[
26
+ 'layers.0.self_attn']
27
+ assert ctx['layers.1.self_attn'].kv_cache[0] is kv_cache[
28
+ 'layers.1.self_attn']
29
+ assert ctx['layers.2.self_attn'].kv_cache[0] is kv_cache[
30
+ 'layers.2.self_attn']
31
+ assert ctx['layers.3.self_attn'].kv_cache[0] is kv_cache[
32
+ 'layers.3.self_attn']
33
+
34
+ assert runner_kv_caches[0] is kv_cache['layers.0.self_attn']
35
+ assert runner_kv_caches[1] is kv_cache['layers.1.self_attn']
36
+ assert runner_kv_caches[2] is kv_cache['layers.2.self_attn']
37
+ assert runner_kv_caches[3] is kv_cache['layers.3.self_attn']
38
+
39
+
40
+def test_bind_kv_cache_non_attention():
41
+ from vllm.attention import Attention
42
+
43
+ # example from Jamba PP=2
44
+ ctx = {
45
+ 'model.layers.20.attn': Attention(32, 128, 0.1),
46
+ 'model.layers.28.attn': Attention(32, 128, 0.1),
47
+ }
48
+ kv_cache = {
49
+ 'model.layers.20.attn': torch.zeros((1, )),
50
+ 'model.layers.28.attn': torch.zeros((1, )),
51
+ }
52
+
53
+ runner_kv_caches: List[torch.Tensor] = []
54
+ bind_kv_cache(kv_cache, ctx, runner_kv_caches)
55
+
56
+ assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[
57
+ 'model.layers.20.attn']
58
+ assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[
59
+ 'model.layers.28.attn']
60
+
61
+ assert runner_kv_caches[0] is kv_cache['model.layers.20.attn']
62
+ assert runner_kv_caches[1] is kv_cache['model.layers.28.attn']
vllm/attention/layer.py
+2
@@ -101,7 +101,9 @@ class Attention(nn.Module):
101
self.num_heads = num_heads
102
self.head_size = head_size
103
self.num_kv_heads = num_kv_heads
104
+ self.sliding_window = sliding_window
105
self.backend = backend_name_to_enum(attn_backend.get_name())
106
+ self.dtype = dtype
107
108
# For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
109
# torch.compile works by registering the attention as one giant
vllm/v1/core/kv_cache_utils.py
+124
@@ -3,7 +3,10 @@ from collections.abc import Sequence
3
from dataclasses import dataclass
4
from typing import Any, List, NamedTuple, Optional, Tuple
5
6
+from vllm.config import VllmConfig
7
from vllm.logger import init_logger
8
+from vllm.v1.kv_cache_interface import (KVCacheConfig, KVCacheSpec,
9
+ KVCacheTensor)
10
from vllm.v1.request import Request
11
12
logger = init_logger(__name__)
@@ -305,3 +308,124 @@ def hash_request_tokens(block_size: int,
308
ret.append(block_hash)
309
parent_block_hash_value = block_hash.hash_value
310
return ret
311
+
312
+
313
+def check_enough_kv_cache_memory(vllm_config: VllmConfig,
314
+ kv_cache_spec: KVCacheSpec,
315
+ available_memory: int):
316
+ """
317
+ Checks whether `available_memory` is enough for the KV cache to hold at
318
+ least one request with the model's max_model_len.
319
+
320
+ Args:
321
+ vllm_config: The global VllmConfig
322
+ kv_cache_spec: The kv cache spec of the model
323
+ available_memory: Memory available for KV cache in bytes.
324
+
325
+ Raises:
326
+ ValueError: If there is not enough memory available for the KV cache.
327
+ """
328
+
329
+ if available_memory <= 0:
330
+ raise ValueError("No available memory for the cache blocks. "
331
+ "Try increasing `gpu_memory_utilization` when "
332
+ "initializing the engine.")
333
+
334
+ max_model_len = vllm_config.model_config.max_model_len
335
+ needed_memory = 0
336
+ for layer_spec in kv_cache_spec.values():
337
+ needed_memory += layer_spec.bytes_for_tokens(max_model_len)
338
+
339
+ if needed_memory > available_memory:
340
+ raise ValueError(
341
+ f"To serve at least one request with the models's max seq len "
342
+ f"({max_model_len}), ({needed_memory/1024/1024/1024:.2f} GB KV "
343
+ f"cache is needed, which is larger than the available KV cache "
344
+ f"memory ({available_memory/1024/1024/1024:.2f} GB). Try "
345
+ f"increasing `gpu_memory_utilization` or decreasing "
346
+ f"`max_model_len` when initializing the engine.")
347
+
348
+
349
+def is_kv_cache_type_uniform(kv_cache_spec: KVCacheSpec) -> bool:
350
+ """
351
+ Whether all layers in the given KVCacheSpec have the same type of KV cache.
352
+
353
+ Args:
354
+ kv_cache_spec: The KVCacheSpec of the model
355
+
356
+ Returns:
357
+ True if all layers have the same type, False otherwise.
358
+ """
359
+
360
+ layer_keys = set(layer.type_id for layer in kv_cache_spec.values())
361
+ return len(layer_keys) == 1
362
+
363
+
364
+def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
365
+ kv_cache_spec: KVCacheSpec,
366
+ available_memory: int) -> KVCacheConfig:
367
+ """
368
+ Generates the KV cache configuration for a model with one type of KV cache.
369
+ Divide the available memory equally among all layers.
370
+
371
+ Args:
372
+ vllm_config: The global VllmConfig
373
+ kv_cache_spec: The kv cache spec of the model
374
+ available_memory: Memory available for KV cache in bytes.
375
+
376
+ Returns:
377
+ The generated KVCacheConfig
378
+ """
379
+
380
+ page_sizes = {layer.page_size_bytes for layer in kv_cache_spec.values()}
381
+ assert len(page_sizes) == 1
382
+ page_size = page_sizes.pop()
383
+
384
+ num_blocks = int(available_memory // page_size // len(kv_cache_spec))
385
+ num_blocks = max(num_blocks, 0)
386
+
387
+ if vllm_config.cache_config.num_gpu_blocks_override is not None:
388
+ num_gpu_blocks_override = \
389
+ vllm_config.cache_config.num_gpu_blocks_override
390
+ logger.info(
391
+ "Overriding num_gpu_blocks=%d with "
392
+ "num_gpu_blocks_override=%d", num_blocks, num_gpu_blocks_override)
393
+ num_blocks = num_gpu_blocks_override
394
+
395
+ logger.info("# GPU blocks: %d", num_blocks)
396
+
397
+ per_layer_size = page_size * num_blocks
398
+
399
+ kv_cache_config = KVCacheConfig(
400
+ num_blocks=num_blocks,
401
+ tensors={
402
+ layer_name: KVCacheTensor(size=per_layer_size)
403
+ for layer_name in kv_cache_spec
404
+ },
405
+ groups=[[layer_name for layer_name in kv_cache_spec]],
406
+ kv_cache_spec=kv_cache_spec)
407
+ return kv_cache_config
408
+
409
+
410
+def get_kv_cache_config(vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec,
411
+ available_memory: int) -> KVCacheConfig:
412
+ """
413
+ Generates the KV cache configuration for a model
414
+ TODO: support hybrid models with more than one type of KV cache.
415
+
416
+ Args:
417
+ vllm_config: The global VllmConfig
418
+ kv_cache_spec: The kv cache spec of the model
419
+ available_memory: Memory available for KV cache in bytes.
420
+
421
+ Returns:
422
+ The generated KVCacheConfig
423
+ """
424
+ check_enough_kv_cache_memory(vllm_config, kv_cache_spec, available_memory)
425
+ if is_kv_cache_type_uniform(kv_cache_spec):
426
+ # KV cache of all layers are the same, which is true for most models.
427
+ # Allocate the same amount of memory for each layer.
428
+ return _get_kv_cache_config_uniform_type(vllm_config, kv_cache_spec,
429
+ available_memory)
430
+ else:
431
+ raise NotImplementedError
vllm/v1/engine/core.py
+18
-13
@@ -11,11 +11,12 @@ import zmq
11
import zmq.asyncio
12
from msgspec import msgpack
13
14
-from vllm.config import CacheConfig, VllmConfig
14
+from vllm.config import VllmConfig
15
from vllm.logger import init_logger
16
from vllm.transformers_utils.config import (
17
maybe_register_config_serialize_by_value)
18
from vllm.utils import get_exception_traceback, zmq_socket_ctx
19
+from vllm.v1.core.kv_cache_utils import get_kv_cache_config
20
from vllm.v1.core.scheduler import Scheduler
21
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
22
EngineCoreRequest, EngineCoreRequestType,
@@ -49,7 +50,7 @@ class EngineCore:
50
51
# Setup KV Caches and update CacheConfig after profiling.
52
num_gpu_blocks, num_cpu_blocks = self._initialize_kv_caches(
52
- vllm_config.cache_config)
53
+ vllm_config)
54
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
55
vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
56
@@ -65,21 +66,25 @@ class EngineCore:
66
vllm_config.model_config)
67
68
def _initialize_kv_caches(self,
68
- cache_config: CacheConfig) -> Tuple[int, int]:
69
+ vllm_config: VllmConfig) -> Tuple[int, int]:
70
start = time.time()
70
- num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
71
- )
71
73
- if cache_config.num_gpu_blocks_override is not None:
74
- num_gpu_blocks_override = cache_config.num_gpu_blocks_override
75
- logger.info(
76
- "Overriding num_gpu_blocks=%d with "
77
- "num_gpu_blocks_override=%d", num_gpu_blocks,
78
- num_gpu_blocks_override)
79
- num_gpu_blocks = num_gpu_blocks_override
72
+ # Get all kv cache needed by the model
73
+ kv_cache_spec = self.model_executor.get_kv_cache_spec()
74
+
75
+ # Profiles the peak memory usage of the model to determine how much
76
+ # memory can be allocated for kv cache.
77
+ availble_gpu_memory = self.model_executor.determine_available_memory()
78
79
+ # Get the kv cache tensor size
80
+ kv_cache_config = get_kv_cache_config(vllm_config, kv_cache_spec,
81
+ availble_gpu_memory)
82
+ num_gpu_blocks = kv_cache_config.num_blocks
83
num_cpu_blocks = 0
82
- self.model_executor.initialize(num_gpu_blocks)
84
+
85
+ # Initialize kv cache and warmup the execution
86
+ self.model_executor.initialize(kv_cache_config)
87
+
88
elapsed = time.time() - start
89
logger.info(("init engine (profile, create kv cache, "
90
"warmup model) took %.2f seconds"), elapsed)
vllm/v1/executor/abstract.py
+8
-3
@@ -1,7 +1,8 @@
1
from abc import ABC, abstractmethod
2
-from typing import Tuple, Type
2
+from typing import Type
3
4
from vllm.config import VllmConfig
5
+from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
6
from vllm.v1.outputs import ModelRunnerOutput
7
8
@@ -31,11 +32,15 @@ class Executor(ABC):
32
raise NotImplementedError
33
34
@abstractmethod
34
- def initialize(self, num_gpu_blocks: int) -> None:
35
+ def initialize(self, kv_cache_config: KVCacheConfig) -> None:
36
raise NotImplementedError
37
38
@abstractmethod
38
- def determine_num_available_blocks(self) -> Tuple[int, int]:
39
+ def determine_available_memory(self) -> int: # in bytes
40
+ raise NotImplementedError
41
+
42
+ @abstractmethod
43
+ def get_kv_cache_spec(self) -> KVCacheSpec:
44
raise NotImplementedError
45
46
@abstractmethod
vllm/v1/executor/multiproc_executor.py
+15
-10
@@ -23,6 +23,7 @@ from vllm.logger import init_logger
23
from vllm.utils import (get_distributed_init_method, get_mp_context,
24
get_open_port, get_open_zmq_ipc_path, zmq_socket_ctx)
25
from vllm.v1.executor.abstract import Executor
26
+from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
27
from vllm.v1.outputs import ModelRunnerOutput
28
from vllm.worker.worker_base import WorkerWrapperBase
29
@@ -90,29 +91,33 @@ class MultiprocExecutor(Executor):
91
for w in self.workers:
92
w.worker_response_mq.wait_until_ready()
93
93
- def initialize(self, num_gpu_blocks: int) -> None:
94
+ def initialize(self, kv_cache_config: KVCacheConfig) -> None:
95
"""
96
Initialize the KV caches and begin the model execution loop of the
97
underlying workers.
98
"""
98
- logger.info("# GPU blocks: %d", num_gpu_blocks)
99
- self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
99
+ self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
100
self.collective_rpc("compile_or_warm_up_model")
101
102
- def determine_num_available_blocks(self) -> Tuple[int, int]:
102
+ def determine_available_memory(self) -> int:
103
"""
104
- Determine the number of available KV blocks by invoking the
104
+ Determine the available memory (in bytes) for KV cache by invoking the
105
underlying worker.
106
"""
107
- num_blocks = self.collective_rpc("determine_num_available_blocks")
107
+ memory_sizes = self.collective_rpc("determine_available_memory")
108
109
# Since we use a shared centralized controller, we take the minimum
110
- # number of blocks across all workers to make sure all the memory
110
+ # memory size across all workers to make sure all the memory
111
# operators can be applied to all workers.
112
- num_gpu_blocks = min(b[0] for b in num_blocks)
113
- num_cpu_blocks = min(b[1] for b in num_blocks)
112
+ return min(memory_sizes)
113
115
- return num_gpu_blocks, num_cpu_blocks
114
+ def get_kv_cache_spec(self) -> KVCacheSpec:
115
+ """
116
+ Get all kv cache needed by the model by invoking the underlying worker.
117
+ """
118
+ kv_cache_specs = self.collective_rpc("get_kv_cache_spec")
119
+ assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
120
+ return kv_cache_specs[0]
121
122
def collective_rpc(self,
123
method: str,
vllm/v1/executor/ray_executor.py
+21
-19
@@ -10,6 +10,7 @@ from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10
from vllm.v1.executor.abstract import Executor
11
from vllm.v1.executor.ray_utils import (RayWorkerWrapper,
12
initialize_ray_cluster, ray)
13
+from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
14
from vllm.v1.outputs import ModelRunnerOutput
15
16
if ray is not None:
@@ -211,39 +212,40 @@ class RayExecutor(Executor):
212
distributed_init_method=distributed_init_method,
213
)
214
214
- def determine_num_available_blocks(self) -> Tuple[int, int]:
215
+ def determine_available_memory(self) -> int:
216
"""
216
- Determine the number of available KV blocks.
217
+ Determine the available GPU memory in bytes.
218
218
- This invokes `determine_num_available_blocks` on each worker and takes
219
+ This invokes `determine_available_memory` on each worker and takes
220
the min of the results, guaranteeing that the selected cache sizes are
221
compatible with all workers.
221
-
222
- Returns:
223
- - tuple[num_gpu_blocks, num_cpu_blocks]
222
"""
225
- # Get the maximum number of blocks that can be allocated on GPU and CPU.
226
- num_blocks = self._run_workers("determine_num_available_blocks")
223
+
224
+ memory_sizes = self._run_workers("determine_available_memory")
225
226
# Since we use a shared centralized controller, we take the minimum
229
- # number of blocks across all workers to make sure all the memory
227
+ # memory size across all workers to make sure all the memory
228
# operators can be applied to all workers.
231
- num_gpu_blocks = min(b[0] for b in num_blocks)
232
- num_cpu_blocks = min(b[1] for b in num_blocks)
229
+ return min(memory_sizes)
230
234
- return num_gpu_blocks, num_cpu_blocks
235
-
236
- def initialize(self, num_gpu_blocks: int) -> None:
231
+ def initialize(self, kv_cache_config: KVCacheConfig) -> None:
232
"""
233
Initialize the KV cache in all workers.
234
"""
240
- # NOTE: This is logged in the executor because there can be >1 worker
241
- # with other executors. We could log in the engine level, but work
242
- # remains to abstract away the device for non-GPU configurations.
243
- logger.info("# GPU blocks: %d", num_gpu_blocks)
244
- self._run_workers("initialize_cache", num_gpu_blocks)
235
+ self._run_workers("initialize_cache", kv_cache_config)
236
self._run_workers("compile_or_warm_up_model")
237
238
+ def get_kv_cache_spec(self) -> KVCacheSpec:
239
+ """
240
+ Get all kv cache needed by the model
241
+
242
+ This invokes `get_kv_cache_spec` on each worker and asserts that
243
+ they are identical. The KVCacheSpec is then returned.
244
+ """
245
+ kv_cache_specs = self._run_workers("get_kv_cache_spec")
246
+ assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
247
+ return kv_cache_specs[0]
248
+
249
def _run_workers(
250
self,
251
method: str,
vllm/v1/executor/uniproc_executor.py
+14
-11
@@ -1,10 +1,11 @@
1
import os
2
-from typing import Optional, Tuple
2
+from typing import Optional
3
4
from vllm.config import VllmConfig
5
from vllm.logger import init_logger
6
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7
from vllm.v1.executor.abstract import Executor
8
+from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
9
from vllm.v1.outputs import ModelRunnerOutput
10
from vllm.v1.worker.gpu_worker import Worker
11
@@ -49,20 +50,22 @@ class UniprocExecutor(Executor):
50
distributed_init_method=distributed_init_method,
51
)
52
52
- def determine_num_available_blocks(self) -> Tuple[int, int]:
53
- """Determine the number of available KV blocks by invoking the
54
- underlying worker.
53
+ def determine_available_memory(self) -> int:
54
+ """Determine the available memory (in bytes) for KV cache by invoking
55
+ the underlying worker.
56
"""
56
- return self.worker.determine_num_available_blocks()
57
+ return self.worker.determine_available_memory()
58
58
- def initialize(self, num_gpu_blocks: int) -> None:
59
+ def get_kv_cache_spec(self) -> KVCacheSpec:
60
+ """Get all kv cache needed by the model by invoking the underlying
61
+ worker.
62
+ """
63
+ return self.worker.get_kv_cache_spec()
64
+
65
+ def initialize(self, kv_cache_config: KVCacheConfig) -> None:
66
"""Initialize the KV cache by invoking the underlying worker.
67
"""
61
- # NOTE: This is logged in the executor because there can be >1 worker
62
- # with other executors. We could log in the engine level, but work
63
- # remains to abstract away the device for non-GPU configurations.
64
- logger.info("# GPU blocks: %d", num_gpu_blocks)
65
- self.worker.initialize_cache(num_gpu_blocks)
68
+ self.worker.initialize_cache(kv_cache_config)
69
self.worker.compile_or_warm_up_model()
70
71
def execute_model(
vllm/v1/kv_cache_interface.py
new
+111
@@ -0,0 +1,111 @@
1
+from dataclasses import dataclass
2
+from typing import Dict, List
3
+
4
+import torch
5
+
6
+from vllm.logger import init_logger
7
+from vllm.utils import cdiv, get_dtype_size
8
+
9
+logger = init_logger(__name__)
10
+
11
+
12
+@dataclass
13
+class KVCacheSpecBase:
14
+ """
15
+ A base class for specifying the KV cache format of one layer.
16
+ """
17
+
18
+ # number of tokens in a block
19
+ block_size: int
20
+
21
+ @property
22
+ def type_id(self) -> str:
23
+ """
24
+ The type identifier of this KV cache.
25
+ Return different strings for layers with different KV cache type (e.g.,
26
+ different number of tokens like full attention vs sliding window
27
+ attention, different KV cache size per token like layers with different
28
+ number of heads)
29
+
30
+ Returns:
31
+ The type identifier of this KV cache.
32
+ """
33
+ raise NotImplementedError
34
+
35
+ @property
36
+ def page_size_bytes(self) -> int:
37
+ """
38
+ The size of a page with `block_size` tokens in bytes.
39
+
40
+ Returns:
41
+ The page size
42
+ """
43
+ raise NotImplementedError
44
+
45
+ def bytes_for_tokens(self, num_tokens: int) -> int:
46
+ """
47
+ The KV cache size for `num_tokens` tokens in bytes. Returns the real
48
+ memory size after padding `num_tokens` to full blocks.
49
+
50
+ Returns:
51
+ The KV cache size
52
+ """
53
+ raise NotImplementedError
54
+
55
+
56
+@dataclass
57
+class FullAttentionSpec(KVCacheSpecBase):
58
+ num_kv_heads: int
59
+ head_size: int
60
+ dtype: torch.dtype
61
+
62
+ @property
63
+ def type_id(self) -> str:
64
+ return f"full_attention_{self.block_size}_{self.page_size_bytes}"
65
+
66
+ @property
67
+ def page_size_bytes(self) -> int:
68
+ return 2 * self.block_size * self.num_kv_heads * self.head_size \
69
+ * get_dtype_size(self.dtype)
70
+
71
+ def bytes_for_tokens(self, num_tokens: int) -> int:
72
+ return cdiv(num_tokens, self.block_size) * self.page_size_bytes
73
+
74
+
75
+KVCacheSpec = Dict[str, KVCacheSpecBase]
76
+
77
+
78
+@dataclass
79
+class KVCacheTensor:
80
+ """
81
+ A dataclass for specifying how the workers should initialize the KV cache
82
+ for a layer. Only contains the size of KV cache for that layer for now. Will
83
+ be extended to support multiple layers sharing the same memory pool.
84
+ """
85
+ size: int # The size of KV cache Tensor in bytes
86
+
87
+
88
+@dataclass
89
+class KVCacheConfig:
90
+ """
91
+ The KV cache configuration of a model.
92
+ """
93
+ """The number of KV cache blocks"""
94
+ num_blocks: int
95
+ """layer_name -> how to initialize KV cache for that layer"""
96
+ tensors: Dict[str, KVCacheTensor]
97
+ """
98
+ A list of kv-cache groups. Each group includes a set of layers with
99
+ the same kv-cache spec, and the total page_size of layers inside a group
100
+ is same across all groups (as the KVCacheManager only supports allocating
101
+ pages of the same size). For example:
102
+ 1. A model only uses full attention: one group with all layers in the model.
103
+ 2. (not implemented yet) A model with the same number of full attention
104
+ layers and sliding window attention layers: two groups, one for full
105
+ attention layers and one for sliding window attention layers.
106
+ 3. (not implemented yet) A model with 2 full attention layers and 4 sliding
107
+ window attention layers: three groups, (full * 2), (sw * 2), (sw * 2).
108
+ """
109
+ groups: List[List[str]]
110
+ """the KVCacheSpec of the model"""
111
+ kv_cache_spec: KVCacheSpec
vllm/v1/utils.py
+54
-2
@@ -1,13 +1,20 @@
1
import multiprocessing
2
import os
3
import weakref
4
+from collections import defaultdict
5
from collections.abc import Sequence
5
-from typing import (Any, Callable, Dict, Generic, List, Optional, TypeVar,
6
- Union, overload)
6
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Generic, List,
7
+ Optional, TypeVar, Union, overload)
8
+
9
+import torch
10
11
from vllm.logger import init_logger
12
+from vllm.model_executor.models.utils import extract_layer_index
13
from vllm.utils import get_mp_context, kill_process_tree
14
15
+if TYPE_CHECKING:
16
+ from vllm.attention.layer import Attention
17
+
18
logger = init_logger(__name__)
19
20
T = TypeVar("T")
@@ -134,3 +141,48 @@ def shutdown(proc: multiprocessing.Process, input_path: str, output_path: str):
141
socket_file = ipc_socket.replace("ipc://", "")
142
if os and os.path.exists(socket_file):
143
os.remove(socket_file)
144
+
145
+
146
+def bind_kv_cache(
147
+ kv_caches: Dict[str, torch.Tensor],
148
+ forward_context: Dict[str, "Attention"],
149
+ runner_kv_caches: List[torch.Tensor],
150
+) -> None:
151
+ """
152
+ Bind the allocated KV cache to both ModelRunner and forward context so
153
+ that the KV cache can be used in the forward pass.
154
+
155
+ This function:
156
+ 1) Fills the ModelRunner's kv cache list (`runner_kv_caches`) with
157
+ kv_caches.
158
+ 2) Associates each attention layer in the `forward_context` with its
159
+ corresponding KV cache in kv_caches.
160
+
161
+ Args:
162
+ kv_caches: The allocated kv_caches with layer names as keys.
163
+ forward_context: The global forward context containing all Attention
164
+ layers with layer names as keys.
165
+ runner_kv_caches: The kv_cache declared by ModelRunner.
166
+ """
167
+ # Bind kv_caches to ModelRunner
168
+ assert len(runner_kv_caches) == 0
169
+
170
+ # Convert kv_caches dict to a list of tensors in the order of layer_index.
171
+ index2name = defaultdict(list)
172
+ for layer_name in kv_caches:
173
+ index2name[extract_layer_index(layer_name)].append(layer_name)
174
+
175
+ for layer_index in sorted(index2name.keys()):
176
+ layer_names = index2name[layer_index]
177
+ if len(layer_names) > 1:
178
+ # One typical case is encoder-decoder model, e.g., bart.
179
+ # The cross attention and self attention in the same decoder layer
180
+ # has different layer_name but the same layer_index.
181
+ raise NotImplementedError
182
+ layer_name = layer_names[0]
183
+ runner_kv_caches.append(kv_caches[layer_name])
184
+
185
+ # Bind kv_caches to forward context
186
+ for layer_name, kv_cache in kv_caches.items():
187
+ # NOTE: Use list because of v0 PP virtual engine.
188
+ forward_context[layer_name].kv_cache = [kv_cache]
vllm/v1/worker/gpu_model_runner.py
+72
-12
@@ -7,6 +7,8 @@ import torch
7
import torch.distributed
8
import torch.nn as nn
9
10
+from vllm.attention.backends.abstract import AttentionType
11
+from vllm.attention.layer import Attention
12
from vllm.config import CompilationLevel, VllmConfig
13
from vllm.distributed.parallel_state import graph_capture
14
from vllm.forward_context import set_forward_context
@@ -16,14 +18,16 @@ from vllm.model_executor.model_loader import get_model
18
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
19
from vllm.sampling_params import SamplingType
20
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19
- LayerBlockType, bind_kv_cache, cdiv,
20
- is_pin_memory_available)
21
+ LayerBlockType, cdiv, is_pin_memory_available)
22
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
23
FlashAttentionMetadata)
24
from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
25
from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
26
+from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
27
+ KVCacheSpec)
28
from vllm.v1.outputs import ModelRunnerOutput
29
from vllm.v1.sample.metadata import SamplingMetadata
30
+from vllm.v1.utils import bind_kv_cache
31
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
32
33
if TYPE_CHECKING:
@@ -856,15 +860,71 @@ class GPUModelRunner:
860
logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
861
elapsed_time, cuda_graph_size / (1 << 30))
862
859
- def initialize_kv_cache(self, num_blocks: int) -> None:
860
- assert len(self.kv_caches) == 0
861
- kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
862
- num_blocks, self.block_size, self.num_kv_heads, self.head_size)
863
- for _ in range(self.num_attn_layers):
864
- self.kv_caches.append(
865
- torch.zeros(kv_cache_shape,
866
- dtype=self.kv_cache_dtype,
867
- device=self.device))
863
+ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
864
+ """
865
+ Initialize KV cache based on `kv_cache_config`.
866
+ Args:
867
+ kv_cache_config: Configuration for the KV cache, including the KV
868
+ cache size of each layer
869
+ """
870
+ if len(kv_cache_config.groups) > 1:
871
+ raise NotImplementedError(
872
+ "Hybrid models with more than one KV cache type are not "
873
+ "supported yet.")
874
+
875
+ kv_caches: Dict[str, torch.Tensor] = {}
876
+
877
+ for layer_name, layer_spec in kv_cache_config.kv_cache_spec.items():
878
+ tensor_config = kv_cache_config.tensors[layer_name]
879
+ assert tensor_config.size % layer_spec.page_size_bytes == 0
880
+ num_blocks = tensor_config.size // layer_spec.page_size_bytes
881
+ if isinstance(layer_spec, FullAttentionSpec):
882
+ kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
883
+ num_blocks, layer_spec.block_size, layer_spec.num_kv_heads,
884
+ layer_spec.head_size)
885
+ dtype = layer_spec.dtype
886
+ kv_caches[layer_name] = torch.zeros(kv_cache_shape,
887
+ dtype=dtype,
888
+ device=self.device)
889
+ else:
890
+ raise NotImplementedError
891
+
892
bind_kv_cache(
893
+ kv_caches,
894
self.vllm_config.compilation_config.static_forward_context,
870
- [self.kv_caches])
895
+ self.kv_caches)
896
+
897
+ def get_kv_cache_spec(self) -> KVCacheSpec:
898
+ """
899
+ Generates the KVCacheSpec by parsing the kv cache format from each
900
+ Attention module in the static forward context.
901
+ Returns:
902
+ KVCacheSpec: A dictionary mapping layer names to their KV cache
903
+ format. Layers that do not need KV cache are not included.
904
+ """
905
+
906
+ forward_ctx = self.vllm_config.compilation_config.static_forward_context
907
+ block_size = self.vllm_config.cache_config.block_size
908
+ kv_cache_spec: KVCacheSpec = {}
909
+ for layer_name, attn_module in forward_ctx.items():
910
+ # TODO: Support other attention modules, e.g., sliding window,
911
+ # cross-attention, MLA.
912
+ assert isinstance(attn_module, Attention)
913
+ if attn_module.attn_type == AttentionType.DECODER:
914
+ kv_cache_spec[layer_name] = FullAttentionSpec(
915
+ block_size=block_size,
916
+ num_kv_heads=attn_module.num_kv_heads,
917
+ head_size=attn_module.head_size,
918
+ dtype=attn_module.dtype,
919
+ )
920
+ elif attn_module.attn_type in (AttentionType.ENCODER,
921
+ AttentionType.ENCODER_ONLY):
922
+ # encoder-only attention does not need KV cache.
923
+ continue
924
+ elif attn_module.attn_type == AttentionType.ENCODER_DECODER:
925
+ raise NotImplementedError
926
+ else:
927
+ raise ValueError(
928
+ f"Unknown attention type: {attn_module.attn_type}")
929
+
930
+ return kv_cache_spec
vllm/v1/worker/gpu_worker.py
+14
-34
@@ -1,7 +1,7 @@
1
"""A GPU worker class."""
2
import gc
3
import os
4
-from typing import TYPE_CHECKING, Optional, Tuple
4
+from typing import TYPE_CHECKING, Optional
5
6
import torch
7
import torch.distributed
@@ -16,6 +16,7 @@ from vllm.model_executor import set_random_seed
16
from vllm.platforms import current_platform
17
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType, get_dtype_size
18
from vllm.v1.core.scheduler import SchedulerOutput
19
+from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
20
from vllm.v1.outputs import ModelRunnerOutput
21
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
22
@@ -112,20 +113,18 @@ class Worker:
113
self.model_runner.load_model()
114
115
@torch.inference_mode()
115
- def determine_num_available_blocks(self) -> Tuple[int, int]:
116
- """Profiles the peak memory usage of the model to determine how many
117
- KV blocks may be allocated without OOMs.
116
+ def determine_available_memory(self) -> int:
117
+ """Profiles the peak memory usage of the model to determine how much
118
+ memory can be used for KV cache without OOMs.
119
120
The engine will first conduct a profiling of the existing memory usage.
120
- Then, it calculate the maximum possible number of GPU and CPU blocks
121
- that can be allocated with the remaining free memory.
121
+ Then, it calculate the free memory that can be used for KV cache in
122
+ bytes.
123
124
.. tip::
125
You may limit the usage of GPU memory
126
by adjusting the `gpu_memory_utilization` parameter.
127
"""
127
- # Profile the memory usage of the model and get the maximum number of
128
- # cache blocks that can be allocated with the remaining free memory.
128
torch.cuda.empty_cache()
129
torch.cuda.reset_peak_memory_stats()
130
@@ -161,33 +160,14 @@ class Worker:
160
total_gpu_memory * self.cache_config.gpu_memory_utilization -
161
peak_memory)
162
164
- # Calculate the number of blocks that can be allocated with the
165
- # profiled peak memory.
166
- cache_block_size = _get_cache_block_size(self.cache_config,
167
- self.model_config,
168
- self.parallel_config)
169
- num_gpu_blocks = int(available_kv_cache_memory // cache_block_size)
170
- num_gpu_blocks = max(num_gpu_blocks, 0)
171
- return num_gpu_blocks, 0
172
-
173
- def initialize_cache(self, num_gpu_blocks: int) -> None:
174
- """Allocate GPU and CPU KV cache with the specified number of blocks."""
175
- if num_gpu_blocks <= 0:
176
- raise ValueError("No available memory for the cache blocks. "
177
- "Try increasing `gpu_memory_utilization` when "
178
- "initializing the engine.")
179
-
180
- max_seq_len = self.cache_config.block_size * num_gpu_blocks
181
- max_model_len = self.model_config.max_model_len
182
- if max_model_len > max_seq_len:
183
- raise ValueError(
184
- f"The model's max seq len ({max_model_len}) "
185
- "is larger than the maximum number of tokens that can be "
186
- f"stored in KV cache ({max_seq_len}). Try increasing "
187
- "`gpu_memory_utilization` or decreasing `max_model_len` when "
188
- "initializing the engine.")
163
+ return int(available_kv_cache_memory)
164
+
165
+ def get_kv_cache_spec(self) -> KVCacheSpec:
166
+ return self.model_runner.get_kv_cache_spec()
167
190
- self.model_runner.initialize_kv_cache(num_gpu_blocks)
168
+ def initialize_cache(self, kv_cache_config: KVCacheConfig) -> None:
169
+ """Allocate GPU KV cache with the specified kv_cache_config."""
170
+ self.model_runner.initialize_kv_cache(kv_cache_config)
171
172
def compile_or_warm_up_model(self) -> None:
173
if not self.model_config.enforce_eager: