@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69d765f5a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Move more control of kv cache initialization from model_executor to EngineCore (#11960)

Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Chen Zhang committed Jan 17, 2025 at 15:39 UTC 69d765f5a5bbbe1ea9843be19b9480660fc5bc8b
12 files changed +515 -104
tests/v1/test_utils.py new
+62
@@ -0,0 +1,62 @@
1 +from typing import List
2 +
3 +import torch
4 +
5 +from vllm.v1.utils import bind_kv_cache
6 +
7 +
8 +def test_bind_kv_cache():
9 + from vllm.attention import Attention
10 +
11 + ctx = {
12 + 'layers.0.self_attn': Attention(32, 128, 0.1),
13 + 'layers.1.self_attn': Attention(32, 128, 0.1),
14 + 'layers.2.self_attn': Attention(32, 128, 0.1),
15 + 'layers.3.self_attn': Attention(32, 128, 0.1),
16 + }
17 + kv_cache = {
18 + 'layers.0.self_attn': torch.zeros((1, )),
19 + 'layers.1.self_attn': torch.zeros((1, )),
20 + 'layers.2.self_attn': torch.zeros((1, )),
21 + 'layers.3.self_attn': torch.zeros((1, )),
22 + }
23 + runner_kv_caches: List[torch.Tensor] = []
24 + bind_kv_cache(kv_cache, ctx, runner_kv_caches)
25 + assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[
26 + 'layers.0.self_attn']
27 + assert ctx['layers.1.self_attn'].kv_cache[0] is kv_cache[
28 + 'layers.1.self_attn']
29 + assert ctx['layers.2.self_attn'].kv_cache[0] is kv_cache[
30 + 'layers.2.self_attn']
31 + assert ctx['layers.3.self_attn'].kv_cache[0] is kv_cache[
32 + 'layers.3.self_attn']
33 +
34 + assert runner_kv_caches[0] is kv_cache['layers.0.self_attn']
35 + assert runner_kv_caches[1] is kv_cache['layers.1.self_attn']
36 + assert runner_kv_caches[2] is kv_cache['layers.2.self_attn']
37 + assert runner_kv_caches[3] is kv_cache['layers.3.self_attn']
38 +
39 +
40 +def test_bind_kv_cache_non_attention():
41 + from vllm.attention import Attention
42 +
43 + # example from Jamba PP=2
44 + ctx = {
45 + 'model.layers.20.attn': Attention(32, 128, 0.1),
46 + 'model.layers.28.attn': Attention(32, 128, 0.1),
47 + }
48 + kv_cache = {
49 + 'model.layers.20.attn': torch.zeros((1, )),
50 + 'model.layers.28.attn': torch.zeros((1, )),
51 + }
52 +
53 + runner_kv_caches: List[torch.Tensor] = []
54 + bind_kv_cache(kv_cache, ctx, runner_kv_caches)
55 +
56 + assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[
57 + 'model.layers.20.attn']
58 + assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[
59 + 'model.layers.28.attn']
60 +
61 + assert runner_kv_caches[0] is kv_cache['model.layers.20.attn']
62 + assert runner_kv_caches[1] is kv_cache['model.layers.28.attn']
vllm/attention/layer.py
+2
@@ -101,7 +101,9 @@ class Attention(nn.Module):
101 self.num_heads = num_heads
102 self.head_size = head_size
103 self.num_kv_heads = num_kv_heads
104 + self.sliding_window = sliding_window
105 self.backend = backend_name_to_enum(attn_backend.get_name())
106 + self.dtype = dtype
107
108 # For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
109 # torch.compile works by registering the attention as one giant
vllm/v1/core/kv_cache_utils.py
+124
@@ -3,7 +3,10 @@ from collections.abc import Sequence
3 from dataclasses import dataclass
4 from typing import Any, List, NamedTuple, Optional, Tuple
5
6 +from vllm.config import VllmConfig
7 from vllm.logger import init_logger
8 +from vllm.v1.kv_cache_interface import (KVCacheConfig, KVCacheSpec,
9 + KVCacheTensor)
10 from vllm.v1.request import Request
11
12 logger = init_logger(__name__)
@@ -305,3 +308,124 @@ def hash_request_tokens(block_size: int,
308 ret.append(block_hash)
309 parent_block_hash_value = block_hash.hash_value
310 return ret
311 +
312 +
313 +def check_enough_kv_cache_memory(vllm_config: VllmConfig,
314 + kv_cache_spec: KVCacheSpec,
315 + available_memory: int):
316 + """
317 + Checks whether `available_memory` is enough for the KV cache to hold at
318 + least one request with the model's max_model_len.
319 +
320 + Args:
321 + vllm_config: The global VllmConfig
322 + kv_cache_spec: The kv cache spec of the model
323 + available_memory: Memory available for KV cache in bytes.
324 +
325 + Raises:
326 + ValueError: If there is not enough memory available for the KV cache.
327 + """
328 +
329 + if available_memory <= 0:
330 + raise ValueError("No available memory for the cache blocks. "
331 + "Try increasing `gpu_memory_utilization` when "
332 + "initializing the engine.")
333 +
334 + max_model_len = vllm_config.model_config.max_model_len
335 + needed_memory = 0
336 + for layer_spec in kv_cache_spec.values():
337 + needed_memory += layer_spec.bytes_for_tokens(max_model_len)
338 +
339 + if needed_memory > available_memory:
340 + raise ValueError(
341 + f"To serve at least one request with the models's max seq len "
342 + f"({max_model_len}), ({needed_memory/1024/1024/1024:.2f} GB KV "
343 + f"cache is needed, which is larger than the available KV cache "
344 + f"memory ({available_memory/1024/1024/1024:.2f} GB). Try "
345 + f"increasing `gpu_memory_utilization` or decreasing "
346 + f"`max_model_len` when initializing the engine.")
347 +
348 +
349 +def is_kv_cache_type_uniform(kv_cache_spec: KVCacheSpec) -> bool:
350 + """
351 + Whether all layers in the given KVCacheSpec have the same type of KV cache.
352 +
353 + Args:
354 + kv_cache_spec: The KVCacheSpec of the model
355 +
356 + Returns:
357 + True if all layers have the same type, False otherwise.
358 + """
359 +
360 + layer_keys = set(layer.type_id for layer in kv_cache_spec.values())
361 + return len(layer_keys) == 1
362 +
363 +
364 +def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
365 + kv_cache_spec: KVCacheSpec,
366 + available_memory: int) -> KVCacheConfig:
367 + """
368 + Generates the KV cache configuration for a model with one type of KV cache.
369 + Divide the available memory equally among all layers.
370 +
371 + Args:
372 + vllm_config: The global VllmConfig
373 + kv_cache_spec: The kv cache spec of the model
374 + available_memory: Memory available for KV cache in bytes.
375 +
376 + Returns:
377 + The generated KVCacheConfig
378 + """
379 +
380 + page_sizes = {layer.page_size_bytes for layer in kv_cache_spec.values()}
381 + assert len(page_sizes) == 1
382 + page_size = page_sizes.pop()
383 +
384 + num_blocks = int(available_memory // page_size // len(kv_cache_spec))
385 + num_blocks = max(num_blocks, 0)
386 +
387 + if vllm_config.cache_config.num_gpu_blocks_override is not None:
388 + num_gpu_blocks_override = \
389 + vllm_config.cache_config.num_gpu_blocks_override
390 + logger.info(
391 + "Overriding num_gpu_blocks=%d with "
392 + "num_gpu_blocks_override=%d", num_blocks, num_gpu_blocks_override)
393 + num_blocks = num_gpu_blocks_override
394 +
395 + logger.info("# GPU blocks: %d", num_blocks)
396 +
397 + per_layer_size = page_size * num_blocks
398 +
399 + kv_cache_config = KVCacheConfig(
400 + num_blocks=num_blocks,
401 + tensors={
402 + layer_name: KVCacheTensor(size=per_layer_size)
403 + for layer_name in kv_cache_spec
404 + },
405 + groups=[[layer_name for layer_name in kv_cache_spec]],
406 + kv_cache_spec=kv_cache_spec)
407 + return kv_cache_config
408 +
409 +
410 +def get_kv_cache_config(vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec,
411 + available_memory: int) -> KVCacheConfig:
412 + """
413 + Generates the KV cache configuration for a model
414 + TODO: support hybrid models with more than one type of KV cache.
415 +
416 + Args:
417 + vllm_config: The global VllmConfig
418 + kv_cache_spec: The kv cache spec of the model
419 + available_memory: Memory available for KV cache in bytes.
420 +
421 + Returns:
422 + The generated KVCacheConfig
423 + """
424 + check_enough_kv_cache_memory(vllm_config, kv_cache_spec, available_memory)
425 + if is_kv_cache_type_uniform(kv_cache_spec):
426 + # KV cache of all layers are the same, which is true for most models.
427 + # Allocate the same amount of memory for each layer.
428 + return _get_kv_cache_config_uniform_type(vllm_config, kv_cache_spec,
429 + available_memory)
430 + else:
431 + raise NotImplementedError
vllm/v1/engine/core.py
+18 -13
@@ -11,11 +11,12 @@ import zmq
11 import zmq.asyncio
12 from msgspec import msgpack
13
14 -from vllm.config import CacheConfig, VllmConfig
14 +from vllm.config import VllmConfig
15 from vllm.logger import init_logger
16 from vllm.transformers_utils.config import (
17 maybe_register_config_serialize_by_value)
18 from vllm.utils import get_exception_traceback, zmq_socket_ctx
19 +from vllm.v1.core.kv_cache_utils import get_kv_cache_config
20 from vllm.v1.core.scheduler import Scheduler
21 from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
22 EngineCoreRequest, EngineCoreRequestType,
@@ -49,7 +50,7 @@ class EngineCore:
50
51 # Setup KV Caches and update CacheConfig after profiling.
52 num_gpu_blocks, num_cpu_blocks = self._initialize_kv_caches(
52 - vllm_config.cache_config)
53 + vllm_config)
54 vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
55 vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks
56
@@ -65,21 +66,25 @@ class EngineCore:
66 vllm_config.model_config)
67
68 def _initialize_kv_caches(self,
68 - cache_config: CacheConfig) -> Tuple[int, int]:
69 + vllm_config: VllmConfig) -> Tuple[int, int]:
70 start = time.time()
70 - num_gpu_blocks, _ = self.model_executor.determine_num_available_blocks(
71 - )
71
73 - if cache_config.num_gpu_blocks_override is not None:
74 - num_gpu_blocks_override = cache_config.num_gpu_blocks_override
75 - logger.info(
76 - "Overriding num_gpu_blocks=%d with "
77 - "num_gpu_blocks_override=%d", num_gpu_blocks,
78 - num_gpu_blocks_override)
79 - num_gpu_blocks = num_gpu_blocks_override
72 + # Get all kv cache needed by the model
73 + kv_cache_spec = self.model_executor.get_kv_cache_spec()
74 +
75 + # Profiles the peak memory usage of the model to determine how much
76 + # memory can be allocated for kv cache.
77 + availble_gpu_memory = self.model_executor.determine_available_memory()
78
79 + # Get the kv cache tensor size
80 + kv_cache_config = get_kv_cache_config(vllm_config, kv_cache_spec,
81 + availble_gpu_memory)
82 + num_gpu_blocks = kv_cache_config.num_blocks
83 num_cpu_blocks = 0
82 - self.model_executor.initialize(num_gpu_blocks)
84 +
85 + # Initialize kv cache and warmup the execution
86 + self.model_executor.initialize(kv_cache_config)
87 +
88 elapsed = time.time() - start
89 logger.info(("init engine (profile, create kv cache, "
90 "warmup model) took %.2f seconds"), elapsed)
vllm/v1/executor/abstract.py
+8 -3
@@ -1,7 +1,8 @@
1 from abc import ABC, abstractmethod
2 -from typing import Tuple, Type
2 +from typing import Type
3
4 from vllm.config import VllmConfig
5 +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
6 from vllm.v1.outputs import ModelRunnerOutput
7
8
@@ -31,11 +32,15 @@ class Executor(ABC):
32 raise NotImplementedError
33
34 @abstractmethod
34 - def initialize(self, num_gpu_blocks: int) -> None:
35 + def initialize(self, kv_cache_config: KVCacheConfig) -> None:
36 raise NotImplementedError
37
38 @abstractmethod
38 - def determine_num_available_blocks(self) -> Tuple[int, int]:
39 + def determine_available_memory(self) -> int: # in bytes
40 + raise NotImplementedError
41 +
42 + @abstractmethod
43 + def get_kv_cache_spec(self) -> KVCacheSpec:
44 raise NotImplementedError
45
46 @abstractmethod
vllm/v1/executor/multiproc_executor.py
+15 -10
@@ -23,6 +23,7 @@ from vllm.logger import init_logger
23 from vllm.utils import (get_distributed_init_method, get_mp_context,
24 get_open_port, get_open_zmq_ipc_path, zmq_socket_ctx)
25 from vllm.v1.executor.abstract import Executor
26 +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
27 from vllm.v1.outputs import ModelRunnerOutput
28 from vllm.worker.worker_base import WorkerWrapperBase
29
@@ -90,29 +91,33 @@ class MultiprocExecutor(Executor):
91 for w in self.workers:
92 w.worker_response_mq.wait_until_ready()
93
93 - def initialize(self, num_gpu_blocks: int) -> None:
94 + def initialize(self, kv_cache_config: KVCacheConfig) -> None:
95 """
96 Initialize the KV caches and begin the model execution loop of the
97 underlying workers.
98 """
98 - logger.info("# GPU blocks: %d", num_gpu_blocks)
99 - self.collective_rpc("initialize_cache", args=(num_gpu_blocks, ))
99 + self.collective_rpc("initialize_cache", args=(kv_cache_config, ))
100 self.collective_rpc("compile_or_warm_up_model")
101
102 - def determine_num_available_blocks(self) -> Tuple[int, int]:
102 + def determine_available_memory(self) -> int:
103 """
104 - Determine the number of available KV blocks by invoking the
104 + Determine the available memory (in bytes) for KV cache by invoking the
105 underlying worker.
106 """
107 - num_blocks = self.collective_rpc("determine_num_available_blocks")
107 + memory_sizes = self.collective_rpc("determine_available_memory")
108
109 # Since we use a shared centralized controller, we take the minimum
110 - # number of blocks across all workers to make sure all the memory
110 + # memory size across all workers to make sure all the memory
111 # operators can be applied to all workers.
112 - num_gpu_blocks = min(b[0] for b in num_blocks)
113 - num_cpu_blocks = min(b[1] for b in num_blocks)
112 + return min(memory_sizes)
113
115 - return num_gpu_blocks, num_cpu_blocks
114 + def get_kv_cache_spec(self) -> KVCacheSpec:
115 + """
116 + Get all kv cache needed by the model by invoking the underlying worker.
117 + """
118 + kv_cache_specs = self.collective_rpc("get_kv_cache_spec")
119 + assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
120 + return kv_cache_specs[0]
121
122 def collective_rpc(self,
123 method: str,
vllm/v1/executor/ray_executor.py
+21 -19
@@ -10,6 +10,7 @@ from vllm.utils import get_distributed_init_method, get_ip, get_open_port
10 from vllm.v1.executor.abstract import Executor
11 from vllm.v1.executor.ray_utils import (RayWorkerWrapper,
12 initialize_ray_cluster, ray)
13 +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
14 from vllm.v1.outputs import ModelRunnerOutput
15
16 if ray is not None:
@@ -211,39 +212,40 @@ class RayExecutor(Executor):
212 distributed_init_method=distributed_init_method,
213 )
214
214 - def determine_num_available_blocks(self) -> Tuple[int, int]:
215 + def determine_available_memory(self) -> int:
216 """
216 - Determine the number of available KV blocks.
217 + Determine the available GPU memory in bytes.
218
218 - This invokes `determine_num_available_blocks` on each worker and takes
219 + This invokes `determine_available_memory` on each worker and takes
220 the min of the results, guaranteeing that the selected cache sizes are
221 compatible with all workers.
221 -
222 - Returns:
223 - - tuple[num_gpu_blocks, num_cpu_blocks]
222 """
225 - # Get the maximum number of blocks that can be allocated on GPU and CPU.
226 - num_blocks = self._run_workers("determine_num_available_blocks")
223 +
224 + memory_sizes = self._run_workers("determine_available_memory")
225
226 # Since we use a shared centralized controller, we take the minimum
229 - # number of blocks across all workers to make sure all the memory
227 + # memory size across all workers to make sure all the memory
228 # operators can be applied to all workers.
231 - num_gpu_blocks = min(b[0] for b in num_blocks)
232 - num_cpu_blocks = min(b[1] for b in num_blocks)
229 + return min(memory_sizes)
230
234 - return num_gpu_blocks, num_cpu_blocks
235 -
236 - def initialize(self, num_gpu_blocks: int) -> None:
231 + def initialize(self, kv_cache_config: KVCacheConfig) -> None:
232 """
233 Initialize the KV cache in all workers.
234 """
240 - # NOTE: This is logged in the executor because there can be >1 worker
241 - # with other executors. We could log in the engine level, but work
242 - # remains to abstract away the device for non-GPU configurations.
243 - logger.info("# GPU blocks: %d", num_gpu_blocks)
244 - self._run_workers("initialize_cache", num_gpu_blocks)
235 + self._run_workers("initialize_cache", kv_cache_config)
236 self._run_workers("compile_or_warm_up_model")
237
238 + def get_kv_cache_spec(self) -> KVCacheSpec:
239 + """
240 + Get all kv cache needed by the model
241 +
242 + This invokes `get_kv_cache_spec` on each worker and asserts that
243 + they are identical. The KVCacheSpec is then returned.
244 + """
245 + kv_cache_specs = self._run_workers("get_kv_cache_spec")
246 + assert all(s == kv_cache_specs[0] for s in kv_cache_specs)
247 + return kv_cache_specs[0]
248 +
249 def _run_workers(
250 self,
251 method: str,
vllm/v1/executor/uniproc_executor.py
+14 -11
@@ -1,10 +1,11 @@
1 import os
2 -from typing import Optional, Tuple
2 +from typing import Optional
3
4 from vllm.config import VllmConfig
5 from vllm.logger import init_logger
6 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7 from vllm.v1.executor.abstract import Executor
8 +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
9 from vllm.v1.outputs import ModelRunnerOutput
10 from vllm.v1.worker.gpu_worker import Worker
11
@@ -49,20 +50,22 @@ class UniprocExecutor(Executor):
50 distributed_init_method=distributed_init_method,
51 )
52
52 - def determine_num_available_blocks(self) -> Tuple[int, int]:
53 - """Determine the number of available KV blocks by invoking the
54 - underlying worker.
53 + def determine_available_memory(self) -> int:
54 + """Determine the available memory (in bytes) for KV cache by invoking
55 + the underlying worker.
56 """
56 - return self.worker.determine_num_available_blocks()
57 + return self.worker.determine_available_memory()
58
58 - def initialize(self, num_gpu_blocks: int) -> None:
59 + def get_kv_cache_spec(self) -> KVCacheSpec:
60 + """Get all kv cache needed by the model by invoking the underlying
61 + worker.
62 + """
63 + return self.worker.get_kv_cache_spec()
64 +
65 + def initialize(self, kv_cache_config: KVCacheConfig) -> None:
66 """Initialize the KV cache by invoking the underlying worker.
67 """
61 - # NOTE: This is logged in the executor because there can be >1 worker
62 - # with other executors. We could log in the engine level, but work
63 - # remains to abstract away the device for non-GPU configurations.
64 - logger.info("# GPU blocks: %d", num_gpu_blocks)
65 - self.worker.initialize_cache(num_gpu_blocks)
68 + self.worker.initialize_cache(kv_cache_config)
69 self.worker.compile_or_warm_up_model()
70
71 def execute_model(
vllm/v1/kv_cache_interface.py new
+111
@@ -0,0 +1,111 @@
1 +from dataclasses import dataclass
2 +from typing import Dict, List
3 +
4 +import torch
5 +
6 +from vllm.logger import init_logger
7 +from vllm.utils import cdiv, get_dtype_size
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +@dataclass
13 +class KVCacheSpecBase:
14 + """
15 + A base class for specifying the KV cache format of one layer.
16 + """
17 +
18 + # number of tokens in a block
19 + block_size: int
20 +
21 + @property
22 + def type_id(self) -> str:
23 + """
24 + The type identifier of this KV cache.
25 + Return different strings for layers with different KV cache type (e.g.,
26 + different number of tokens like full attention vs sliding window
27 + attention, different KV cache size per token like layers with different
28 + number of heads)
29 +
30 + Returns:
31 + The type identifier of this KV cache.
32 + """
33 + raise NotImplementedError
34 +
35 + @property
36 + def page_size_bytes(self) -> int:
37 + """
38 + The size of a page with `block_size` tokens in bytes.
39 +
40 + Returns:
41 + The page size
42 + """
43 + raise NotImplementedError
44 +
45 + def bytes_for_tokens(self, num_tokens: int) -> int:
46 + """
47 + The KV cache size for `num_tokens` tokens in bytes. Returns the real
48 + memory size after padding `num_tokens` to full blocks.
49 +
50 + Returns:
51 + The KV cache size
52 + """
53 + raise NotImplementedError
54 +
55 +
56 +@dataclass
57 +class FullAttentionSpec(KVCacheSpecBase):
58 + num_kv_heads: int
59 + head_size: int
60 + dtype: torch.dtype
61 +
62 + @property
63 + def type_id(self) -> str:
64 + return f"full_attention_{self.block_size}_{self.page_size_bytes}"
65 +
66 + @property
67 + def page_size_bytes(self) -> int:
68 + return 2 * self.block_size * self.num_kv_heads * self.head_size \
69 + * get_dtype_size(self.dtype)
70 +
71 + def bytes_for_tokens(self, num_tokens: int) -> int:
72 + return cdiv(num_tokens, self.block_size) * self.page_size_bytes
73 +
74 +
75 +KVCacheSpec = Dict[str, KVCacheSpecBase]
76 +
77 +
78 +@dataclass
79 +class KVCacheTensor:
80 + """
81 + A dataclass for specifying how the workers should initialize the KV cache
82 + for a layer. Only contains the size of KV cache for that layer for now. Will
83 + be extended to support multiple layers sharing the same memory pool.
84 + """
85 + size: int # The size of KV cache Tensor in bytes
86 +
87 +
88 +@dataclass
89 +class KVCacheConfig:
90 + """
91 + The KV cache configuration of a model.
92 + """
93 + """The number of KV cache blocks"""
94 + num_blocks: int
95 + """layer_name -> how to initialize KV cache for that layer"""
96 + tensors: Dict[str, KVCacheTensor]
97 + """
98 + A list of kv-cache groups. Each group includes a set of layers with
99 + the same kv-cache spec, and the total page_size of layers inside a group
100 + is same across all groups (as the KVCacheManager only supports allocating
101 + pages of the same size). For example:
102 + 1. A model only uses full attention: one group with all layers in the model.
103 + 2. (not implemented yet) A model with the same number of full attention
104 + layers and sliding window attention layers: two groups, one for full
105 + attention layers and one for sliding window attention layers.
106 + 3. (not implemented yet) A model with 2 full attention layers and 4 sliding
107 + window attention layers: three groups, (full * 2), (sw * 2), (sw * 2).
108 + """
109 + groups: List[List[str]]
110 + """the KVCacheSpec of the model"""
111 + kv_cache_spec: KVCacheSpec
vllm/v1/utils.py
+54 -2
@@ -1,13 +1,20 @@
1 import multiprocessing
2 import os
3 import weakref
4 +from collections import defaultdict
5 from collections.abc import Sequence
5 -from typing import (Any, Callable, Dict, Generic, List, Optional, TypeVar,
6 - Union, overload)
6 +from typing import (TYPE_CHECKING, Any, Callable, Dict, Generic, List,
7 + Optional, TypeVar, Union, overload)
8 +
9 +import torch
10
11 from vllm.logger import init_logger
12 +from vllm.model_executor.models.utils import extract_layer_index
13 from vllm.utils import get_mp_context, kill_process_tree
14
15 +if TYPE_CHECKING:
16 + from vllm.attention.layer import Attention
17 +
18 logger = init_logger(__name__)
19
20 T = TypeVar("T")
@@ -134,3 +141,48 @@ def shutdown(proc: multiprocessing.Process, input_path: str, output_path: str):
141 socket_file = ipc_socket.replace("ipc://", "")
142 if os and os.path.exists(socket_file):
143 os.remove(socket_file)
144 +
145 +
146 +def bind_kv_cache(
147 + kv_caches: Dict[str, torch.Tensor],
148 + forward_context: Dict[str, "Attention"],
149 + runner_kv_caches: List[torch.Tensor],
150 +) -> None:
151 + """
152 + Bind the allocated KV cache to both ModelRunner and forward context so
153 + that the KV cache can be used in the forward pass.
154 +
155 + This function:
156 + 1) Fills the ModelRunner's kv cache list (`runner_kv_caches`) with
157 + kv_caches.
158 + 2) Associates each attention layer in the `forward_context` with its
159 + corresponding KV cache in kv_caches.
160 +
161 + Args:
162 + kv_caches: The allocated kv_caches with layer names as keys.
163 + forward_context: The global forward context containing all Attention
164 + layers with layer names as keys.
165 + runner_kv_caches: The kv_cache declared by ModelRunner.
166 + """
167 + # Bind kv_caches to ModelRunner
168 + assert len(runner_kv_caches) == 0
169 +
170 + # Convert kv_caches dict to a list of tensors in the order of layer_index.
171 + index2name = defaultdict(list)
172 + for layer_name in kv_caches:
173 + index2name[extract_layer_index(layer_name)].append(layer_name)
174 +
175 + for layer_index in sorted(index2name.keys()):
176 + layer_names = index2name[layer_index]
177 + if len(layer_names) > 1:
178 + # One typical case is encoder-decoder model, e.g., bart.
179 + # The cross attention and self attention in the same decoder layer
180 + # has different layer_name but the same layer_index.
181 + raise NotImplementedError
182 + layer_name = layer_names[0]
183 + runner_kv_caches.append(kv_caches[layer_name])
184 +
185 + # Bind kv_caches to forward context
186 + for layer_name, kv_cache in kv_caches.items():
187 + # NOTE: Use list because of v0 PP virtual engine.
188 + forward_context[layer_name].kv_cache = [kv_cache]
vllm/v1/worker/gpu_model_runner.py
+72 -12
@@ -7,6 +7,8 @@ import torch
7 import torch.distributed
8 import torch.nn as nn
9
10 +from vllm.attention.backends.abstract import AttentionType
11 +from vllm.attention.layer import Attention
12 from vllm.config import CompilationLevel, VllmConfig
13 from vllm.distributed.parallel_state import graph_capture
14 from vllm.forward_context import set_forward_context
@@ -16,14 +18,16 @@ from vllm.model_executor.model_loader import get_model
18 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
19 from vllm.sampling_params import SamplingType
20 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19 - LayerBlockType, bind_kv_cache, cdiv,
20 - is_pin_memory_available)
21 + LayerBlockType, cdiv, is_pin_memory_available)
22 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
23 FlashAttentionMetadata)
24 from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
25 from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
26 +from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
27 + KVCacheSpec)
28 from vllm.v1.outputs import ModelRunnerOutput
29 from vllm.v1.sample.metadata import SamplingMetadata
30 +from vllm.v1.utils import bind_kv_cache
31 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
32
33 if TYPE_CHECKING:
@@ -856,15 +860,71 @@ class GPUModelRunner:
860 logger.info("Graph capturing finished in %.0f secs, took %.2f GiB",
861 elapsed_time, cuda_graph_size / (1 << 30))
862
859 - def initialize_kv_cache(self, num_blocks: int) -> None:
860 - assert len(self.kv_caches) == 0
861 - kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
862 - num_blocks, self.block_size, self.num_kv_heads, self.head_size)
863 - for _ in range(self.num_attn_layers):
864 - self.kv_caches.append(
865 - torch.zeros(kv_cache_shape,
866 - dtype=self.kv_cache_dtype,
867 - device=self.device))
863 + def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
864 + """
865 + Initialize KV cache based on `kv_cache_config`.
866 + Args:
867 + kv_cache_config: Configuration for the KV cache, including the KV
868 + cache size of each layer
869 + """
870 + if len(kv_cache_config.groups) > 1:
871 + raise NotImplementedError(
872 + "Hybrid models with more than one KV cache type are not "
873 + "supported yet.")
874 +
875 + kv_caches: Dict[str, torch.Tensor] = {}
876 +
877 + for layer_name, layer_spec in kv_cache_config.kv_cache_spec.items():
878 + tensor_config = kv_cache_config.tensors[layer_name]
879 + assert tensor_config.size % layer_spec.page_size_bytes == 0
880 + num_blocks = tensor_config.size // layer_spec.page_size_bytes
881 + if isinstance(layer_spec, FullAttentionSpec):
882 + kv_cache_shape = FlashAttentionBackend.get_kv_cache_shape(
883 + num_blocks, layer_spec.block_size, layer_spec.num_kv_heads,
884 + layer_spec.head_size)
885 + dtype = layer_spec.dtype
886 + kv_caches[layer_name] = torch.zeros(kv_cache_shape,
887 + dtype=dtype,
888 + device=self.device)
889 + else:
890 + raise NotImplementedError
891 +
892 bind_kv_cache(
893 + kv_caches,
894 self.vllm_config.compilation_config.static_forward_context,
870 - [self.kv_caches])
895 + self.kv_caches)
896 +
897 + def get_kv_cache_spec(self) -> KVCacheSpec:
898 + """
899 + Generates the KVCacheSpec by parsing the kv cache format from each
900 + Attention module in the static forward context.
901 + Returns:
902 + KVCacheSpec: A dictionary mapping layer names to their KV cache
903 + format. Layers that do not need KV cache are not included.
904 + """
905 +
906 + forward_ctx = self.vllm_config.compilation_config.static_forward_context
907 + block_size = self.vllm_config.cache_config.block_size
908 + kv_cache_spec: KVCacheSpec = {}
909 + for layer_name, attn_module in forward_ctx.items():
910 + # TODO: Support other attention modules, e.g., sliding window,
911 + # cross-attention, MLA.
912 + assert isinstance(attn_module, Attention)
913 + if attn_module.attn_type == AttentionType.DECODER:
914 + kv_cache_spec[layer_name] = FullAttentionSpec(
915 + block_size=block_size,
916 + num_kv_heads=attn_module.num_kv_heads,
917 + head_size=attn_module.head_size,
918 + dtype=attn_module.dtype,
919 + )
920 + elif attn_module.attn_type in (AttentionType.ENCODER,
921 + AttentionType.ENCODER_ONLY):
922 + # encoder-only attention does not need KV cache.
923 + continue
924 + elif attn_module.attn_type == AttentionType.ENCODER_DECODER:
925 + raise NotImplementedError
926 + else:
927 + raise ValueError(
928 + f"Unknown attention type: {attn_module.attn_type}")
929 +
930 + return kv_cache_spec
vllm/v1/worker/gpu_worker.py
+14 -34
@@ -1,7 +1,7 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -from typing import TYPE_CHECKING, Optional, Tuple
4 +from typing import TYPE_CHECKING, Optional
5
6 import torch
7 import torch.distributed
@@ -16,6 +16,7 @@ from vllm.model_executor import set_random_seed
16 from vllm.platforms import current_platform
17 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType, get_dtype_size
18 from vllm.v1.core.scheduler import SchedulerOutput
19 +from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
20 from vllm.v1.outputs import ModelRunnerOutput
21 from vllm.v1.worker.gpu_model_runner import GPUModelRunner
22
@@ -112,20 +113,18 @@ class Worker:
113 self.model_runner.load_model()
114
115 @torch.inference_mode()
115 - def determine_num_available_blocks(self) -> Tuple[int, int]:
116 - """Profiles the peak memory usage of the model to determine how many
117 - KV blocks may be allocated without OOMs.
116 + def determine_available_memory(self) -> int:
117 + """Profiles the peak memory usage of the model to determine how much
118 + memory can be used for KV cache without OOMs.
119
120 The engine will first conduct a profiling of the existing memory usage.
120 - Then, it calculate the maximum possible number of GPU and CPU blocks
121 - that can be allocated with the remaining free memory.
121 + Then, it calculate the free memory that can be used for KV cache in
122 + bytes.
123
124 .. tip::
125 You may limit the usage of GPU memory
126 by adjusting the `gpu_memory_utilization` parameter.
127 """
127 - # Profile the memory usage of the model and get the maximum number of
128 - # cache blocks that can be allocated with the remaining free memory.
128 torch.cuda.empty_cache()
129 torch.cuda.reset_peak_memory_stats()
130
@@ -161,33 +160,14 @@ class Worker:
160 total_gpu_memory * self.cache_config.gpu_memory_utilization -
161 peak_memory)
162
164 - # Calculate the number of blocks that can be allocated with the
165 - # profiled peak memory.
166 - cache_block_size = _get_cache_block_size(self.cache_config,
167 - self.model_config,
168 - self.parallel_config)
169 - num_gpu_blocks = int(available_kv_cache_memory // cache_block_size)
170 - num_gpu_blocks = max(num_gpu_blocks, 0)
171 - return num_gpu_blocks, 0
172 -
173 - def initialize_cache(self, num_gpu_blocks: int) -> None:
174 - """Allocate GPU and CPU KV cache with the specified number of blocks."""
175 - if num_gpu_blocks <= 0:
176 - raise ValueError("No available memory for the cache blocks. "
177 - "Try increasing `gpu_memory_utilization` when "
178 - "initializing the engine.")
179 -
180 - max_seq_len = self.cache_config.block_size * num_gpu_blocks
181 - max_model_len = self.model_config.max_model_len
182 - if max_model_len > max_seq_len:
183 - raise ValueError(
184 - f"The model's max seq len ({max_model_len}) "
185 - "is larger than the maximum number of tokens that can be "
186 - f"stored in KV cache ({max_seq_len}). Try increasing "
187 - "`gpu_memory_utilization` or decreasing `max_model_len` when "
188 - "initializing the engine.")
163 + return int(available_kv_cache_memory)
164 +
165 + def get_kv_cache_spec(self) -> KVCacheSpec:
166 + return self.model_runner.get_kv_cache_spec()
167
190 - self.model_runner.initialize_kv_cache(num_gpu_blocks)
168 + def initialize_cache(self, kv_cache_config: KVCacheConfig) -> None:
169 + """Allocate GPU KV cache with the specified kv_cache_config."""
170 + self.model_runner.initialize_kv_cache(kv_cache_config)
171
172 def compile_or_warm_up_model(self) -> None:
173 if not self.model_config.enforce_eager: