@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7c7067b45
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Misc] [Core] Implement RFC "Augment BaseExecutor interfaces to enable hardware-agnostic speculative decoding" (#3837)

Cade Daniel committed Apr 9, 2024 at 11:44 UTC e7c7067b45c9f604e0c68015ee6e0fe345288111
20 files changed +453 -277
tests/core/block/e2e/test_correctness.py
+3 -3
@@ -16,7 +16,7 @@ from vllm import SamplingParams
16
17 # Allow only 5 sequences of ~1024 tokens in worst case.
18 "block_size": 16,
19 - "forced_num_gpu_blocks": 5 * (64 + 1),
19 + "num_gpu_blocks_override": 5 * (64 + 1),
20 }])
21 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
22 @pytest.mark.parametrize("baseline_llm_kwargs", [{
@@ -162,14 +162,14 @@ def test_v1_v2_greedy_equality_with_cow(baseline_llm_generator,
162
163 # Allow only 2 sequences of ~128 tokens in worst case.
164 # Note 8 = 128/block_size
165 - "forced_num_gpu_blocks": 2 * (8 + 1),
165 + "num_gpu_blocks_override": 2 * (8 + 1),
166 },
167 {
168 "block_size": 8,
169
170 # Allow only 2 sequences of ~128 tokens in worst case.
171 # Note 16 = 128/block_size
172 - "forced_num_gpu_blocks": 2 * (16 + 1),
172 + "num_gpu_blocks_override": 2 * (16 + 1),
173 }
174 ])
175 @pytest.mark.parametrize("baseline_llm_kwargs", [{
tests/lora/test_worker.py
+6 -2
@@ -3,8 +3,8 @@ import random
3 import tempfile
4 from unittest.mock import patch
5
6 -from vllm.config import (DeviceConfig, LoRAConfig, ModelConfig, ParallelConfig,
7 - SchedulerConfig)
6 +from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7 + ParallelConfig, SchedulerConfig)
8 from vllm.lora.models import LoRAMapping
9 from vllm.lora.request import LoRARequest
10 from vllm.worker.worker import Worker
@@ -27,6 +27,10 @@ def test_worker_apply_lora(sql_lora_files):
27 parallel_config=ParallelConfig(1, 1, False),
28 scheduler_config=SchedulerConfig(32, 32, 32),
29 device_config=DeviceConfig("cuda"),
30 + cache_config=CacheConfig(block_size=16,
31 + gpu_memory_utilization=1.,
32 + swap_space=0,
33 + cache_dtype="auto"),
34 local_rank=0,
35 rank=0,
36 lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
tests/spec_decode/test_spec_decode_worker.py
+13 -22
@@ -512,8 +512,8 @@ def test_init_device():
512
513
514 @torch.inference_mode()
515 -def test_init_cache_engine():
516 - """Verify SpecDecodeWorker invokes init_cache_engine on proposer/scorer
515 +def test_initialize_cache():
516 + """Verify SpecDecodeWorker invokes initialize_cache on proposer/scorer
517 workers.
518 """
519 draft_worker = mock_worker(cls=MultiStepWorker)
@@ -525,12 +525,11 @@ def test_init_cache_engine():
525 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
526 metrics_collector)
527
528 - cache_config = MagicMock()
528 + kwargs = {"num_gpu_blocks": 1024, "num_cpu_blocks": 1023}
529 + worker.initialize_cache(**kwargs)
530
530 - worker.init_cache_engine(cache_config)
531 -
532 - draft_worker.init_cache_engine.assert_called_once_with(cache_config)
533 - target_worker.init_cache_engine.assert_called_once_with(cache_config)
531 + draft_worker.initialize_cache.assert_called_once_with(**kwargs)
532 + target_worker.initialize_cache.assert_called_once_with(**kwargs)
533
534
535 @pytest.mark.parametrize('available_gpu_blocks', [1, 1024])
@@ -538,10 +537,10 @@ def test_init_cache_engine():
537 @pytest.mark.parametrize('target_cache_block_size_bytes', [2 * 2 * 4096])
538 @pytest.mark.parametrize('draft_kv_size_bytes', [0, 2 * 2 * 768, 2 * 2 * 4096])
539 @pytest.mark.skip_global_cleanup
541 -def test_profile_num_available_blocks(available_gpu_blocks: int,
542 - available_cpu_blocks: int,
543 - target_cache_block_size_bytes: int,
544 - draft_kv_size_bytes: int):
540 +def test_determine_num_available_blocks(available_gpu_blocks: int,
541 + available_cpu_blocks: int,
542 + target_cache_block_size_bytes: int,
543 + draft_kv_size_bytes: int):
544 """Verify SpecDecodeWorker correctly profiles num available GPU blocks.
545 Specifically, it should run profiling in the scorer worker, and then evenly
546 split the blocks between proposer and scorer worker.
@@ -552,7 +551,7 @@ def test_profile_num_available_blocks(available_gpu_blocks: int,
551 rejection_sampler.token_id_dtype = torch.int64
552 metrics_collector = MagicMock(spec=AsyncMetricsCollector)
553
555 - target_worker.profile_num_available_blocks.return_value = (
554 + target_worker.determine_num_available_blocks.return_value = (
555 available_gpu_blocks, available_cpu_blocks)
556 target_worker.get_cache_block_size_bytes.return_value = (
557 target_cache_block_size_bytes)
@@ -561,17 +560,9 @@ def test_profile_num_available_blocks(available_gpu_blocks: int,
560 worker = SpecDecodeWorker(draft_worker, target_worker, rejection_sampler,
561 metrics_collector)
562
564 - # These values do not directly impact the adjusted block size calculation,
565 - # so they can be fixed.
566 - gpu_memory_utilization = 0.9
567 - cpu_swap_space = 100
568 - block_size = 16
569 -
570 - num_gpu_blocks, num_cpu_blocks = worker.profile_num_available_blocks(
571 - block_size, gpu_memory_utilization, cpu_swap_space, cache_dtype="auto")
563 + num_gpu_blocks, num_cpu_blocks = worker.determine_num_available_blocks()
564
573 - target_worker.profile_num_available_blocks.assert_called_once_with(
574 - block_size, gpu_memory_utilization, cpu_swap_space, "auto")
565 + target_worker.determine_num_available_blocks.assert_called_once()
566 assert num_cpu_blocks == available_cpu_blocks
567
568 assert num_gpu_blocks == split_num_cache_blocks_evenly(
tests/spec_decode/utils.py
+4 -2
@@ -117,6 +117,7 @@ def create_worker(cls: type,
117 parallel_config=engine_config.parallel_config,
118 scheduler_config=engine_config.scheduler_config,
119 device_config=engine_config.device_config,
120 + cache_config=engine_config.cache_config,
121 local_rank=0,
122 rank=0,
123 distributed_init_method=distributed_init_method,
@@ -128,8 +129,9 @@ def create_worker(cls: type,
129
130 engine_config.cache_config.num_gpu_blocks = num_gpu_blocks
131 engine_config.cache_config.num_cpu_blocks = 0
131 - worker.init_cache_engine(engine_config.cache_config)
132 - worker.warm_up_model()
132 + worker.initialize_cache(
133 + num_gpu_blocks=engine_config.cache_config.num_gpu_blocks,
134 + num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
135
136 return worker
137
tests/worker/test_swap.py
+6 -4
@@ -11,8 +11,8 @@ def test_swap() -> None:
11 dtype="half",
12 load_format="dummy")
13 engine_config = engine_args.create_engine_config()
14 - engine_config.cache_config.num_gpu_blocks = 100
15 - engine_config.cache_config.num_cpu_blocks = 100
14 + engine_config.cache_config.num_gpu_blocks = 1000
15 + engine_config.cache_config.num_cpu_blocks = 1000
16
17 # Create the worker.
18 distributed_init_method = get_distributed_init_method(
@@ -22,6 +22,7 @@ def test_swap() -> None:
22 parallel_config=engine_config.parallel_config,
23 scheduler_config=engine_config.scheduler_config,
24 device_config=engine_config.device_config,
25 + cache_config=engine_config.cache_config,
26 local_rank=0,
27 rank=0,
28 distributed_init_method=distributed_init_method,
@@ -31,8 +32,9 @@ def test_swap() -> None:
32 # Initialize the worker.
33 worker.init_device()
34 worker.load_model()
34 - worker.init_cache_engine(engine_config.cache_config)
35 - worker.warm_up_model()
35 + worker.initialize_cache(
36 + num_gpu_blocks=engine_config.cache_config.num_gpu_blocks,
37 + num_cpu_blocks=engine_config.cache_config.num_cpu_blocks)
38
39 # Randomly initialize the cache.
40 gpu_cache = worker.cache_engine.gpu_cache
vllm/config.py
+3 -3
@@ -334,7 +334,7 @@ class CacheConfig:
334 vLLM execution.
335 swap_space: Size of the CPU swap space per GPU (in GiB).
336 cache_dtype: Data type for kv cache storage.
337 - forced_num_gpu_blocks: Number of GPU blocks to use. This overrides the
337 + num_gpu_blocks_override: Number of GPU blocks to use. This overrides the
338 profiled num_gpu_blocks if specified. Does nothing if None.
339 """
340
@@ -344,14 +344,14 @@ class CacheConfig:
344 gpu_memory_utilization: float,
345 swap_space: int,
346 cache_dtype: str,
347 - forced_num_gpu_blocks: Optional[int] = None,
347 + num_gpu_blocks_override: Optional[int] = None,
348 sliding_window: Optional[int] = None,
349 enable_prefix_caching: bool = False,
350 ) -> None:
351 self.block_size = block_size
352 self.gpu_memory_utilization = gpu_memory_utilization
353 self.swap_space_bytes = swap_space * _GB
354 - self.forced_num_gpu_blocks = forced_num_gpu_blocks
354 + self.num_gpu_blocks_override = num_gpu_blocks_override
355 self.cache_dtype = cache_dtype
356 self.sliding_window = sliding_window
357 self.enable_prefix_caching = enable_prefix_caching
vllm/engine/arg_utils.py
+3 -3
@@ -55,7 +55,7 @@ class EngineArgs:
55 max_cpu_loras: Optional[int] = None
56 device: str = 'auto'
57 ray_workers_use_nsight: bool = False
58 - forced_num_gpu_blocks: Optional[int] = None
58 + num_gpu_blocks_override: Optional[int] = None
59 num_lookahead_slots: int = 0
60
61 # Related to Vision-language models such as llava
@@ -246,7 +246,7 @@ class EngineArgs:
246 'the model executor, which can range from 0 to 1.'
247 'If unspecified, will use the default value of 0.9.')
248 parser.add_argument(
249 - '--forced-num-gpu-blocks',
249 + '--num-gpu-blocks-override',
250 type=int,
251 default=None,
252 help='If specified, ignore GPU profiling result and use this number'
@@ -426,7 +426,7 @@ class EngineArgs:
426 cache_config = CacheConfig(self.block_size,
427 self.gpu_memory_utilization,
428 self.swap_space, self.kv_cache_dtype,
429 - self.forced_num_gpu_blocks,
429 + self.num_gpu_blocks_override,
430 model_config.get_sliding_window(),
431 self.enable_prefix_caching)
432 parallel_config = ParallelConfig(
vllm/engine/llm_engine.py
+22
@@ -127,6 +127,8 @@ class LLMEngine:
127 speculative_config=speculative_config,
128 )
129
130 + self._initialize_kv_caches()
131 +
132 # If usage stat is enabled, collect relevant info.
133 if is_usage_stats_enabled():
134 from vllm.model_executor.model_loader import (
@@ -178,6 +180,26 @@ class LLMEngine:
180 labels=dict(model_name=model_config.model))
181 self.stat_logger.info("cache_config", self.cache_config)
182
183 + def _initialize_kv_caches(self) -> None:
184 + """Initialize the KV cache in the worker(s).
185 +
186 + The workers will determine the number of blocks in both the GPU cache
187 + and the swap CPU cache.
188 + """
189 + num_gpu_blocks, num_cpu_blocks = (
190 + self.model_executor.determine_num_available_blocks())
191 +
192 + if self.cache_config.num_gpu_blocks_override is not None:
193 + num_gpu_blocks_override = self.cache_config.num_gpu_blocks_override
194 + logger.info(f"Overriding {num_gpu_blocks=} with "
195 + f"{num_gpu_blocks_override=}")
196 + num_gpu_blocks = num_gpu_blocks_override
197 +
198 + self.cache_config.num_gpu_blocks = num_gpu_blocks
199 + self.cache_config.num_cpu_blocks = num_cpu_blocks
200 +
201 + self.model_executor.initialize_cache(num_gpu_blocks, num_cpu_blocks)
202 +
203 @classmethod
204 def from_engine_args(
205 cls,
vllm/executor/cpu_executor.py
+22 -36
@@ -35,7 +35,6 @@ class CPUExecutor(ExecutorBase):
35
36 # Instantiate the worker and load the model to CPU.
37 self._init_worker()
38 - self._init_cache()
38
39 def _init_worker(self):
40 from vllm.worker.cpu_worker import CPUWorker
@@ -46,10 +45,11 @@ class CPUExecutor(ExecutorBase):
45 distributed_init_method = get_distributed_init_method(
46 get_ip(), get_open_port())
47 self.driver_worker = CPUWorker(
49 - self.model_config,
50 - self.parallel_config,
51 - self.scheduler_config,
52 - self.device_config,
48 + model_config=self.model_config,
49 + parallel_config=self.parallel_config,
50 + scheduler_config=self.scheduler_config,
51 + device_config=self.device_config,
52 + cache_config=self.cache_config,
53 local_rank=0,
54 rank=0,
55 distributed_init_method=distributed_init_method,
@@ -60,35 +60,21 @@ class CPUExecutor(ExecutorBase):
60 self.driver_worker.init_device()
61 self.driver_worker.load_model()
62
63 - def _init_cache(self) -> None:
64 - num_cpu_blocks = self.driver_worker.get_cpu_cache_block_num(
65 - block_size=self.cache_config.block_size,
66 - cache_space=self.cache_config.cpu_kvcache_space_bytes,
67 - cache_dtype=self.cache_config.cache_dtype,
68 - )
69 -
63 + def determine_num_available_blocks(self) -> tuple[int, int]:
64 + """Determine the number of available KV blocks by invoking the
65 + underlying worker.
66 + """
67 + return self.driver_worker.determine_num_available_blocks()
68 +
69 + def initialize_cache(self, num_gpu_blocks: int,
70 + num_cpu_blocks: int) -> None:
71 + """Initialize the KV cache by invoking the underlying worker.
72 + """
73 + # NOTE: We log here to avoid multiple logs when number of workers is
74 + # greater than one. We could log in the engine, but not all executors
75 + # have GPUs.
76 logger.info(f"# CPU blocks: {num_cpu_blocks}")
71 - if num_cpu_blocks <= 0:
72 - raise ValueError("No available memory for the cache blocks. "
73 - "Try increasing `VLLM_CPU_KVCACHE_SPACE` when "
74 - "initializing the engine.")
75 -
76 - max_seq_len = self.cache_config.block_size * num_cpu_blocks
77 - if self.model_config.max_model_len > max_seq_len:
78 - raise ValueError(
79 - f"The model's max seq len ({self.model_config.max_model_len}) "
80 - "is larger than the maximum number of tokens that can be "
81 - f"stored in KV cache ({max_seq_len}). Try increasing "
82 - "`VLLM_CPU_KVCACHE_SPACE` or decreasing `max_model_len` when "
83 - "initializing the engine.")
84 -
85 - # Note: To reuse the cache management procedure,
86 - # use cpu cache as 'gpu cache'.
87 - self.cache_config.num_gpu_blocks = num_cpu_blocks # type: ignore
88 - self.cache_config.num_cpu_blocks = 0 # type: ignore
89 -
90 - # Initialize the cache.
91 - self.driver_worker.init_cache_engine(cache_config=self.cache_config)
77 + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
78
79 def execute_model(self,
80 seq_group_metadata_list: List[SequenceGroupMetadata],
@@ -104,13 +90,13 @@ class CPUExecutor(ExecutorBase):
90 return output
91
92 def add_lora(self, lora_request: LoRARequest) -> bool:
107 - raise NotImplementedError("LoRA is not implemented for cpu backend.")
93 + return self.driver_worker.add_lora(lora_request)
94
95 def remove_lora(self, lora_id: int) -> bool:
110 - raise NotImplementedError("LoRA is not implemented for cpu backend.")
96 + return self.driver_worker.remove_lora(lora_id)
97
98 def list_loras(self) -> List[int]:
113 - raise NotImplementedError("LoRA is not implemented for cpu backend.")
99 + return self.driver_worker.list_loras()
100
101 def check_health(self) -> None:
102 # CPUExecutor will always be healthy as long as
vllm/executor/executor_base.py
+23
@@ -30,6 +30,29 @@ class ExecutorBase(ABC):
30 ) -> None:
31 raise NotImplementedError
32
33 + @abstractmethod
34 + def determine_num_available_blocks(self) -> tuple[int, int]:
35 + """Determine the number of available blocks for the GPU KV cache and
36 + swappable CPU KV cache.
37 +
38 + Normally, this should simply delegate to the underlying Worker. Some
39 + ExecutorBase may require modification of the result, e.g. to ensure the
40 + selected cache sizes are compatible with all workers.
41 +
42 + Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
43 + are blocks that are "active" on the device and can be appended to.
44 + num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
45 + appended to.
46 + """
47 + raise NotImplementedError
48 +
49 + @abstractmethod
50 + def initialize_cache(self, num_gpu_blocks: int,
51 + num_cpu_blocks: int) -> None:
52 + """Initialize the KV cache with the given size in blocks.
53 + """
54 + raise NotImplementedError
55 +
56 @abstractmethod
57 def execute_model(self,
58 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/executor/gpu_executor.py
+16 -44
@@ -4,7 +4,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 ParallelConfig, SchedulerConfig, SpeculativeConfig,
5 VisionLanguageConfig)
6 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7 -from vllm.executor.utils import check_block_size_valid
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
9 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -41,9 +40,6 @@ class GPUExecutor(ExecutorBase):
40 # Instantiate the worker and load the model to GPU.
41 self._init_worker()
42
44 - # Profile the memory usage and initialize the cache.
45 - self._init_cache()
46 -
43 def _init_worker(self):
44 # Lazy import the Worker to avoid importing torch.cuda/xformers
45 # before CUDA_VISIBLE_DEVICES is set in the Worker
@@ -55,61 +51,37 @@ class GPUExecutor(ExecutorBase):
51 distributed_init_method = get_distributed_init_method(
52 get_ip(), get_open_port())
53 self.driver_worker = Worker(
58 - self.model_config,
59 - self.parallel_config,
60 - self.scheduler_config,
61 - self.device_config,
54 + model_config=self.model_config,
55 + parallel_config=self.parallel_config,
56 + scheduler_config=self.scheduler_config,
57 + device_config=self.device_config,
58 + cache_config=self.cache_config,
59 local_rank=0,
60 rank=0,
61 distributed_init_method=distributed_init_method,
62 lora_config=self.lora_config,
63 vision_language_config=self.vision_language_config,
67 - kv_cache_dtype=self.cache_config.cache_dtype,
64 is_driver_worker=True,
65 )
66 self.driver_worker.init_device()
67 self.driver_worker.load_model()
68
73 - def _init_cache(self) -> None:
74 - """Profiles the memory usage and initializes the KV cache.
75 -
76 - The engine first profiles the existing memory usage.
77 - Then, it allocates the remaining memory for KV blocks.
78 -
79 - .. tip::
80 - You may limit the usage of GPU memory
81 - by adjusting the `gpu_memory_utilization` parameter.
69 + def determine_num_available_blocks(self) -> tuple[int, int]:
70 + """Determine the number of available KV blocks by invoking the
71 + underlying worker.
72 """
83 - # Get the maximum number of blocks that can be allocated on GPU and CPU.
84 - num_gpu_blocks, num_cpu_blocks = (
85 - self.driver_worker.profile_num_available_blocks(
86 - block_size=self.cache_config.block_size,
87 - gpu_memory_utilization=self.cache_config.
88 - gpu_memory_utilization,
89 - cpu_swap_space=self.cache_config.swap_space_bytes,
90 - cache_dtype=self.cache_config.cache_dtype,
91 - ))
92 -
93 - if self.cache_config.forced_num_gpu_blocks is not None:
94 - forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
95 - logger.info(f"Replacing profiled {num_gpu_blocks=} with "
96 - f"{forced_num_gpu_blocks=}")
97 - num_gpu_blocks = forced_num_gpu_blocks
73 + return self.driver_worker.determine_num_available_blocks()
74
75 + def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks) -> None:
76 + """Initialize the KV cache by invoking the underlying worker.
77 + """
78 + # NOTE: This is logged in the executor because there can be >1 worker
79 + # with other executors. We could log in the engine level, but work
80 + # remains to abstract away the device for non-GPU configurations.
81 logger.info(f"# GPU blocks: {num_gpu_blocks}, "
82 f"# CPU blocks: {num_cpu_blocks}")
83
102 - check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
103 - self.model_config.max_model_len)
104 -
105 - self.cache_config.num_gpu_blocks = num_gpu_blocks
106 - self.cache_config.num_cpu_blocks = num_cpu_blocks
107 -
108 - # Initialize the cache.
109 - self.driver_worker.init_cache_engine(cache_config=self.cache_config)
110 - # Warm up the model. This includes capturing the model into CUDA graph
111 - # if enforce_eager is False.
112 - self.driver_worker.warm_up_model()
84 + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
85
86 def execute_model(self,
87 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/executor/neuron_executor.py
+15 -13
@@ -25,7 +25,6 @@ class NeuronExecutor(ExecutorBase):
25 speculative_config: Optional[SpeculativeConfig],
26 ) -> None:
27 self.model_config = model_config
28 - self.cache_config = cache_config
28 assert lora_config is None, "LoRA is not supported for Neuron backend."
29 self.parallel_config = parallel_config
30 self.scheduler_config = scheduler_config
@@ -33,12 +32,6 @@ class NeuronExecutor(ExecutorBase):
32 assert (not speculative_config
33 ), "Speculative decoding not yet supported for Neuron backend."
34
36 - # Set the number of GPU blocks to be the same as the maximum number of
37 - # sequences that can be processed in a single batch. This is equivalent
38 - # to schedule without PagedAttention.
39 - self.cache_config.num_gpu_blocks = self.scheduler_config.max_num_seqs
40 - self.cache_config.num_cpu_blocks = 0
41 -
35 # Instantiate the worker and load the model to the device.
36 self._init_worker()
37
@@ -54,6 +47,18 @@ class NeuronExecutor(ExecutorBase):
47 self.driver_worker.init_device()
48 self.driver_worker.load_model()
49
50 + def determine_num_available_blocks(self) -> tuple[int, int]:
51 + """Determine the number of available KV blocks by invoking the
52 + underlying worker.
53 + """
54 + return self.driver_worker.determine_num_available_blocks()
55 +
56 + def initialize_cache(self, num_gpu_blocks: int,
57 + num_cpu_blocks: int) -> None:
58 + """Initialize the KV cache by invoking the underlying worker.
59 + """
60 + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
61 +
62 def execute_model(self,
63 seq_group_metadata_list: List[SequenceGroupMetadata],
64 blocks_to_swap_in: Dict[int, int],
@@ -68,16 +73,13 @@ class NeuronExecutor(ExecutorBase):
73 return output
74
75 def add_lora(self, lora_request: LoRARequest) -> bool:
71 - raise NotImplementedError(
72 - "LoRA is not implemented for neuron backend.")
76 + return self.driver_worker.add_lora(lora_request)
77
78 def remove_lora(self, lora_id: int) -> bool:
75 - raise NotImplementedError(
76 - "LoRA is not implemented for neuron backend.")
79 + return self.driver_worker.remove_lora(lora_id)
80
81 def list_loras(self) -> List[int]:
79 - raise NotImplementedError(
80 - "LoRA is not implemented for neuron backend.")
82 + return self.driver_worker.list_loras()
83
84 def check_health(self) -> None:
85 # NeuronExecutor will always be healthy as long as
vllm/executor/ray_gpu_executor.py
+37 -59
@@ -10,7 +10,6 @@ from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
10 VisionLanguageConfig)
11 from vllm.engine.ray_utils import RayWorkerVllm, ray
12 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13 -from vllm.executor.utils import check_block_size_valid
13 from vllm.logger import init_logger
14 from vllm.lora.request import LoRARequest
15 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -65,9 +64,6 @@ class RayGPUExecutor(ExecutorBase):
64 # Create the parallel GPU workers.
65 self._init_workers_ray(placement_group)
66
68 - # Profile the memory usage and initialize the cache.
69 - self._init_cache()
70 -
67 self.forward_dag = None
68 if USE_RAY_COMPILED_DAG:
69 self.forward_dag = self._compiled_ray_dag()
@@ -154,8 +150,8 @@ class RayGPUExecutor(ExecutorBase):
150 scheduler_config = copy.deepcopy(self.scheduler_config)
151 device_config = copy.deepcopy(self.device_config)
152 lora_config = copy.deepcopy(self.lora_config)
153 + cache_config = copy.deepcopy(self.cache_config)
154 vision_language_config = copy.deepcopy(self.vision_language_config)
158 - kv_cache_dtype = self.cache_config.cache_dtype
155
156 # Initialize the actual workers with the Worker class.
157 for rank, (worker, (node_id, _)) in enumerate(
@@ -165,32 +161,32 @@ class RayGPUExecutor(ExecutorBase):
161 local_rank = node_workers[node_id].index(rank)
162 worker.init_worker.remote(
163 lambda rank=rank, local_rank=local_rank: Worker(
168 - model_config,
169 - parallel_config,
170 - scheduler_config,
171 - device_config,
172 - local_rank,
173 - rank,
174 - distributed_init_method,
164 + model_config=model_config,
165 + parallel_config=parallel_config,
166 + scheduler_config=scheduler_config,
167 + device_config=device_config,
168 + cache_config=cache_config,
169 + local_rank=local_rank,
170 + rank=rank,
171 + distributed_init_method=distributed_init_method,
172 lora_config=lora_config,
173 vision_language_config=vision_language_config,
177 - kv_cache_dtype=kv_cache_dtype,
174 ))
175
176 # Initialize the driver worker with the Worker class.
177 driver_rank = 0
178 driver_local_rank = node_workers[driver_node_id].index(driver_rank)
179 self.driver_worker = Worker(
184 - self.model_config,
185 - self.parallel_config,
186 - self.scheduler_config,
187 - self.device_config,
188 - driver_local_rank,
189 - driver_rank,
190 - distributed_init_method,
180 + model_config=self.model_config,
181 + parallel_config=self.parallel_config,
182 + scheduler_config=self.scheduler_config,
183 + device_config=self.device_config,
184 + cache_config=self.cache_config,
185 + local_rank=driver_local_rank,
186 + rank=driver_rank,
187 + distributed_init_method=distributed_init_method,
188 lora_config=self.lora_config,
189 vision_language_config=self.vision_language_config,
193 - kv_cache_dtype=kv_cache_dtype,
190 is_driver_worker=True,
191 )
192
@@ -201,35 +197,18 @@ class RayGPUExecutor(ExecutorBase):
197 max_parallel_loading_workers,
198 )
199
204 - def _init_cache(self) -> None:
205 - """Profiles the memory usage and initializes the KV cache.
206 -
207 - The engine will first conduct a profiling of the existing memory usage.
208 - Then, it calculate the maximum possible number of GPU and CPU blocks
209 - that can be allocated with the remaining free memory.
210 - More details can be found in the
211 - :meth:`~vllm.worker.worker.Worker.profile_num_available_blocks` method
212 - from class :class:`~vllm.worker.Worker`.
200 + def determine_num_available_blocks(self) -> tuple[int, int]:
201 + """Determine the number of available KV blocks.
202
214 - Afterwards, as there may be multiple workers,
215 - we take the minimum number of blocks across all workers
216 - to ensure this can be applied to all of them.
203 + This invokes `determine_num_available_blocks` on each worker and takes
204 + the min of the results, guaranteeing that the selected cache sizes are
205 + compatible with all workers.
206
218 - Finally, the engine will initialize the KV cache
219 - with the calculated number of blocks.
220 -
221 - .. tip::
222 - You may limit the usage of GPU memory
223 - by adjusting the `gpu_memory_utilization` parameter.
207 + Returns:
208 + - tuple[num_gpu_blocks, num_cpu_blocks]
209 """
210 # Get the maximum number of blocks that can be allocated on GPU and CPU.
226 - num_blocks = self._run_workers(
227 - "profile_num_available_blocks",
228 - block_size=self.cache_config.block_size,
229 - gpu_memory_utilization=self.cache_config.gpu_memory_utilization,
230 - cpu_swap_space=self.cache_config.swap_space_bytes,
231 - cache_dtype=self.cache_config.cache_dtype,
232 - )
211 + num_blocks = self._run_workers("determine_num_available_blocks", )
212
213 # Since we use a shared centralized controller, we take the minimum
214 # number of blocks across all workers to make sure all the memory
@@ -237,26 +216,25 @@ class RayGPUExecutor(ExecutorBase):
216 num_gpu_blocks = min(b[0] for b in num_blocks)
217 num_cpu_blocks = min(b[1] for b in num_blocks)
218
240 - if self.cache_config.forced_num_gpu_blocks is not None:
241 - forced_num_gpu_blocks = self.cache_config.forced_num_gpu_blocks
242 - logger.info(f"Replacing profiled {num_gpu_blocks=} with "
243 - f"{forced_num_gpu_blocks=}")
244 - num_gpu_blocks = forced_num_gpu_blocks
219 + return num_gpu_blocks, num_cpu_blocks
220
221 + def initialize_cache(self, num_gpu_blocks: int,
222 + num_cpu_blocks: int) -> None:
223 + """Initialize the KV cache in all workers.
224 + """
225 +
226 + # NOTE: We log here to avoid multiple logs when number of workers is
227 + # greater than one. We could log in the engine, but not all executors
228 + # have GPUs.
229 logger.info(f"# GPU blocks: {num_gpu_blocks}, "
230 f"# CPU blocks: {num_cpu_blocks}")
231
249 - check_block_size_valid(num_gpu_blocks, self.cache_config.block_size,
250 - self.model_config.max_model_len)
251 -
232 self.cache_config.num_gpu_blocks = num_gpu_blocks
233 self.cache_config.num_cpu_blocks = num_cpu_blocks
234
255 - # Initialize the cache.
256 - self._run_workers("init_cache_engine", cache_config=self.cache_config)
257 - # Warm up the model. This includes capturing the model into CUDA graph
258 - # if enforce_eager is False.
259 - self._run_workers("warm_up_model")
235 + self._run_workers("initialize_cache",
236 + num_gpu_blocks=num_gpu_blocks,
237 + num_cpu_blocks=num_cpu_blocks)
238
239 def execute_model(self,
240 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/executor/utils.py deleted
-13
@@ -1,13 +0,0 @@
1 -def check_block_size_valid(num_gpu_blocks, block_size, max_model_len) -> None:
2 - if num_gpu_blocks <= 0:
3 - raise ValueError("No available memory for the cache blocks. "
4 - "Try increasing `gpu_memory_utilization` when "
5 - "initializing the engine.")
6 - max_seq_len = block_size * num_gpu_blocks
7 - if max_model_len > max_seq_len:
8 - raise ValueError(
9 - f"The model's max seq len ({max_model_len}) "
10 - "is larger than the maximum number of tokens that can be "
11 - f"stored in KV cache ({max_seq_len}). Try increasing "
12 - "`gpu_memory_utilization` or decreasing `max_model_len` when "
13 - "initializing the engine.")
vllm/spec_decode/spec_decode_worker.py
+22 -16
@@ -3,7 +3,6 @@ from typing import Dict, List, Optional, Tuple
3
4 import torch
5
6 -from vllm.config import CacheConfig
6 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
7 from vllm.sequence import (SamplerOutput, SequenceGroupMetadata,
8 SequenceGroupOutput, SequenceOutput)
@@ -15,9 +14,10 @@ from vllm.spec_decode.multi_step_worker import MultiStepWorker
14 from vllm.spec_decode.util import (get_all_seq_ids, nvtx_range,
15 split_batch_by_proposal_len)
16 from vllm.worker.worker import Worker
17 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase
18
19
20 -class SpecDecodeWorker:
20 +class SpecDecodeWorker(LoraNotSupportedWorkerBase):
21 """Worker which implements speculative decoding.
22
23 Speculative decoding reduces decoding per-token latency by using a proposal
@@ -94,10 +94,7 @@ class SpecDecodeWorker:
94 device=self.device,
95 vocab_size=self._vocab_size)
96
97 - def profile_num_available_blocks(self, block_size: int,
98 - gpu_memory_utilization: float,
99 - cpu_swap_space: int,
100 - cache_dtype: str) -> Tuple[int, int]:
97 + def determine_num_available_blocks(self) -> Tuple[int, int]:
98 """Determine the number of cache blocks to use.
99
100 This is done by profiling the scorer model (which is typically the
@@ -106,27 +103,26 @@ class SpecDecodeWorker:
103 such that the number of blocks is equal in both KV caches.
104 """
105 num_gpu_blocks, num_cpu_blocks = (
109 - self.scorer_worker.profile_num_available_blocks(
110 - block_size, gpu_memory_utilization, cpu_swap_space,
111 - cache_dtype))
106 + self.scorer_worker.determine_num_available_blocks())
107
108 scorer_cache_block_size_bytes = (
114 - self.scorer_worker.get_cache_block_size_bytes(
115 - block_size, cache_dtype))
109 + self.scorer_worker.get_cache_block_size_bytes())
110 proposer_cache_block_size_bytes = (
117 - self.proposer_worker.get_cache_block_size_bytes(
118 - block_size, cache_dtype))
111 + self.proposer_worker.get_cache_block_size_bytes())
112
113 new_num_gpu_blocks = split_num_cache_blocks_evenly(
114 scorer_cache_block_size_bytes, proposer_cache_block_size_bytes,
115 num_gpu_blocks)
116 return new_num_gpu_blocks, num_cpu_blocks
117
125 - def init_cache_engine(self, cache_config: CacheConfig):
118 + def initialize_cache(self, num_gpu_blocks: int,
119 + num_cpu_blocks: int) -> None:
120 """Initialize the cache engine of the scorer and proposer workers.
121 """
128 - self.scorer_worker.init_cache_engine(cache_config)
129 - self.proposer_worker.init_cache_engine(cache_config)
122 + self.scorer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
123 + num_cpu_blocks=num_cpu_blocks)
124 + self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
125 + num_cpu_blocks=num_cpu_blocks)
126
127 @torch.inference_mode()
128 def execute_model(
@@ -351,6 +347,16 @@ class SpecDecodeWorker:
347 def device(self):
348 return self.scorer_worker.device
349
350 + def get_cache_block_size_bytes(self):
351 + """Return the size of a cache block in bytes.
352 +
353 + This function is only used to compose workers within a SpecDecodeWorker.
354 + We leave composing a SpecDecodeWorker within a SpecDecodeWorker
355 + undefined for now, although it could be implemented in the future.
356 + See https://arxiv.org/abs/2308.04623.
357 + """
358 + raise NotImplementedError
359 +
360
361 def split_num_cache_blocks_evenly(scorer_cache_block_size_bytes: int,
362 proposer_cache_block_size_bytes: int,
vllm/worker/cache_engine.py
+4 -5
@@ -82,8 +82,7 @@ class CacheEngine:
82
83 @staticmethod
84 def get_cache_block_size(
85 - block_size: int,
86 - cache_dtype: str,
85 + cache_config: CacheConfig,
86 model_config: ModelConfig,
87 parallel_config: ParallelConfig,
88 ) -> int:
@@ -91,13 +90,13 @@ class CacheEngine:
90 num_heads = model_config.get_num_kv_heads(parallel_config)
91 num_layers = model_config.get_num_layers(parallel_config)
92
94 - key_cache_block = block_size * num_heads * head_size
93 + key_cache_block = cache_config.block_size * num_heads * head_size
94 value_cache_block = key_cache_block
95 total = num_layers * (key_cache_block + value_cache_block)
97 - if cache_dtype == "auto":
96 + if cache_config.cache_dtype == "auto":
97 dtype = model_config.dtype
98 else:
100 - dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
99 + dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
100 dtype_size = _get_dtype_size(dtype)
101 return dtype_size * total
102
vllm/worker/cpu_worker.py
+70 -19
@@ -17,6 +17,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
17 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19 from vllm.worker.model_runner import ModelRunner
20 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase
21
22 logger = init_logger(__name__)
23
@@ -112,7 +113,7 @@ class CPUCacheEngine:
113 return dtype_size * total
114
115
115 -class CPUWorker:
116 +class CPUWorker(LoraNotSupportedWorkerBase):
117 """A worker class that executes (a partition of) the model on a CPU socket.
118
119 Each worker is associated with a single CPU socket. The worker is
@@ -127,6 +128,7 @@ class CPUWorker:
128 parallel_config: ParallelConfig,
129 scheduler_config: SchedulerConfig,
130 device_config: DeviceConfig,
131 + cache_config: CacheConfig,
132 local_rank: int,
133 rank: int,
134 distributed_init_method: str,
@@ -138,6 +140,7 @@ class CPUWorker:
140 self.parallel_config = parallel_config
141 self.scheduler_config = scheduler_config
142 self.device_config = device_config
143 + self.cache_config = cache_config
144 self.local_rank = local_rank
145 self.rank = rank
146 self.distributed_init_method = distributed_init_method
@@ -154,8 +157,7 @@ class CPUWorker:
157 kv_cache_dtype=kv_cache_dtype,
158 is_driver_worker=is_driver_worker)
159 # Uninitialized cache engine. Will be initialized by
157 - # self.init_cache_engine().
158 - self.cache_config = None
160 + # initialize_cache.
161 self.cache_engine = None
162 self.cpu_cache = None
163
@@ -167,28 +169,70 @@ class CPUWorker:
169 def load_model(self):
170 self.model_runner.load_model()
171
170 - def get_cpu_cache_block_num(
171 - self,
172 - block_size: int,
173 - cache_space: int,
174 - cache_dtype: str,
175 - ) -> int:
176 - """
177 - Args:
178 - block_size: The size of the cache block.
179 - cache_space: The size of the CPU KV cache space in bytes.
172 + def determine_num_available_blocks(self) -> tuple[int, int]:
173 + """Determine the number of blocks available for the KV cache.
174 +
175 + This determines how many KV blocks can fit into the configured CPU
176 + KV cache space.
177 +
178 + Note that since vLLM assumes a block resides on GPU if it can be
179 + modified, we return num_gpu_blocks=num_cpu_blocks and num_cpu_blocks=0.
180 + This allows us to reuse the scheduler of vLLM without generalizing it
181 + to different devices.
182 """
183 # For CPU device, the block number will be calculated based on the
184 # cpu_kvcache_space.
183 - cache_block_size = CPUCacheEngine.get_cache_block_size(
184 - block_size, cache_dtype, self.model_config, self.parallel_config)
185 - num_cpu_blocks = int(cache_space // cache_block_size)
185 + cache_block_size = self.get_cache_block_size_bytes()
186 + num_cpu_blocks = int(self.cache_config.cpu_kvcache_space_bytes //
187 + cache_block_size)
188 num_cpu_blocks = max(num_cpu_blocks, 0)
189
188 - return num_cpu_blocks
190 + # Note: To reuse the cache management procedure,
191 + # use cpu cache as 'gpu cache'.
192 + num_gpu_blocks = num_cpu_blocks
193 + num_cpu_blocks = 0
194 + return num_gpu_blocks, num_cpu_blocks
195
190 - def init_cache_engine(self, cache_config: CacheConfig) -> None:
191 - self.cache_config = cache_config
196 + def initialize_cache(self, num_gpu_blocks: int,
197 + num_cpu_blocks: int) -> None:
198 + """Initialize the KV cache. Currently, swappable CPU memory is not
199 + supported.
200 +
201 + Since this worker does not support GPUs, we use the num_gpu_blocks to
202 + determine how many non-swappable CPU blocks to allocate.
203 + """
204 + assert (num_cpu_blocks == 0
205 + ), f"{type(self)} does not support swappable cache"
206 +
207 + # Note: To reuse the cache management procedure,
208 + # use cpu cache as 'gpu cache'.
209 + num_cpu_blocks = num_gpu_blocks
210 +
211 + self._validate_num_cpu_blocks(num_cpu_blocks)
212 + self.cache_config.num_gpu_blocks = num_cpu_blocks
213 + self.cache_config.num_cpu_blocks = 0
214 +
215 + # Initialize the cache.
216 + self._init_cache_engine()
217 +
218 + def _validate_num_cpu_blocks(self, num_cpu_blocks: int) -> None:
219 + """Raise errors if the num_cpu_blocks is invalid.
220 + """
221 + if num_cpu_blocks <= 0:
222 + raise ValueError("No available memory for the cache blocks. "
223 + "Try increasing `VLLM_CPU_KVCACHE_SPACE` when "
224 + "initializing the engine.")
225 +
226 + max_seq_len = self.cache_config.block_size * num_cpu_blocks
227 + if self.model_config.max_model_len > max_seq_len:
228 + raise ValueError(
229 + f"The model's max seq len ({self.model_config.max_model_len}) "
230 + "is larger than the maximum number of tokens that can be "
231 + f"stored in KV cache ({max_seq_len}). Try increasing "
232 + "`VLLM_CPU_KVCACHE_SPACE` or decreasing `max_model_len` when "
233 + "initializing the engine.")
234 +
235 + def _init_cache_engine(self) -> None:
236 self.cache_engine = CPUCacheEngine(self.cache_config,
237 self.model_config,
238 self.parallel_config,
@@ -264,3 +308,10 @@ class CPUWorker:
308 ensure_model_parallel_initialized(
309 parallel_config.tensor_parallel_size,
310 parallel_config.pipeline_parallel_size)
311 +
312 + def get_cache_block_size_bytes(self) -> int:
313 + """Return the size in bytes of a single KV cache block.
314 + """
315 + return CPUCacheEngine.get_cache_block_size(
316 + self.cache_config.block_size, self.cache_config.cache_dtype,
317 + self.model_config, self.parallel_config)
vllm/worker/neuron_worker.py
+42 -3
@@ -4,14 +4,15 @@ from typing import List, Optional
4 import torch
5 import torch.distributed
6
7 -from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8 - SchedulerConfig)
7 +from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
8 + ParallelConfig, SchedulerConfig)
9 from vllm.model_executor import set_random_seed
10 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
11 from vllm.worker.neuron_model_runner import NeuronModelRunner
12 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase
13
14
14 -class NeuronWorker:
15 +class NeuronWorker(LoraNotSupportedWorkerBase):
16 """A worker class that executes the model on a group of neuron cores.
17 """
18
@@ -21,11 +22,13 @@ class NeuronWorker:
22 parallel_config: ParallelConfig,
23 scheduler_config: SchedulerConfig,
24 device_config: DeviceConfig,
25 + cache_config: CacheConfig,
26 ) -> None:
27 self.model_config = model_config
28 self.parallel_config = parallel_config
29 self.scheduler_config = scheduler_config
30 self.device_config = device_config
31 + self.cache_config = cache_config
32
33 self.model_runner = NeuronModelRunner(model_config, parallel_config,
34 scheduler_config, device_config)
@@ -37,6 +40,35 @@ class NeuronWorker:
40 def load_model(self):
41 self.model_runner.load_model()
42
43 + def determine_num_available_blocks(self) -> tuple[int, int]:
44 + """Determine the number of available KV blocks.
45 +
46 + Swapping is not yet supported, so always return num_cpu_blocks=0.
47 +
48 + We configure num_gpu_blocks to be equal to max_num_seqs.
49 + """
50 + # Set the number of GPU blocks to be the same as the maximum number of
51 + # sequences that can be processed in a single batch. This is equivalent
52 + # to schedule without PagedAttention.
53 + num_gpu_blocks = self.scheduler_config.max_num_seqs
54 +
55 + # Swap not yet supported with Neuron backend.
56 + num_cpu_blocks = 0
57 +
58 + return num_gpu_blocks, num_cpu_blocks
59 +
60 + def initialize_cache(self, num_gpu_blocks: int,
61 + num_cpu_blocks: int) -> None:
62 + """Initialize the KV cache.
63 + """
64 +
65 + # Different values are not tested.
66 + assert num_cpu_blocks == 0
67 + assert num_gpu_blocks == self.scheduler_config.max_num_seqs
68 +
69 + self.cache_config.num_gpu_blocks = num_gpu_blocks
70 + self.cache_config.num_cpu_blocks = num_cpu_blocks
71 +
72 @torch.inference_mode()
73 def execute_model(
74 self,
@@ -50,3 +82,10 @@ class NeuronWorker:
82
83 output = self.model_runner.execute_model(seq_group_metadata_list)
84 return output
85 +
86 + def get_cache_block_size_bytes(self) -> int:
87 + """Determine the size in bytes of a cache block.
88 +
89 + This is required for speculative decoding; it is not yet implemented.
90 + """
91 + raise NotImplementedError
vllm/worker/worker.py
+59 -30
@@ -19,9 +19,10 @@ from vllm.model_executor.parallel_utils.parallel_state import (
19 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
20 from vllm.worker.cache_engine import CacheEngine
21 from vllm.worker.model_runner import ModelRunner
22 +from vllm.worker.worker_base import WorkerBase
23
24
24 -class Worker:
25 +class Worker(WorkerBase):
26 """A worker class that executes (a partition of) the model on a GPU.
27
28 Each worker is associated with a single GPU. The worker is responsible for
@@ -35,18 +36,19 @@ class Worker:
36 parallel_config: ParallelConfig,
37 scheduler_config: SchedulerConfig,
38 device_config: DeviceConfig,
39 + cache_config: CacheConfig,
40 local_rank: int,
41 rank: int,
42 distributed_init_method: str,
43 lora_config: Optional[LoRAConfig] = None,
44 vision_language_config: Optional[VisionLanguageConfig] = None,
43 - kv_cache_dtype: Optional[str] = "auto",
45 is_driver_worker: bool = False,
46 ) -> None:
47 self.model_config = model_config
48 self.parallel_config = parallel_config
49 self.scheduler_config = scheduler_config
50 self.device_config = device_config
51 + self.cache_config = cache_config
52 self.local_rank = local_rank
53 self.rank = rank
54 self.distributed_init_method = distributed_init_method
@@ -66,12 +68,11 @@ class Worker:
68 scheduler_config,
69 device_config,
70 lora_config=self.lora_config,
69 - kv_cache_dtype=kv_cache_dtype,
71 + kv_cache_dtype=self.cache_config.cache_dtype,
72 is_driver_worker=is_driver_worker,
73 vision_language_config=vision_language_config)
74 # Uninitialized cache engine. Will be initialized by
73 - # self.init_cache_engine().
74 - self.cache_config = None
75 + # initialize_cache.
76 self.cache_engine = None
77 self.gpu_cache = None
78
@@ -107,20 +108,17 @@ class Worker:
108 self.model_runner.load_model()
109
110 @torch.inference_mode()
110 - def profile_num_available_blocks(
111 - self,
112 - block_size: int,
113 - gpu_memory_utilization: float,
114 - cpu_swap_space: int,
115 - cache_dtype: str,
116 - ) -> Tuple[int, int]:
117 - """Profiles the peak memory usage of the model and returns the maximum
118 - number of GPU and CPU cache blocks that can be allocated.
119 -
120 - Args:
121 - block_size: The size of the cache block.
122 - gpu_memory_utilization: The fraction of the total GPU memory to use.
123 - cpu_swap_space: The size of the CPU swap space in bytes.
111 + def determine_num_available_blocks(self) -> Tuple[int, int]:
112 + """Profiles the peak memory usage of the model to determine how many
113 + KV blocks may be allocated without OOMs.
114 +
115 + The engine will first conduct a profiling of the existing memory usage.
116 + Then, it calculate the maximum possible number of GPU and CPU blocks
117 + that can be allocated with the remaining free memory.
118 +
119 + .. tip::
120 + You may limit the usage of GPU memory
121 + by adjusting the `gpu_memory_utilization` parameter.
122 """
123 # Profile the memory usage of the model and get the maximum number of
124 # cache blocks that can be allocated with the remaining free memory.
@@ -141,12 +139,12 @@ class Worker:
139 "Error in memory profiling. This happens when the GPU memory was "
140 "not properly cleaned up before initializing the vLLM instance.")
141
144 - cache_block_size = self.get_cache_block_size_bytes(
145 - block_size, cache_dtype)
142 + cache_block_size = self.get_cache_block_size_bytes()
143 num_gpu_blocks = int(
147 - (total_gpu_memory * gpu_memory_utilization - peak_memory) //
148 - cache_block_size)
149 - num_cpu_blocks = int(cpu_swap_space // cache_block_size)
144 + (total_gpu_memory * self.cache_config.gpu_memory_utilization -
145 + peak_memory) // cache_block_size)
146 + num_cpu_blocks = int(self.cache_config.swap_space_bytes //
147 + cache_block_size)
148 num_gpu_blocks = max(num_gpu_blocks, 0)
149 num_cpu_blocks = max(num_cpu_blocks, 0)
150 if self.model_runner.lora_manager:
@@ -155,14 +153,30 @@ class Worker:
153 torch.cuda.empty_cache()
154 return num_gpu_blocks, num_cpu_blocks
155
158 - def init_cache_engine(self, cache_config: CacheConfig) -> None:
159 - self.cache_config = cache_config
156 + def initialize_cache(self, num_gpu_blocks: int,
157 + num_cpu_blocks: int) -> None:
158 + """Allocate GPU and CPU KV cache with the specified number of blocks.
159 +
160 + This also warms up the model, which may record CUDA graphs.
161 + """
162 + raise_if_cache_size_invalid(num_gpu_blocks,
163 + self.cache_config.block_size,
164 + self.model_config.max_model_len)
165 +
166 + self.cache_config.num_gpu_blocks = num_gpu_blocks
167 + self.cache_config.num_cpu_blocks = num_cpu_blocks
168 +
169 + self._init_cache_engine()
170 + self._warm_up_model()
171 +
172 + def _init_cache_engine(self):
173 + assert self.cache_config.num_gpu_blocks is not None
174 self.cache_engine = CacheEngine(self.cache_config, self.model_config,
175 self.parallel_config)
176 self.gpu_cache = self.cache_engine.gpu_cache
177 self.model_runner.set_block_size(self.cache_engine.block_size)
178
165 - def warm_up_model(self) -> None:
179 + def _warm_up_model(self) -> None:
180 if not self.model_config.enforce_eager:
181 self.model_runner.capture_model(self.gpu_cache)
182 # Reset the seed to ensure that the random state is not affected by
@@ -239,11 +253,10 @@ class Worker:
253 def vocab_size(self) -> int:
254 return self.model_runner.vocab_size
255
242 - def get_cache_block_size_bytes(self, block_size: int,
243 - cache_dtype: str) -> int:
256 + def get_cache_block_size_bytes(self) -> int:
257 """Get the size of the KV cache block size in bytes.
258 """
246 - return CacheEngine.get_cache_block_size(block_size, cache_dtype,
259 + return CacheEngine.get_cache_block_size(self.cache_config,
260 self.model_config,
261 self.parallel_config)
262
@@ -300,3 +313,19 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
313 f"{compute_capability[0]}.{compute_capability[1]}. "
314 "You can use float16 instead by explicitly setting the"
315 "`dtype` flag in CLI, for example: --dtype=half.")
316 +
317 +
318 +def raise_if_cache_size_invalid(num_gpu_blocks, block_size,
319 + max_model_len) -> None:
320 + if num_gpu_blocks <= 0:
321 + raise ValueError("No available memory for the cache blocks. "
322 + "Try increasing `gpu_memory_utilization` when "
323 + "initializing the engine.")
324 + max_seq_len = block_size * num_gpu_blocks
325 + if max_model_len > max_seq_len:
326 + raise ValueError(
327 + f"The model's max seq len ({max_model_len}) "
328 + "is larger than the maximum number of tokens that can be "
329 + f"stored in KV cache ({max_seq_len}). Try increasing "
330 + "`gpu_memory_utilization` or decreasing `max_model_len` when "
331 + "initializing the engine.")
vllm/worker/worker_base.py new
+83
@@ -0,0 +1,83 @@
1 +from abc import ABC, abstractmethod
2 +from typing import Dict, List
3 +
4 +from vllm.lora.request import LoRARequest
5 +from vllm.sequence import SamplerOutput, SequenceGroupMetadata
6 +
7 +
8 +class WorkerBase(ABC):
9 + """Worker interface that allows vLLM to cleanly separate implementations for
10 + different hardware.
11 + """
12 +
13 + @abstractmethod
14 + def init_device(self) -> None:
15 + """Initialize device state, such as loading the model or other on-device
16 + memory allocations.
17 + """
18 + raise NotImplementedError
19 +
20 + @abstractmethod
21 + def determine_num_available_blocks(self) -> tuple[int, int]:
22 + """Determine the number of available blocks for the GPU KV cache and
23 + swappable CPU KV cache.
24 +
25 + The implementation may run profiling or other heuristics to determine
26 + the size of caches.
27 +
28 + Returns a tuple[num_gpu_blocks, num_cpu_blocks], where num_gpu_blocks
29 + are blocks that are "active" on the device and can be appended to.
30 + num_cpu_blocks refers to "swapped" blocks in CPU memory and cannot be
31 + appended to.
32 + """
33 + raise NotImplementedError
34 +
35 + @abstractmethod
36 + def initialize_cache(self, num_gpu_blocks: int,
37 + num_cpu_blocks: int) -> None:
38 + """Initialize the KV cache with the given size in blocks.
39 + """
40 + raise NotImplementedError
41 +
42 + @abstractmethod
43 + def execute_model(self,
44 + seq_group_metadata_list: List[SequenceGroupMetadata],
45 + blocks_to_swap_in: Dict[int, int],
46 + blocks_to_swap_out: Dict[int, int],
47 + blocks_to_copy: Dict[int, List[int]]) -> SamplerOutput:
48 + """Executes one model step on the given sequences."""
49 + raise NotImplementedError
50 +
51 + @abstractmethod
52 + def get_cache_block_size_bytes() -> int:
53 + """Return the size of a single cache block, in bytes. Used in
54 + speculative decoding.
55 + """
56 + raise NotImplementedError
57 +
58 + @abstractmethod
59 + def add_lora(self, lora_request: LoRARequest) -> bool:
60 + raise NotImplementedError
61 +
62 + @abstractmethod
63 + def remove_lora(self, lora_id: int) -> bool:
64 + raise NotImplementedError
65 +
66 + @abstractmethod
67 + def list_loras(self) -> List[int]:
68 + raise NotImplementedError
69 +
70 +
71 +class LoraNotSupportedWorkerBase(WorkerBase):
72 + """Partial implementation of WorkerBase that raises exceptions when LoRA
73 + methods are invoked.
74 + """
75 +
76 + def add_lora(self, lora_request: LoRARequest) -> bool:
77 + raise ValueError(f"{type(self)} does not support LoRA")
78 +
79 + def remove_lora(self, lora_id: int) -> bool:
80 + raise ValueError(f"{type(self)} does not support LoRA")
81 +
82 + def list_loras(self) -> List[int]:
83 + raise ValueError(f"{type(self)} does not support LoRA")