@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf5f000d21
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[torch.compile] Hide KV cache behind torch.compile boundary (#11677)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Jan 10, 2025 at 13:14 UTC cf5f000d218fbcbc4bf404de8ed9d9607a128c3b
18 files changed +198 -44
tests/kernels/test_encoder_decoder_attn.py
+12 -6
@@ -142,12 +142,18 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
142 torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
143
144 # Construct KV cache
145 - kv_cache = make_kv_cache(test_pt.num_blocks,
146 - test_pt.num_heads,
147 - test_pt.head_size,
148 - test_pt.block_size,
149 - device=CUDA_DEVICE,
150 - backend=test_pt.backend_name)
145 + if test_pt.attn_type in (AttentionType.DECODER,
146 + AttentionType.ENCODER_DECODER):
147 + kv_cache = make_kv_cache(test_pt.num_blocks,
148 + test_pt.num_heads,
149 + test_pt.head_size,
150 + test_pt.block_size,
151 + device=CUDA_DEVICE,
152 + backend=test_pt.backend_name)
153 + else:
154 + kv_cache = torch.tensor([])
155 +
156 + attn.kv_cache = [kv_cache]
157 return TestResources(scale, attn, kv_cache)
158
159
tests/test_utils.py
+83 -2
@@ -7,9 +7,11 @@ import pytest
7 import torch
8 from vllm_test_utils import monitor
9
10 +from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
11 from vllm.utils import (FlexibleArgumentParser, PlaceholderModule,
11 - StoreBoolean, deprecate_kwargs, get_open_port,
12 - memory_profiling, merge_async_iterators, supports_kw)
12 + StoreBoolean, bind_kv_cache, deprecate_kwargs,
13 + get_open_port, memory_profiling, merge_async_iterators,
14 + supports_kw)
15
16 from .utils import error_on_warning, fork_new_process_for_each_test
17
@@ -325,6 +327,85 @@ def test_memory_profiling():
327 lib.cudaFree(handle2)
328
329
330 +def test_bind_kv_cache():
331 + from vllm.attention import Attention
332 +
333 + ctx = {
334 + 'layers.0.self_attn': Attention(32, 128, 0.1),
335 + 'layers.1.self_attn': Attention(32, 128, 0.1),
336 + 'layers.2.self_attn': Attention(32, 128, 0.1),
337 + 'layers.3.self_attn': Attention(32, 128, 0.1),
338 + }
339 + kv_cache = [
340 + torch.zeros((1, )),
341 + torch.zeros((1, )),
342 + torch.zeros((1, )),
343 + torch.zeros((1, )),
344 + ]
345 + bind_kv_cache(ctx, [kv_cache])
346 + assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[0]
347 + assert ctx['layers.1.self_attn'].kv_cache[0] is kv_cache[1]
348 + assert ctx['layers.2.self_attn'].kv_cache[0] is kv_cache[2]
349 + assert ctx['layers.3.self_attn'].kv_cache[0] is kv_cache[3]
350 +
351 +def test_bind_kv_cache_non_attention():
352 + from vllm.attention import Attention
353 +
354 + # example from Jamba PP=2
355 + ctx = {
356 + 'model.layers.20.attn': Attention(32, 128, 0.1),
357 + 'model.layers.28.attn': Attention(32, 128, 0.1),
358 + }
359 + kv_cache = [
360 + torch.zeros((1, )),
361 + torch.zeros((1, )),
362 + ]
363 + bind_kv_cache(ctx, [kv_cache])
364 + assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[0]
365 + assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[1]
366 +
367 +
368 +def test_bind_kv_cache_encoder_decoder():
369 + from vllm.attention import Attention, AttentionType
370 +
371 + # example from bart
372 + ctx = {
373 + 'encoder.layers.0.self_attn.attn':
374 + Attention(32, 128, 0.1, attn_type=AttentionType.ENCODER),
375 + 'decoder.layers.0.encoder_attn.attn':
376 + Attention(32, 128, 0.1, attn_type=AttentionType.ENCODER_DECODER),
377 + 'decoder.layers.0.self_attn.attn':
378 + Attention(32, 128, 0.1, attn_type=AttentionType.DECODER),
379 + }
380 +
381 + kv_cache = [
382 + torch.zeros((1, )),
383 + ]
384 + encoder_kv_cache = ctx['encoder.layers.0.self_attn.attn'].kv_cache
385 +
386 + bind_kv_cache(ctx, [kv_cache])
387 + assert ctx['encoder.layers.0.self_attn.attn'].kv_cache is encoder_kv_cache
388 + assert ctx['decoder.layers.0.encoder_attn.attn'].kv_cache[0] is kv_cache[0]
389 + assert ctx['decoder.layers.0.self_attn.attn'].kv_cache[0] is kv_cache[0]
390 +
391 +
392 +def test_bind_kv_cache_pp():
393 + cfg = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=2))
394 + with set_current_vllm_config(cfg):
395 + from vllm.attention import Attention
396 +
397 + ctx = {
398 + 'layers.0.self_attn': Attention(32, 128, 0.1),
399 + }
400 + kv_cache = [
401 + [torch.zeros((1, ))],
402 + [torch.zeros((1, ))]
403 + ]
404 + bind_kv_cache(ctx, kv_cache)
405 + assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[0][0]
406 + assert ctx['layers.0.self_attn'].kv_cache[1] is kv_cache[1][0]
407 +
408 +
409 def test_placeholder_module_error_handling():
410 placeholder = PlaceholderModule("placeholder_1234")
411
tests/v1/engine/test_engine_core.py
+3
@@ -4,6 +4,7 @@ import uuid
4 import pytest
5 from transformers import AutoTokenizer
6
7 +from tests.utils import fork_new_process_for_each_test
8 from vllm import SamplingParams
9 from vllm.engine.arg_utils import EngineArgs
10 from vllm.platforms import current_platform
@@ -36,6 +37,7 @@ def make_request() -> EngineCoreRequest:
37 )
38
39
40 +@fork_new_process_for_each_test
41 def test_engine_core(monkeypatch):
42
43 with monkeypatch.context() as m:
@@ -138,6 +140,7 @@ def test_engine_core(monkeypatch):
140 assert len(engine_core.scheduler.running) == 0
141
142
143 +@fork_new_process_for_each_test
144 def test_engine_core_advanced_sampling(monkeypatch):
145 """
146 A basic end-to-end test to verify that the engine functions correctly
tests/v1/engine/test_engine_core_client.py
+3
@@ -6,6 +6,7 @@ from typing import Dict, List
6 import pytest
7 from transformers import AutoTokenizer
8
9 +from tests.utils import fork_new_process_for_each_test
10 from vllm import SamplingParams
11 from vllm.engine.arg_utils import EngineArgs
12 from vllm.platforms import current_platform
@@ -75,6 +76,7 @@ async def loop_until_done_async(client: EngineCoreClient, outputs: Dict):
76 break
77
78
79 +@fork_new_process_for_each_test
80 @pytest.mark.parametrize("multiprocessing_mode", [True, False])
81 def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
82
@@ -143,6 +145,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
145 client.abort_requests([request.request_id])
146
147
148 +@fork_new_process_for_each_test
149 @pytest.mark.asyncio
150 async def test_engine_core_client_asyncio(monkeypatch):
151
vllm/attention/layer.py
+17 -12
@@ -121,6 +121,13 @@ class Attention(nn.Module):
121 compilation_config.static_forward_context[prefix] = self
122 self.layer_name = prefix
123 self.attn_type = attn_type
124 + # use a placeholder kv cache tensor during init, which will be replaced
125 + # by bind_kv_cache
126 + # this variable will not be accessed if use_direct_call is True
127 + self.kv_cache = [
128 + torch.tensor([]) for _ in range(get_current_vllm_config(
129 + ).parallel_config.pipeline_parallel_size)
130 + ]
131
132 def forward(
133 self,
@@ -148,11 +155,11 @@ class Attention(nn.Module):
155 if value is not None:
156 value = value.view(-1, self.num_kv_heads, self.head_size)
157 torch.ops.vllm.unified_attention_with_output(
151 - query, key, value, output, kv_cache, self.layer_name)
158 + query, key, value, output, self.layer_name)
159 return output.view(-1, hidden_size)
160 else:
161 return torch.ops.vllm.unified_attention(query, key, value,
155 - kv_cache, self.layer_name)
162 + self.layer_name)
163
164 def extra_repr(self) -> str:
165 s = f"head_size={self.impl.head_size}" # type: ignore
@@ -230,12 +237,12 @@ def unified_attention(
237 query: torch.Tensor,
238 key: torch.Tensor,
239 value: torch.Tensor,
233 - kv_cache: torch.Tensor,
240 layer_name: str,
241 ) -> torch.Tensor:
242 forward_context: ForwardContext = get_forward_context()
237 - attn_metadata = forward_context.dynamic_forward_context
238 - self = forward_context.static_forward_context[layer_name]
243 + attn_metadata = forward_context.attn_metadata
244 + self = forward_context.attn_layers[layer_name]
245 + kv_cache = self.kv_cache[forward_context.virtual_engine]
246 return self.impl.forward(query, key, value, kv_cache, attn_metadata,
247 self._k_scale, self._v_scale)
248
@@ -244,7 +251,6 @@ def unified_attention_fake(
251 query: torch.Tensor,
252 key: torch.Tensor,
253 value: torch.Tensor,
247 - kv_cache: torch.Tensor,
254 layer_name: str,
255 ) -> torch.Tensor:
256 return torch.empty_like(query).contiguous()
@@ -253,7 +259,7 @@ def unified_attention_fake(
259 direct_register_custom_op(
260 op_name="unified_attention",
261 op_func=unified_attention,
256 - mutates_args=["kv_cache"],
262 + mutates_args=[],
263 fake_impl=unified_attention_fake,
264 dispatch_key=current_platform.dispatch_key,
265 )
@@ -264,12 +270,12 @@ def unified_attention_with_output(
270 key: torch.Tensor,
271 value: torch.Tensor,
272 output: torch.Tensor,
267 - kv_cache: torch.Tensor,
273 layer_name: str,
274 ) -> None:
275 forward_context: ForwardContext = get_forward_context()
271 - attn_metadata = forward_context.dynamic_forward_context
272 - self = forward_context.static_forward_context[layer_name]
276 + attn_metadata = forward_context.attn_metadata
277 + self = forward_context.attn_layers[layer_name]
278 + kv_cache = self.kv_cache[forward_context.virtual_engine]
279 self.impl.forward(query,
280 key,
281 value,
@@ -285,7 +291,6 @@ def unified_attention_with_output_fake(
291 key: torch.Tensor,
292 value: torch.Tensor,
293 output: torch.Tensor,
288 - kv_cache: torch.Tensor,
294 layer_name: str,
295 ) -> None:
296 return
@@ -294,7 +299,7 @@ def unified_attention_with_output_fake(
299 direct_register_custom_op(
300 op_name="unified_attention_with_output",
301 op_func=unified_attention_with_output,
297 - mutates_args=["kv_cache", "output"],
302 + mutates_args=["output"],
303 fake_impl=unified_attention_with_output_fake,
304 dispatch_key=current_platform.dispatch_key,
305 )
vllm/config.py
-1
@@ -2780,7 +2780,6 @@ class CompilationConfig(BaseModel):
2780 compilation_time: float = PrivateAttr
2781
2782 # Per-model forward context
2783 - # Mainly used to store attention cls
2783 # Map from layer name to the attention cls
2784 static_forward_context: Dict[str, Any] = PrivateAttr
2785
vllm/forward_context.py
+20 -13
@@ -2,7 +2,7 @@ import time
2 from collections import defaultdict
3 from contextlib import contextmanager
4 from dataclasses import dataclass
5 -from typing import Any, Dict, Optional
5 +from typing import TYPE_CHECKING, Any, Dict, Optional
6
7 import torch
8
@@ -10,6 +10,9 @@ import vllm.envs as envs
10 from vllm.config import VllmConfig
11 from vllm.logger import init_logger
12
13 +if TYPE_CHECKING:
14 + from vllm.attention.backends.abstract import AttentionMetadata
15 +
16 logger = init_logger(__name__)
17
18 track_batchsize: bool = envs.VLLM_LOG_BATCHSIZE_INTERVAL >= 0
@@ -21,9 +24,12 @@ batchsize_forward_time: defaultdict = defaultdict(list)
24
25 @dataclass
26 class ForwardContext:
24 - static_forward_context: Dict[str, Any]
27 + # copy from vllm_config.compilation_config.static_forward_context
28 + attn_layers: Dict[str, Any]
29 # TODO: extend to support per-layer dynamic forward context
26 - dynamic_forward_context: Any
30 + attn_metadata: "AttentionMetadata" # set dynamically for each forward pass
31 + # TODO: remove after making all virtual_engines share the same kv cache
32 + virtual_engine: int # set dynamically for each forward pass
33
34
35 _forward_context: Optional[ForwardContext] = None
@@ -38,34 +44,35 @@ def get_forward_context() -> ForwardContext:
44
45
46 @contextmanager
41 -def set_forward_context(context: Any, vllm_config: VllmConfig):
47 +def set_forward_context(attn_metadata: Any,
48 + vllm_config: VllmConfig,
49 + virtual_engine: int = 0):
50 """A context manager that stores the current forward context,
51 can be attention metadata, etc.
52 Here we can inject common logic for every model forward pass.
53 """
54 global forward_start_time
47 - need_to_track_batchsize = track_batchsize and context is not None
55 + need_to_track_batchsize = track_batchsize and attn_metadata is not None
56 if need_to_track_batchsize:
57 forward_start_time = time.perf_counter()
58 global _forward_context
59 prev_context = _forward_context
60 _forward_context = ForwardContext(
53 - static_forward_context=vllm_config.compilation_config.
54 - static_forward_context,
55 - dynamic_forward_context=context)
61 + attn_layers=vllm_config.compilation_config.static_forward_context,
62 + virtual_engine=virtual_engine,
63 + attn_metadata=attn_metadata)
64 try:
65 yield
66 finally:
59 - global batchsize_counter
67 global last_logging_time, batchsize_logging_interval
68 if need_to_track_batchsize:
62 - if hasattr(context, "num_prefill_tokens"):
69 + if hasattr(attn_metadata, "num_prefill_tokens"):
70 # for v0 attention backends
64 - batchsize = context.num_prefill_tokens + \
65 - context.num_decode_tokens
71 + batchsize = attn_metadata.num_prefill_tokens + \
72 + attn_metadata.num_decode_tokens
73 else:
74 # for v1 attention backends
68 - batchsize = context.num_input_tokens
75 + batchsize = attn_metadata.num_input_tokens
76 # we use synchronous scheduling right now,
77 # adding a sync point here should not affect
78 # scheduling of the next batch
vllm/utils.py
+35
@@ -2138,3 +2138,38 @@ def get_mp_context():
2138 _check_multiproc_method()
2139 mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
2140 return multiprocessing.get_context(mp_method)
2141 +
2142 +
2143 +def bind_kv_cache(
2144 + ctx: Dict[str, Any],
2145 + kv_cache: List[List[torch.Tensor]], # [virtual_engine][layer_index]
2146 +) -> None:
2147 + # Bind the kv_cache tensor to Attention modules, similar to
2148 + # ctx[layer_name].kv_cache[ve]=kv_cache[ve][extract_layer_index(layer_name)]
2149 + # Special things handled here:
2150 + # 1. Some models have non-attention layers, e.g., Jamba
2151 + # 2. Pipeline parallelism, each rank only has a subset of layers
2152 + # 3. Encoder attention has no kv cache
2153 + # 4. Encoder-decoder models, encoder-decoder attention and decoder-only
2154 + # attention of the same layer (e.g., bart's decoder.layers.1.self_attn
2155 + # and decoder.layers.1.encoder_attn) is mapped to the same kv cache
2156 + # tensor
2157 + from vllm.attention import AttentionType
2158 + from vllm.model_executor.models.utils import extract_layer_index
2159 + layer_need_kv_cache = [
2160 + layer_name for layer_name in ctx
2161 + if ctx[layer_name].attn_type in (AttentionType.DECODER,
2162 + AttentionType.ENCODER_DECODER)
2163 + ]
2164 + layer_index_sorted = sorted(
2165 + set(
2166 + extract_layer_index(layer_name)
2167 + for layer_name in layer_need_kv_cache))
2168 + for layer_name in layer_need_kv_cache:
2169 + kv_cache_idx = layer_index_sorted.index(
2170 + extract_layer_index(layer_name))
2171 + forward_ctx = ctx[layer_name]
2172 + assert len(forward_ctx.kv_cache) == len(kv_cache)
2173 + for ve, ve_kv_cache in enumerate(kv_cache):
2174 + assert forward_ctx.kv_cache[ve].numel() == 0
2175 + forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]
vllm/v1/worker/gpu_model_runner.py
+5 -1
@@ -16,7 +16,8 @@ from vllm.model_executor.model_loader import get_model
16 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
17 from vllm.sampling_params import SamplingType
18 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19 - LayerBlockType, cdiv, is_pin_memory_available)
19 + LayerBlockType, bind_kv_cache, cdiv,
20 + is_pin_memory_available)
21 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
22 FlashAttentionMetadata)
23 from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
@@ -860,3 +861,6 @@ class GPUModelRunner:
861 torch.zeros(kv_cache_shape,
862 dtype=self.kv_cache_dtype,
863 device=self.device))
864 + bind_kv_cache(
865 + self.vllm_config.compilation_config.static_forward_context,
866 + [self.kv_caches])
vllm/worker/cpu_enc_dec_model_runner.py
+2 -1
@@ -305,7 +305,8 @@ class CPUEncoderDecoderModelRunner(
305 intermediate_tensors,
306 }
307
308 - with set_forward_context(model_input.attn_metadata, self.vllm_config):
308 + with set_forward_context(model_input.attn_metadata, self.vllm_config,
309 + model_input.virtual_engine):
310 hidden_states = model_executable(**execute_model_kwargs)
311
312 # Compute the logits.
vllm/worker/cpu_model_runner.py
+2 -1
@@ -526,7 +526,8 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
526 execute_model_kwargs.update(
527 {"previous_hidden_states": previous_hidden_states})
528
529 - with set_forward_context(model_input.attn_metadata, self.vllm_config):
529 + with set_forward_context(model_input.attn_metadata, self.vllm_config,
530 + model_input.virtual_engine):
531 hidden_states = model_executable(
532 input_ids=model_input.input_tokens,
533 positions=model_input.input_positions,
vllm/worker/cpu_pooling_model_runner.py
+2 -1
@@ -69,7 +69,8 @@ class CPUPoolingModelRunner(
69 intermediate_tensors,
70 }
71
72 - with set_forward_context(model_input.attn_metadata, self.vllm_config):
72 + with set_forward_context(model_input.attn_metadata, self.vllm_config,
73 + model_input.virtual_engine):
74 hidden_states = model_executable(**execute_model_kwargs)
75
76 # Only perform pooling in the driver worker.
vllm/worker/cpu_worker.py
+3 -1
@@ -13,7 +13,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
13 from vllm.logger import init_logger
14 from vllm.model_executor import set_random_seed
15 from vllm.sequence import ExecuteModelRequest
16 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
16 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, bind_kv_cache
17 from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18 from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
19 from vllm.worker.cpu_pooling_model_runner import CPUPoolingModelRunner
@@ -293,6 +293,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
293 self.cache_engine[ve].cpu_cache
294 for ve in range(self.parallel_config.pipeline_parallel_size)
295 ]
296 + bind_kv_cache(self.compilation_config.static_forward_context,
297 + self.cpu_cache)
298 self.model_runner.block_size = self.cache_engine[0].block_size
299
300 assert all(
vllm/worker/enc_dec_model_runner.py
+2 -1
@@ -175,7 +175,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
175 } if self.has_inner_state else {}
176
177 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
178 - with set_forward_context(model_input.attn_metadata, self.vllm_config):
178 + with set_forward_context(model_input.attn_metadata, self.vllm_config,
179 + model_input.virtual_engine):
180 hidden_or_intermediate_states = model_executable(
181 input_ids=model_input.input_tokens,
182 positions=model_input.input_positions,
vllm/worker/model_runner.py
+3 -2
@@ -1527,7 +1527,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1527 self._update_inputs_to_capture_for_enc_dec_model(
1528 capture_inputs)
1529
1530 - with set_forward_context(attn_metadata, self.vllm_config):
1530 + with set_forward_context(attn_metadata, self.vllm_config,
1531 + virtual_engine):
1532 graph_runner.capture(**capture_inputs)
1533 self.graph_memory_pool = graph_runner.graph.pool()
1534 self.graph_runners[virtual_engine][batch_size] = (
@@ -1695,7 +1696,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1696
1697 if not bypass_model_exec:
1698 with set_forward_context(model_input.attn_metadata,
1698 - self.vllm_config):
1699 + self.vllm_config, virtual_engine):
1700 hidden_or_intermediate_states = model_executable(
1701 input_ids=model_input.input_tokens,
1702 positions=model_input.input_positions,
vllm/worker/pooling_model_runner.py
+2 -1
@@ -105,7 +105,8 @@ class PoolingModelRunner(
105 if model_input.token_types is not None:
106 cross_enc_kwargs["token_type_ids"] = model_input.token_types
107
108 - with set_forward_context(model_input.attn_metadata, self.vllm_config):
108 + with set_forward_context(model_input.attn_metadata, self.vllm_config,
109 + virtual_engine):
110 hidden_or_intermediate_states = model_executable(
111 input_ids=model_input.input_tokens,
112 positions=model_input.input_positions,
vllm/worker/worker.py
+3 -1
@@ -21,7 +21,7 @@ from vllm.platforms import current_platform
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
23 SequenceGroupMetadata, SequenceGroupMetadataDelta)
24 -from vllm.utils import GiB_bytes, memory_profiling
24 +from vllm.utils import GiB_bytes, bind_kv_cache, memory_profiling
25 from vllm.worker.cache_engine import CacheEngine
26 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
27 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -285,6 +285,8 @@ class Worker(LocalOrDistributedWorkerBase):
285 self.cache_engine[ve].gpu_cache
286 for ve in range(self.parallel_config.pipeline_parallel_size)
287 ]
288 + bind_kv_cache(self.compilation_config.static_forward_context,
289 + self.gpu_cache)
290
291 def _warm_up_model(self) -> None:
292 if not self.model_config.enforce_eager:
vllm/worker/worker_base.py
+1
@@ -43,6 +43,7 @@ class WorkerBase(ABC):
43 self.prompt_adapter_config = vllm_config.prompt_adapter_config
44 self.observability_config = vllm_config.observability_config
45 self.kv_transfer_config = vllm_config.kv_transfer_config
46 + self.compilation_config = vllm_config.compilation_config
47 from vllm.platforms import current_platform
48 self.current_platform = current_platform
49