A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[torch.compile] Hide KV cache behind torch.compile boundary (#11677)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Jan 10, 2025 at 13:14 UTC
cf5f000d218fbcbc4bf404de8ed9d9607a128c3b
18 files changed
+198
-44
tests/kernels/test_encoder_decoder_attn.py
+12
-6
@@ -142,12 +142,18 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
142
torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
143
144
# Construct KV cache
145
- kv_cache = make_kv_cache(test_pt.num_blocks,
146
- test_pt.num_heads,
147
- test_pt.head_size,
148
- test_pt.block_size,
149
- device=CUDA_DEVICE,
150
- backend=test_pt.backend_name)
145
+ if test_pt.attn_type in (AttentionType.DECODER,
146
+ AttentionType.ENCODER_DECODER):
147
+ kv_cache = make_kv_cache(test_pt.num_blocks,
148
+ test_pt.num_heads,
149
+ test_pt.head_size,
150
+ test_pt.block_size,
151
+ device=CUDA_DEVICE,
152
+ backend=test_pt.backend_name)
153
+ else:
154
+ kv_cache = torch.tensor([])
155
+
156
+ attn.kv_cache = [kv_cache]
157
return TestResources(scale, attn, kv_cache)
158
159
tests/test_utils.py
+83
-2
@@ -7,9 +7,11 @@ import pytest
7
import torch
8
from vllm_test_utils import monitor
9
10
+from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
11
from vllm.utils import (FlexibleArgumentParser, PlaceholderModule,
11
- StoreBoolean, deprecate_kwargs, get_open_port,
12
- memory_profiling, merge_async_iterators, supports_kw)
12
+ StoreBoolean, bind_kv_cache, deprecate_kwargs,
13
+ get_open_port, memory_profiling, merge_async_iterators,
14
+ supports_kw)
15
16
from .utils import error_on_warning, fork_new_process_for_each_test
17
@@ -325,6 +327,85 @@ def test_memory_profiling():
327
lib.cudaFree(handle2)
328
329
330
+def test_bind_kv_cache():
331
+ from vllm.attention import Attention
332
+
333
+ ctx = {
334
+ 'layers.0.self_attn': Attention(32, 128, 0.1),
335
+ 'layers.1.self_attn': Attention(32, 128, 0.1),
336
+ 'layers.2.self_attn': Attention(32, 128, 0.1),
337
+ 'layers.3.self_attn': Attention(32, 128, 0.1),
338
+ }
339
+ kv_cache = [
340
+ torch.zeros((1, )),
341
+ torch.zeros((1, )),
342
+ torch.zeros((1, )),
343
+ torch.zeros((1, )),
344
+ ]
345
+ bind_kv_cache(ctx, [kv_cache])
346
+ assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[0]
347
+ assert ctx['layers.1.self_attn'].kv_cache[0] is kv_cache[1]
348
+ assert ctx['layers.2.self_attn'].kv_cache[0] is kv_cache[2]
349
+ assert ctx['layers.3.self_attn'].kv_cache[0] is kv_cache[3]
350
+
351
+def test_bind_kv_cache_non_attention():
352
+ from vllm.attention import Attention
353
+
354
+ # example from Jamba PP=2
355
+ ctx = {
356
+ 'model.layers.20.attn': Attention(32, 128, 0.1),
357
+ 'model.layers.28.attn': Attention(32, 128, 0.1),
358
+ }
359
+ kv_cache = [
360
+ torch.zeros((1, )),
361
+ torch.zeros((1, )),
362
+ ]
363
+ bind_kv_cache(ctx, [kv_cache])
364
+ assert ctx['model.layers.20.attn'].kv_cache[0] is kv_cache[0]
365
+ assert ctx['model.layers.28.attn'].kv_cache[0] is kv_cache[1]
366
+
367
+
368
+def test_bind_kv_cache_encoder_decoder():
369
+ from vllm.attention import Attention, AttentionType
370
+
371
+ # example from bart
372
+ ctx = {
373
+ 'encoder.layers.0.self_attn.attn':
374
+ Attention(32, 128, 0.1, attn_type=AttentionType.ENCODER),
375
+ 'decoder.layers.0.encoder_attn.attn':
376
+ Attention(32, 128, 0.1, attn_type=AttentionType.ENCODER_DECODER),
377
+ 'decoder.layers.0.self_attn.attn':
378
+ Attention(32, 128, 0.1, attn_type=AttentionType.DECODER),
379
+ }
380
+
381
+ kv_cache = [
382
+ torch.zeros((1, )),
383
+ ]
384
+ encoder_kv_cache = ctx['encoder.layers.0.self_attn.attn'].kv_cache
385
+
386
+ bind_kv_cache(ctx, [kv_cache])
387
+ assert ctx['encoder.layers.0.self_attn.attn'].kv_cache is encoder_kv_cache
388
+ assert ctx['decoder.layers.0.encoder_attn.attn'].kv_cache[0] is kv_cache[0]
389
+ assert ctx['decoder.layers.0.self_attn.attn'].kv_cache[0] is kv_cache[0]
390
+
391
+
392
+def test_bind_kv_cache_pp():
393
+ cfg = VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=2))
394
+ with set_current_vllm_config(cfg):
395
+ from vllm.attention import Attention
396
+
397
+ ctx = {
398
+ 'layers.0.self_attn': Attention(32, 128, 0.1),
399
+ }
400
+ kv_cache = [
401
+ [torch.zeros((1, ))],
402
+ [torch.zeros((1, ))]
403
+ ]
404
+ bind_kv_cache(ctx, kv_cache)
405
+ assert ctx['layers.0.self_attn'].kv_cache[0] is kv_cache[0][0]
406
+ assert ctx['layers.0.self_attn'].kv_cache[1] is kv_cache[1][0]
407
+
408
+
409
def test_placeholder_module_error_handling():
410
placeholder = PlaceholderModule("placeholder_1234")
411
tests/v1/engine/test_engine_core.py
+3
@@ -4,6 +4,7 @@ import uuid
4
import pytest
5
from transformers import AutoTokenizer
6
7
+from tests.utils import fork_new_process_for_each_test
8
from vllm import SamplingParams
9
from vllm.engine.arg_utils import EngineArgs
10
from vllm.platforms import current_platform
@@ -36,6 +37,7 @@ def make_request() -> EngineCoreRequest:
37
)
38
39
40
+@fork_new_process_for_each_test
41
def test_engine_core(monkeypatch):
42
43
with monkeypatch.context() as m:
@@ -138,6 +140,7 @@ def test_engine_core(monkeypatch):
140
assert len(engine_core.scheduler.running) == 0
141
142
143
+@fork_new_process_for_each_test
144
def test_engine_core_advanced_sampling(monkeypatch):
145
"""
146
A basic end-to-end test to verify that the engine functions correctly
tests/v1/engine/test_engine_core_client.py
+3
@@ -6,6 +6,7 @@ from typing import Dict, List
6
import pytest
7
from transformers import AutoTokenizer
8
9
+from tests.utils import fork_new_process_for_each_test
10
from vllm import SamplingParams
11
from vllm.engine.arg_utils import EngineArgs
12
from vllm.platforms import current_platform
@@ -75,6 +76,7 @@ async def loop_until_done_async(client: EngineCoreClient, outputs: Dict):
76
break
77
78
79
+@fork_new_process_for_each_test
80
@pytest.mark.parametrize("multiprocessing_mode", [True, False])
81
def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
82
@@ -143,6 +145,7 @@ def test_engine_core_client(monkeypatch, multiprocessing_mode: bool):
145
client.abort_requests([request.request_id])
146
147
148
+@fork_new_process_for_each_test
149
@pytest.mark.asyncio
150
async def test_engine_core_client_asyncio(monkeypatch):
151
vllm/attention/layer.py
+17
-12
@@ -121,6 +121,13 @@ class Attention(nn.Module):
121
compilation_config.static_forward_context[prefix] = self
122
self.layer_name = prefix
123
self.attn_type = attn_type
124
+ # use a placeholder kv cache tensor during init, which will be replaced
125
+ # by bind_kv_cache
126
+ # this variable will not be accessed if use_direct_call is True
127
+ self.kv_cache = [
128
+ torch.tensor([]) for _ in range(get_current_vllm_config(
129
+ ).parallel_config.pipeline_parallel_size)
130
+ ]
131
132
def forward(
133
self,
@@ -148,11 +155,11 @@ class Attention(nn.Module):
155
if value is not None:
156
value = value.view(-1, self.num_kv_heads, self.head_size)
157
torch.ops.vllm.unified_attention_with_output(
151
- query, key, value, output, kv_cache, self.layer_name)
158
+ query, key, value, output, self.layer_name)
159
return output.view(-1, hidden_size)
160
else:
161
return torch.ops.vllm.unified_attention(query, key, value,
155
- kv_cache, self.layer_name)
162
+ self.layer_name)
163
164
def extra_repr(self) -> str:
165
s = f"head_size={self.impl.head_size}" # type: ignore
@@ -230,12 +237,12 @@ def unified_attention(
237
query: torch.Tensor,
238
key: torch.Tensor,
239
value: torch.Tensor,
233
- kv_cache: torch.Tensor,
240
layer_name: str,
241
) -> torch.Tensor:
242
forward_context: ForwardContext = get_forward_context()
237
- attn_metadata = forward_context.dynamic_forward_context
238
- self = forward_context.static_forward_context[layer_name]
243
+ attn_metadata = forward_context.attn_metadata
244
+ self = forward_context.attn_layers[layer_name]
245
+ kv_cache = self.kv_cache[forward_context.virtual_engine]
246
return self.impl.forward(query, key, value, kv_cache, attn_metadata,
247
self._k_scale, self._v_scale)
248
@@ -244,7 +251,6 @@ def unified_attention_fake(
251
query: torch.Tensor,
252
key: torch.Tensor,
253
value: torch.Tensor,
247
- kv_cache: torch.Tensor,
254
layer_name: str,
255
) -> torch.Tensor:
256
return torch.empty_like(query).contiguous()
@@ -253,7 +259,7 @@ def unified_attention_fake(
259
direct_register_custom_op(
260
op_name="unified_attention",
261
op_func=unified_attention,
256
- mutates_args=["kv_cache"],
262
+ mutates_args=[],
263
fake_impl=unified_attention_fake,
264
dispatch_key=current_platform.dispatch_key,
265
)
@@ -264,12 +270,12 @@ def unified_attention_with_output(
270
key: torch.Tensor,
271
value: torch.Tensor,
272
output: torch.Tensor,
267
- kv_cache: torch.Tensor,
273
layer_name: str,
274
) -> None:
275
forward_context: ForwardContext = get_forward_context()
271
- attn_metadata = forward_context.dynamic_forward_context
272
- self = forward_context.static_forward_context[layer_name]
276
+ attn_metadata = forward_context.attn_metadata
277
+ self = forward_context.attn_layers[layer_name]
278
+ kv_cache = self.kv_cache[forward_context.virtual_engine]
279
self.impl.forward(query,
280
key,
281
value,
@@ -285,7 +291,6 @@ def unified_attention_with_output_fake(
291
key: torch.Tensor,
292
value: torch.Tensor,
293
output: torch.Tensor,
288
- kv_cache: torch.Tensor,
294
layer_name: str,
295
) -> None:
296
return
@@ -294,7 +299,7 @@ def unified_attention_with_output_fake(
299
direct_register_custom_op(
300
op_name="unified_attention_with_output",
301
op_func=unified_attention_with_output,
297
- mutates_args=["kv_cache", "output"],
302
+ mutates_args=["output"],
303
fake_impl=unified_attention_with_output_fake,
304
dispatch_key=current_platform.dispatch_key,
305
)
vllm/config.py
-1
@@ -2780,7 +2780,6 @@ class CompilationConfig(BaseModel):
2780
compilation_time: float = PrivateAttr
2781
2782
# Per-model forward context
2783
- # Mainly used to store attention cls
2783
# Map from layer name to the attention cls
2784
static_forward_context: Dict[str, Any] = PrivateAttr
2785
vllm/forward_context.py
+20
-13
@@ -2,7 +2,7 @@ import time
2
from collections import defaultdict
3
from contextlib import contextmanager
4
from dataclasses import dataclass
5
-from typing import Any, Dict, Optional
5
+from typing import TYPE_CHECKING, Any, Dict, Optional
6
7
import torch
8
@@ -10,6 +10,9 @@ import vllm.envs as envs
10
from vllm.config import VllmConfig
11
from vllm.logger import init_logger
12
13
+if TYPE_CHECKING:
14
+ from vllm.attention.backends.abstract import AttentionMetadata
15
+
16
logger = init_logger(__name__)
17
18
track_batchsize: bool = envs.VLLM_LOG_BATCHSIZE_INTERVAL >= 0
@@ -21,9 +24,12 @@ batchsize_forward_time: defaultdict = defaultdict(list)
24
25
@dataclass
26
class ForwardContext:
24
- static_forward_context: Dict[str, Any]
27
+ # copy from vllm_config.compilation_config.static_forward_context
28
+ attn_layers: Dict[str, Any]
29
# TODO: extend to support per-layer dynamic forward context
26
- dynamic_forward_context: Any
30
+ attn_metadata: "AttentionMetadata" # set dynamically for each forward pass
31
+ # TODO: remove after making all virtual_engines share the same kv cache
32
+ virtual_engine: int # set dynamically for each forward pass
33
34
35
_forward_context: Optional[ForwardContext] = None
@@ -38,34 +44,35 @@ def get_forward_context() -> ForwardContext:
44
45
46
@contextmanager
41
-def set_forward_context(context: Any, vllm_config: VllmConfig):
47
+def set_forward_context(attn_metadata: Any,
48
+ vllm_config: VllmConfig,
49
+ virtual_engine: int = 0):
50
"""A context manager that stores the current forward context,
51
can be attention metadata, etc.
52
Here we can inject common logic for every model forward pass.
53
"""
54
global forward_start_time
47
- need_to_track_batchsize = track_batchsize and context is not None
55
+ need_to_track_batchsize = track_batchsize and attn_metadata is not None
56
if need_to_track_batchsize:
57
forward_start_time = time.perf_counter()
58
global _forward_context
59
prev_context = _forward_context
60
_forward_context = ForwardContext(
53
- static_forward_context=vllm_config.compilation_config.
54
- static_forward_context,
55
- dynamic_forward_context=context)
61
+ attn_layers=vllm_config.compilation_config.static_forward_context,
62
+ virtual_engine=virtual_engine,
63
+ attn_metadata=attn_metadata)
64
try:
65
yield
66
finally:
59
- global batchsize_counter
67
global last_logging_time, batchsize_logging_interval
68
if need_to_track_batchsize:
62
- if hasattr(context, "num_prefill_tokens"):
69
+ if hasattr(attn_metadata, "num_prefill_tokens"):
70
# for v0 attention backends
64
- batchsize = context.num_prefill_tokens + \
65
- context.num_decode_tokens
71
+ batchsize = attn_metadata.num_prefill_tokens + \
72
+ attn_metadata.num_decode_tokens
73
else:
74
# for v1 attention backends
68
- batchsize = context.num_input_tokens
75
+ batchsize = attn_metadata.num_input_tokens
76
# we use synchronous scheduling right now,
77
# adding a sync point here should not affect
78
# scheduling of the next batch
vllm/utils.py
+35
@@ -2138,3 +2138,38 @@ def get_mp_context():
2138
_check_multiproc_method()
2139
mp_method = envs.VLLM_WORKER_MULTIPROC_METHOD
2140
return multiprocessing.get_context(mp_method)
2141
+
2142
+
2143
+def bind_kv_cache(
2144
+ ctx: Dict[str, Any],
2145
+ kv_cache: List[List[torch.Tensor]], # [virtual_engine][layer_index]
2146
+) -> None:
2147
+ # Bind the kv_cache tensor to Attention modules, similar to
2148
+ # ctx[layer_name].kv_cache[ve]=kv_cache[ve][extract_layer_index(layer_name)]
2149
+ # Special things handled here:
2150
+ # 1. Some models have non-attention layers, e.g., Jamba
2151
+ # 2. Pipeline parallelism, each rank only has a subset of layers
2152
+ # 3. Encoder attention has no kv cache
2153
+ # 4. Encoder-decoder models, encoder-decoder attention and decoder-only
2154
+ # attention of the same layer (e.g., bart's decoder.layers.1.self_attn
2155
+ # and decoder.layers.1.encoder_attn) is mapped to the same kv cache
2156
+ # tensor
2157
+ from vllm.attention import AttentionType
2158
+ from vllm.model_executor.models.utils import extract_layer_index
2159
+ layer_need_kv_cache = [
2160
+ layer_name for layer_name in ctx
2161
+ if ctx[layer_name].attn_type in (AttentionType.DECODER,
2162
+ AttentionType.ENCODER_DECODER)
2163
+ ]
2164
+ layer_index_sorted = sorted(
2165
+ set(
2166
+ extract_layer_index(layer_name)
2167
+ for layer_name in layer_need_kv_cache))
2168
+ for layer_name in layer_need_kv_cache:
2169
+ kv_cache_idx = layer_index_sorted.index(
2170
+ extract_layer_index(layer_name))
2171
+ forward_ctx = ctx[layer_name]
2172
+ assert len(forward_ctx.kv_cache) == len(kv_cache)
2173
+ for ve, ve_kv_cache in enumerate(kv_cache):
2174
+ assert forward_ctx.kv_cache[ve].numel() == 0
2175
+ forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]
vllm/v1/worker/gpu_model_runner.py
+5
-1
@@ -16,7 +16,8 @@ from vllm.model_executor.model_loader import get_model
16
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
17
from vllm.sampling_params import SamplingType
18
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19
- LayerBlockType, cdiv, is_pin_memory_available)
19
+ LayerBlockType, bind_kv_cache, cdiv,
20
+ is_pin_memory_available)
21
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
22
FlashAttentionMetadata)
23
from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
@@ -860,3 +861,6 @@ class GPUModelRunner:
861
torch.zeros(kv_cache_shape,
862
dtype=self.kv_cache_dtype,
863
device=self.device))
864
+ bind_kv_cache(
865
+ self.vllm_config.compilation_config.static_forward_context,
866
+ [self.kv_caches])
vllm/worker/cpu_enc_dec_model_runner.py
+2
-1
@@ -305,7 +305,8 @@ class CPUEncoderDecoderModelRunner(
305
intermediate_tensors,
306
}
307
308
- with set_forward_context(model_input.attn_metadata, self.vllm_config):
308
+ with set_forward_context(model_input.attn_metadata, self.vllm_config,
309
+ model_input.virtual_engine):
310
hidden_states = model_executable(**execute_model_kwargs)
311
312
# Compute the logits.
vllm/worker/cpu_model_runner.py
+2
-1
@@ -526,7 +526,8 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
526
execute_model_kwargs.update(
527
{"previous_hidden_states": previous_hidden_states})
528
529
- with set_forward_context(model_input.attn_metadata, self.vllm_config):
529
+ with set_forward_context(model_input.attn_metadata, self.vllm_config,
530
+ model_input.virtual_engine):
531
hidden_states = model_executable(
532
input_ids=model_input.input_tokens,
533
positions=model_input.input_positions,
vllm/worker/cpu_pooling_model_runner.py
+2
-1
@@ -69,7 +69,8 @@ class CPUPoolingModelRunner(
69
intermediate_tensors,
70
}
71
72
- with set_forward_context(model_input.attn_metadata, self.vllm_config):
72
+ with set_forward_context(model_input.attn_metadata, self.vllm_config,
73
+ model_input.virtual_engine):
74
hidden_states = model_executable(**execute_model_kwargs)
75
76
# Only perform pooling in the driver worker.
vllm/worker/cpu_worker.py
+3
-1
@@ -13,7 +13,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
13
from vllm.logger import init_logger
14
from vllm.model_executor import set_random_seed
15
from vllm.sequence import ExecuteModelRequest
16
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
16
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, bind_kv_cache
17
from vllm.worker.cpu_enc_dec_model_runner import CPUEncoderDecoderModelRunner
18
from vllm.worker.cpu_model_runner import CPUModelRunner, CPUModelRunnerBase
19
from vllm.worker.cpu_pooling_model_runner import CPUPoolingModelRunner
@@ -293,6 +293,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
293
self.cache_engine[ve].cpu_cache
294
for ve in range(self.parallel_config.pipeline_parallel_size)
295
]
296
+ bind_kv_cache(self.compilation_config.static_forward_context,
297
+ self.cpu_cache)
298
self.model_runner.block_size = self.cache_engine[0].block_size
299
300
assert all(
vllm/worker/enc_dec_model_runner.py
+2
-1
@@ -175,7 +175,8 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
175
} if self.has_inner_state else {}
176
177
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
178
- with set_forward_context(model_input.attn_metadata, self.vllm_config):
178
+ with set_forward_context(model_input.attn_metadata, self.vllm_config,
179
+ model_input.virtual_engine):
180
hidden_or_intermediate_states = model_executable(
181
input_ids=model_input.input_tokens,
182
positions=model_input.input_positions,
vllm/worker/model_runner.py
+3
-2
@@ -1527,7 +1527,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1527
self._update_inputs_to_capture_for_enc_dec_model(
1528
capture_inputs)
1529
1530
- with set_forward_context(attn_metadata, self.vllm_config):
1530
+ with set_forward_context(attn_metadata, self.vllm_config,
1531
+ virtual_engine):
1532
graph_runner.capture(**capture_inputs)
1533
self.graph_memory_pool = graph_runner.graph.pool()
1534
self.graph_runners[virtual_engine][batch_size] = (
@@ -1695,7 +1696,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1696
1697
if not bypass_model_exec:
1698
with set_forward_context(model_input.attn_metadata,
1698
- self.vllm_config):
1699
+ self.vllm_config, virtual_engine):
1700
hidden_or_intermediate_states = model_executable(
1701
input_ids=model_input.input_tokens,
1702
positions=model_input.input_positions,
vllm/worker/pooling_model_runner.py
+2
-1
@@ -105,7 +105,8 @@ class PoolingModelRunner(
105
if model_input.token_types is not None:
106
cross_enc_kwargs["token_type_ids"] = model_input.token_types
107
108
- with set_forward_context(model_input.attn_metadata, self.vllm_config):
108
+ with set_forward_context(model_input.attn_metadata, self.vllm_config,
109
+ virtual_engine):
110
hidden_or_intermediate_states = model_executable(
111
input_ids=model_input.input_tokens,
112
positions=model_input.input_positions,
vllm/worker/worker.py
+3
-1
@@ -21,7 +21,7 @@ from vllm.platforms import current_platform
21
from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
23
SequenceGroupMetadata, SequenceGroupMetadataDelta)
24
-from vllm.utils import GiB_bytes, memory_profiling
24
+from vllm.utils import GiB_bytes, bind_kv_cache, memory_profiling
25
from vllm.worker.cache_engine import CacheEngine
26
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
27
from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -285,6 +285,8 @@ class Worker(LocalOrDistributedWorkerBase):
285
self.cache_engine[ve].gpu_cache
286
for ve in range(self.parallel_config.pipeline_parallel_size)
287
]
288
+ bind_kv_cache(self.compilation_config.static_forward_context,
289
+ self.gpu_cache)
290
291
def _warm_up_model(self) -> None:
292
if not self.model_config.enforce_eager:
vllm/worker/worker_base.py
+1
@@ -43,6 +43,7 @@ class WorkerBase(ABC):
43
self.prompt_adapter_config = vllm_config.prompt_adapter_config
44
self.observability_config = vllm_config.observability_config
45
self.kv_transfer_config = vllm_config.kv_transfer_config
46
+ self.compilation_config = vllm_config.compilation_config
47
from vllm.platforms import current_platform
48
self.current_platform = current_platform
49