A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Kernel] unified_attention for Attention.forward (#11967)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Jan 13, 2025 at 19:28 UTC
0f8cafe2d1550a33803fb64b2224e6bf3f913449
10 files changed
+87
-45
vllm/attention/layer.py
+14
-12
@@ -134,15 +134,10 @@ class Attention(nn.Module):
134
query: torch.Tensor,
135
key: torch.Tensor,
136
value: torch.Tensor,
137
- kv_cache: torch.Tensor,
138
- attn_metadata: AttentionMetadata,
137
+ _kv_cache: torch.Tensor,
138
+ _attn_metadata: AttentionMetadata,
139
) -> torch.Tensor:
140
-
141
- if self.use_direct_call:
142
- return self.impl.forward(query, key, value, kv_cache,
143
- attn_metadata, self._k_scale,
144
- self._v_scale)
145
- elif self.use_output:
140
+ if self.use_output:
141
output = torch.empty_like(query)
142
hidden_size = query.size(-1)
143
# Reshape the query, key, and value tensors.
@@ -154,12 +149,19 @@ class Attention(nn.Module):
149
key = key.view(-1, self.num_kv_heads, self.head_size)
150
if value is not None:
151
value = value.view(-1, self.num_kv_heads, self.head_size)
157
- torch.ops.vllm.unified_attention_with_output(
158
- query, key, value, output, self.layer_name)
152
+ if self.use_direct_call:
153
+ unified_attention_with_output(query, key, value, output,
154
+ self.layer_name)
155
+ else:
156
+ torch.ops.vllm.unified_attention_with_output(
157
+ query, key, value, output, self.layer_name)
158
return output.view(-1, hidden_size)
159
else:
161
- return torch.ops.vllm.unified_attention(query, key, value,
162
- self.layer_name)
160
+ if self.use_direct_call:
161
+ return unified_attention(query, key, value, self.layer_name)
162
+ else:
163
+ return torch.ops.vllm.unified_attention(
164
+ query, key, value, self.layer_name)
165
166
def extra_repr(self) -> str:
167
s = f"head_size={self.impl.head_size}" # type: ignore
vllm/utils.py
-1
@@ -2171,5 +2171,4 @@ def bind_kv_cache(
2171
forward_ctx = ctx[layer_name]
2172
assert len(forward_ctx.kv_cache) == len(kv_cache)
2173
for ve, ve_kv_cache in enumerate(kv_cache):
2174
- assert forward_ctx.kv_cache[ve].numel() == 0
2174
forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]
vllm/worker/hpu_model_runner.py
+11
-2
@@ -28,6 +28,7 @@ from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler,
28
from vllm.attention import AttentionMetadata, get_attn_backend
29
from vllm.config import DeviceConfig, VllmConfig
30
from vllm.distributed.parallel_state import get_world_group
31
+from vllm.forward_context import set_forward_context
32
from vllm.logger import init_logger
33
from vllm.lora.layers import LoRAMapping
34
from vllm.lora.request import LoRARequest
@@ -40,7 +41,8 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
41
from vllm.sampling_params import SamplingParams
42
from vllm.sequence import (IntermediateTensors, SequenceData,
43
SequenceGroupMetadata)
43
-from vllm.utils import is_pin_memory_available, make_tensor_with_pad
44
+from vllm.utils import (bind_kv_cache, is_pin_memory_available,
45
+ make_tensor_with_pad)
46
from vllm.worker.model_runner_base import (
47
ModelRunnerBase, ModelRunnerInputBase,
48
_add_attn_metadata_broadcastable_dict,
@@ -1286,6 +1288,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1288
def profile_run(self) -> None:
1289
num_layers = self.model_config.get_num_layers(self.parallel_config)
1290
kv_caches = [None] * num_layers
1291
+ bind_kv_cache(
1292
+ self.vllm_config.compilation_config.static_forward_context,
1293
+ [kv_caches])
1294
max_seq_len = self.bucketing_global_state.prompt_seq_bucket_cfg[-1]
1295
max_batch_size = min(self.max_num_batched_tokens // max_seq_len,
1296
self.scheduler_config.max_num_seqs)
@@ -1943,7 +1948,11 @@ class HPUModelRunner(HPUModelRunnerBase[ModelInputForHPUWithSamplingMetadata]):
1948
f"graphs{'T' if use_graphs else 'F'}")
1949
else:
1950
model_event_name = 'model_executable'
1946
- with self.profiler.record_event('internal', model_event_name):
1951
+ with set_forward_context(
1952
+ model_input.attn_metadata, self.vllm_config,
1953
+ model_input.virtual_engine), \
1954
+ self.profiler.record_event(
1955
+ 'internal', model_event_name):
1956
hidden_states = self.model.forward(
1957
**execute_model_kwargs,
1958
selected_token_indices=sampling_metadata.selected_token_indices
vllm/worker/hpu_worker.py
+3
@@ -20,6 +20,7 @@ from vllm.lora.request import LoRARequest
20
from vllm.model_executor import set_random_seed
21
from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sequence import ExecuteModelRequest
23
+from vllm.utils import bind_kv_cache
24
from vllm.worker.cache_engine import CacheEngine
25
from vllm.worker.hpu_model_runner import HPUModelRunner
26
from vllm.worker.model_runner_base import ModelRunnerBase
@@ -215,6 +216,8 @@ class HPUWorker(LocalOrDistributedWorkerBase):
216
self.cache_engine[ve].gpu_cache
217
for ve in range(self.parallel_config.pipeline_parallel_size)
218
]
219
+ bind_kv_cache(self.compilation_config.static_forward_context,
220
+ self.hpu_cache)
221
222
def _warm_up_model(self) -> None:
223
# NOTE(kzawora): We should use virtual engine index here
vllm/worker/neuron_model_runner.py
+10
-7
@@ -8,6 +8,7 @@ from torch import nn
8
from transformers_neuronx.config import GenerationConfig
9
10
from vllm.config import VllmConfig
11
+from vllm.forward_context import set_forward_context
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -314,13 +315,15 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
315
raise ValueError(
316
"NeuronModelRunner does not support multi-step execution.")
317
317
- hidden_states = self.model(
318
- input_ids=model_input.input_tokens,
319
- positions=model_input.input_positions,
320
- input_block_ids=model_input.input_block_ids,
321
- **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
322
- device=self.device),
323
- )
318
+ with set_forward_context(None, self.vllm_config, 0):
319
+ hidden_states = self.model(
320
+ input_ids=model_input.input_tokens,
321
+ positions=model_input.input_positions,
322
+ input_block_ids=model_input.input_block_ids,
323
+ **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs
324
+ or {},
325
+ device=self.device),
326
+ )
327
328
# Compute the logits only if the on-device sampling is turned off as
329
# on-device sampling outputs the token ids.
vllm/worker/openvino_model_runner.py
+3
-1
@@ -8,6 +8,7 @@ from torch import nn
8
from vllm.attention import get_attn_backend
9
from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
10
from vllm.config import VllmConfig
11
+from vllm.forward_context import set_forward_context
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.layers.sampler import SamplerOutput
@@ -350,7 +351,8 @@ class OpenVINOModelRunner(ModelRunnerBase):
351
device=self.device),
352
}
353
353
- hidden_states = model_executable(**execute_model_kwargs)
354
+ with set_forward_context(attn_metadata, self.vllm_config, 0):
355
+ hidden_states = model_executable(**execute_model_kwargs)
356
357
# Compute the logits.
358
logits = self.model.compute_logits(hidden_states, sampling_metadata)
vllm/worker/openvino_worker.py
+11
-2
@@ -20,6 +20,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
20
from vllm.platforms import current_platform
21
from vllm.sampling_params import SamplingParams
22
from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
23
+from vllm.utils import bind_kv_cache
24
from vllm.worker.openvino_model_runner import OpenVINOModelRunner
25
from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
26
@@ -339,6 +340,8 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
340
ov_device,
341
)
342
self.kv_cache = self.cache_engine.kv_cache
343
+ bind_kv_cache(self.compilation_config.static_forward_context,
344
+ [self.kv_cache])
345
self.model_runner.block_size = self.cache_engine.block_size
346
347
assert self.kv_cache is not None
@@ -507,12 +510,18 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
510
511
self.model_runner.block_size = tmp_cache_config.block_size
512
513
+ bind_kv_cache(self.compilation_config.static_forward_context,
514
+ profiling_cache_engine.kv_cache)
515
# Run the model with the dummy inputs.
516
self.model_runner.execute_model(seqs,
517
profiling_cache_engine.kv_cache)
518
514
- # explicitly delete temporary KV cache manager to free KV cache
515
- # when real inputs will be passed to OV
519
+ # Explicitly revert bind_kv_cache and delete temporary KV cache
520
+ # manager to free KV cache when real inputs will be passed to OV
521
+ bind_kv_cache(self.compilation_config.static_forward_context, [[
522
+ torch.tensor([])
523
+ for _ in range(len(profiling_cache_engine.kv_cache))
524
+ ]])
525
del profiling_cache_engine
526
527
logger.info(
vllm/worker/tpu_model_runner.py
+18
-10
@@ -13,6 +13,7 @@ import torch_xla.runtime as xr
13
14
from vllm.attention import AttentionMetadata, get_attn_backend
15
from vllm.config import VllmConfig
16
+from vllm.forward_context import set_forward_context
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.sampler import SamplerOutput
19
from vllm.model_executor.model_loader import get_model
@@ -265,8 +266,9 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
266
torch._dynamo.mark_dynamic(t, 0)
267
torch._dynamo.mark_dynamic(p, 0)
268
# Dummy run.
268
- self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
269
- num_samples, kv_caches)
269
+ with set_forward_context(attn_metadata, self.vllm_config, 0):
270
+ self.model(token_ids, position_ids, attn_metadata, input_lens, t,
271
+ p, num_samples, kv_caches)
272
273
def warmup_model(
274
self,
@@ -663,10 +665,13 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
665
input_lens = model_input.input_lens[i:i + 1].to(self.device)
666
t = model_input.t[i:i + 1].to(self.device)
667
p = model_input.p[i:i + 1].to(self.device)
666
- output_token_ids = self.model(token_ids, position_ids,
667
- attn_metadata, input_lens, t, p,
668
- model_input.num_samples,
669
- kv_caches)
668
+ with set_forward_context(model_input.attn_metadata,
669
+ self.vllm_config,
670
+ model_input.virtual_engine):
671
+ output_token_ids = self.model(token_ids, position_ids,
672
+ attn_metadata, input_lens, t,
673
+ p, model_input.num_samples,
674
+ kv_caches)
675
next_token_ids.append(output_token_ids[0])
676
start_idx = end_idx
677
@@ -711,10 +716,13 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
716
input_lens = model_input.input_lens.to(self.device)
717
for i in range(num_steps):
718
slot_mapping = attn_metadata.slot_mapping
714
- output_token_ids = self.model(token_ids, position_ids,
715
- attn_metadata, input_lens, t, p,
716
- model_input.num_samples,
717
- kv_caches)
719
+ with set_forward_context(model_input.attn_metadata,
720
+ self.vllm_config,
721
+ model_input.virtual_engine):
722
+ output_token_ids = self.model(token_ids, position_ids,
723
+ attn_metadata, input_lens, t,
724
+ p, model_input.num_samples,
725
+ kv_caches)
726
self.cached_step_outputs.append(output_token_ids)
727
728
if i < num_steps - 1:
vllm/worker/tpu_worker.py
+5
-1
@@ -12,7 +12,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
12
from vllm.logger import init_logger
13
from vllm.model_executor import set_random_seed
14
from vllm.sequence import ExecuteModelRequest
15
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
15
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, bind_kv_cache, get_dtype_size
16
from vllm.worker.tpu_model_runner import ExecutionMode, TPUModelRunner
17
from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
18
LoraNotSupportedWorkerBase, WorkerBase,
@@ -108,6 +108,8 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
108
torch.tensor([], dtype=torch.float32,
109
device=self.device))
110
for _ in range(num_layers)]
111
+ bind_kv_cache(self.compilation_config.static_forward_context,
112
+ [kv_caches])
113
self.model_runner._dummy_run(
114
batch_size=1,
115
seq_len=self.scheduler_config.max_num_batched_tokens,
@@ -170,6 +172,8 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
172
device="cpu")
173
cpu_v_cache = torch.zeros_like(cpu_k_cache)
174
self.cpu_cache.append((cpu_k_cache, cpu_v_cache))
175
+ bind_kv_cache(self.compilation_config.static_forward_context,
176
+ [self.tpu_cache])
177
self._warmup_model()
178
179
def _warmup_model(self) -> None:
vllm/worker/xpu_model_runner.py
+12
-9
@@ -12,6 +12,7 @@ import torch.nn as nn
12
from vllm.attention import get_attn_backend
13
from vllm.config import VllmConfig
14
from vllm.distributed import get_pp_group
15
+from vllm.forward_context import set_forward_context
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry
17
from vllm.logger import init_logger
18
from vllm.model_executor import SamplingMetadataCache
@@ -562,15 +563,17 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
563
if (self.observability_config is not None
564
and self.observability_config.collect_model_forward_time):
565
model_forward_start_time = time.time()
565
-
566
- hidden_or_intermediate_states = model_executable(
567
- input_ids=model_input.input_tokens,
568
- positions=model_input.input_positions,
569
- kv_caches=kv_caches,
570
- attn_metadata=model_input.attn_metadata,
571
- intermediate_tensors=intermediate_tensors,
572
- **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
573
- device=self.device))
566
+ with set_forward_context(model_input.attn_metadata, self.vllm_config,
567
+ model_input.virtual_engine):
568
+ hidden_or_intermediate_states = model_executable(
569
+ input_ids=model_input.input_tokens,
570
+ positions=model_input.input_positions,
571
+ kv_caches=kv_caches,
572
+ attn_metadata=model_input.attn_metadata,
573
+ intermediate_tensors=intermediate_tensors,
574
+ **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs
575
+ or {},
576
+ device=self.device))
577
# Compute the logits in the last pipeline stage.
578
if not get_pp_group().is_last_rank:
579
return hidden_or_intermediate_states