@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f8cafe2d1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Kernel] unified_attention for Attention.forward (#11967)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>

Chen Zhang committed Jan 13, 2025 at 19:28 UTC 0f8cafe2d1550a33803fb64b2224e6bf3f913449
10 files changed +87 -45
vllm/attention/layer.py
+14 -12
@@ -134,15 +134,10 @@ class Attention(nn.Module):
134 query: torch.Tensor,
135 key: torch.Tensor,
136 value: torch.Tensor,
137 - kv_cache: torch.Tensor,
138 - attn_metadata: AttentionMetadata,
137 + _kv_cache: torch.Tensor,
138 + _attn_metadata: AttentionMetadata,
139 ) -> torch.Tensor:
140 -
141 - if self.use_direct_call:
142 - return self.impl.forward(query, key, value, kv_cache,
143 - attn_metadata, self._k_scale,
144 - self._v_scale)
145 - elif self.use_output:
140 + if self.use_output:
141 output = torch.empty_like(query)
142 hidden_size = query.size(-1)
143 # Reshape the query, key, and value tensors.
@@ -154,12 +149,19 @@ class Attention(nn.Module):
149 key = key.view(-1, self.num_kv_heads, self.head_size)
150 if value is not None:
151 value = value.view(-1, self.num_kv_heads, self.head_size)
157 - torch.ops.vllm.unified_attention_with_output(
158 - query, key, value, output, self.layer_name)
152 + if self.use_direct_call:
153 + unified_attention_with_output(query, key, value, output,
154 + self.layer_name)
155 + else:
156 + torch.ops.vllm.unified_attention_with_output(
157 + query, key, value, output, self.layer_name)
158 return output.view(-1, hidden_size)
159 else:
161 - return torch.ops.vllm.unified_attention(query, key, value,
162 - self.layer_name)
160 + if self.use_direct_call:
161 + return unified_attention(query, key, value, self.layer_name)
162 + else:
163 + return torch.ops.vllm.unified_attention(
164 + query, key, value, self.layer_name)
165
166 def extra_repr(self) -> str:
167 s = f"head_size={self.impl.head_size}" # type: ignore
vllm/utils.py
-1
@@ -2171,5 +2171,4 @@ def bind_kv_cache(
2171 forward_ctx = ctx[layer_name]
2172 assert len(forward_ctx.kv_cache) == len(kv_cache)
2173 for ve, ve_kv_cache in enumerate(kv_cache):
2174 - assert forward_ctx.kv_cache[ve].numel() == 0
2174 forward_ctx.kv_cache[ve] = ve_kv_cache[kv_cache_idx]
vllm/worker/hpu_model_runner.py
+11 -2
@@ -28,6 +28,7 @@ from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler,
28 from vllm.attention import AttentionMetadata, get_attn_backend
29 from vllm.config import DeviceConfig, VllmConfig
30 from vllm.distributed.parallel_state import get_world_group
31 +from vllm.forward_context import set_forward_context
32 from vllm.logger import init_logger
33 from vllm.lora.layers import LoRAMapping
34 from vllm.lora.request import LoRARequest
@@ -40,7 +41,8 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
41 from vllm.sampling_params import SamplingParams
42 from vllm.sequence import (IntermediateTensors, SequenceData,
43 SequenceGroupMetadata)
43 -from vllm.utils import is_pin_memory_available, make_tensor_with_pad
44 +from vllm.utils import (bind_kv_cache, is_pin_memory_available,
45 + make_tensor_with_pad)
46 from vllm.worker.model_runner_base import (
47 ModelRunnerBase, ModelRunnerInputBase,
48 _add_attn_metadata_broadcastable_dict,
@@ -1286,6 +1288,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1288 def profile_run(self) -> None:
1289 num_layers = self.model_config.get_num_layers(self.parallel_config)
1290 kv_caches = [None] * num_layers
1291 + bind_kv_cache(
1292 + self.vllm_config.compilation_config.static_forward_context,
1293 + [kv_caches])
1294 max_seq_len = self.bucketing_global_state.prompt_seq_bucket_cfg[-1]
1295 max_batch_size = min(self.max_num_batched_tokens // max_seq_len,
1296 self.scheduler_config.max_num_seqs)
@@ -1943,7 +1948,11 @@ class HPUModelRunner(HPUModelRunnerBase[ModelInputForHPUWithSamplingMetadata]):
1948 f"graphs{'T' if use_graphs else 'F'}")
1949 else:
1950 model_event_name = 'model_executable'
1946 - with self.profiler.record_event('internal', model_event_name):
1951 + with set_forward_context(
1952 + model_input.attn_metadata, self.vllm_config,
1953 + model_input.virtual_engine), \
1954 + self.profiler.record_event(
1955 + 'internal', model_event_name):
1956 hidden_states = self.model.forward(
1957 **execute_model_kwargs,
1958 selected_token_indices=sampling_metadata.selected_token_indices
vllm/worker/hpu_worker.py
+3
@@ -20,6 +20,7 @@ from vllm.lora.request import LoRARequest
20 from vllm.model_executor import set_random_seed
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sequence import ExecuteModelRequest
23 +from vllm.utils import bind_kv_cache
24 from vllm.worker.cache_engine import CacheEngine
25 from vllm.worker.hpu_model_runner import HPUModelRunner
26 from vllm.worker.model_runner_base import ModelRunnerBase
@@ -215,6 +216,8 @@ class HPUWorker(LocalOrDistributedWorkerBase):
216 self.cache_engine[ve].gpu_cache
217 for ve in range(self.parallel_config.pipeline_parallel_size)
218 ]
219 + bind_kv_cache(self.compilation_config.static_forward_context,
220 + self.hpu_cache)
221
222 def _warm_up_model(self) -> None:
223 # NOTE(kzawora): We should use virtual engine index here
vllm/worker/neuron_model_runner.py
+10 -7
@@ -8,6 +8,7 @@ from torch import nn
8 from transformers_neuronx.config import GenerationConfig
9
10 from vllm.config import VllmConfig
11 +from vllm.forward_context import set_forward_context
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -314,13 +315,15 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
315 raise ValueError(
316 "NeuronModelRunner does not support multi-step execution.")
317
317 - hidden_states = self.model(
318 - input_ids=model_input.input_tokens,
319 - positions=model_input.input_positions,
320 - input_block_ids=model_input.input_block_ids,
321 - **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
322 - device=self.device),
323 - )
318 + with set_forward_context(None, self.vllm_config, 0):
319 + hidden_states = self.model(
320 + input_ids=model_input.input_tokens,
321 + positions=model_input.input_positions,
322 + input_block_ids=model_input.input_block_ids,
323 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs
324 + or {},
325 + device=self.device),
326 + )
327
328 # Compute the logits only if the on-device sampling is turned off as
329 # on-device sampling outputs the token ids.
vllm/worker/openvino_model_runner.py
+3 -1
@@ -8,6 +8,7 @@ from torch import nn
8 from vllm.attention import get_attn_backend
9 from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
10 from vllm.config import VllmConfig
11 +from vllm.forward_context import set_forward_context
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.layers.sampler import SamplerOutput
@@ -350,7 +351,8 @@ class OpenVINOModelRunner(ModelRunnerBase):
351 device=self.device),
352 }
353
353 - hidden_states = model_executable(**execute_model_kwargs)
354 + with set_forward_context(attn_metadata, self.vllm_config, 0):
355 + hidden_states = model_executable(**execute_model_kwargs)
356
357 # Compute the logits.
358 logits = self.model.compute_logits(hidden_states, sampling_metadata)
vllm/worker/openvino_worker.py
+11 -2
@@ -20,6 +20,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
20 from vllm.platforms import current_platform
21 from vllm.sampling_params import SamplingParams
22 from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
23 +from vllm.utils import bind_kv_cache
24 from vllm.worker.openvino_model_runner import OpenVINOModelRunner
25 from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
26
@@ -339,6 +340,8 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
340 ov_device,
341 )
342 self.kv_cache = self.cache_engine.kv_cache
343 + bind_kv_cache(self.compilation_config.static_forward_context,
344 + [self.kv_cache])
345 self.model_runner.block_size = self.cache_engine.block_size
346
347 assert self.kv_cache is not None
@@ -507,12 +510,18 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
510
511 self.model_runner.block_size = tmp_cache_config.block_size
512
513 + bind_kv_cache(self.compilation_config.static_forward_context,
514 + profiling_cache_engine.kv_cache)
515 # Run the model with the dummy inputs.
516 self.model_runner.execute_model(seqs,
517 profiling_cache_engine.kv_cache)
518
514 - # explicitly delete temporary KV cache manager to free KV cache
515 - # when real inputs will be passed to OV
519 + # Explicitly revert bind_kv_cache and delete temporary KV cache
520 + # manager to free KV cache when real inputs will be passed to OV
521 + bind_kv_cache(self.compilation_config.static_forward_context, [[
522 + torch.tensor([])
523 + for _ in range(len(profiling_cache_engine.kv_cache))
524 + ]])
525 del profiling_cache_engine
526
527 logger.info(
vllm/worker/tpu_model_runner.py
+18 -10
@@ -13,6 +13,7 @@ import torch_xla.runtime as xr
13
14 from vllm.attention import AttentionMetadata, get_attn_backend
15 from vllm.config import VllmConfig
16 +from vllm.forward_context import set_forward_context
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.sampler import SamplerOutput
19 from vllm.model_executor.model_loader import get_model
@@ -265,8 +266,9 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
266 torch._dynamo.mark_dynamic(t, 0)
267 torch._dynamo.mark_dynamic(p, 0)
268 # Dummy run.
268 - self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
269 - num_samples, kv_caches)
269 + with set_forward_context(attn_metadata, self.vllm_config, 0):
270 + self.model(token_ids, position_ids, attn_metadata, input_lens, t,
271 + p, num_samples, kv_caches)
272
273 def warmup_model(
274 self,
@@ -663,10 +665,13 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
665 input_lens = model_input.input_lens[i:i + 1].to(self.device)
666 t = model_input.t[i:i + 1].to(self.device)
667 p = model_input.p[i:i + 1].to(self.device)
666 - output_token_ids = self.model(token_ids, position_ids,
667 - attn_metadata, input_lens, t, p,
668 - model_input.num_samples,
669 - kv_caches)
668 + with set_forward_context(model_input.attn_metadata,
669 + self.vllm_config,
670 + model_input.virtual_engine):
671 + output_token_ids = self.model(token_ids, position_ids,
672 + attn_metadata, input_lens, t,
673 + p, model_input.num_samples,
674 + kv_caches)
675 next_token_ids.append(output_token_ids[0])
676 start_idx = end_idx
677
@@ -711,10 +716,13 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
716 input_lens = model_input.input_lens.to(self.device)
717 for i in range(num_steps):
718 slot_mapping = attn_metadata.slot_mapping
714 - output_token_ids = self.model(token_ids, position_ids,
715 - attn_metadata, input_lens, t, p,
716 - model_input.num_samples,
717 - kv_caches)
719 + with set_forward_context(model_input.attn_metadata,
720 + self.vllm_config,
721 + model_input.virtual_engine):
722 + output_token_ids = self.model(token_ids, position_ids,
723 + attn_metadata, input_lens, t,
724 + p, model_input.num_samples,
725 + kv_caches)
726 self.cached_step_outputs.append(output_token_ids)
727
728 if i < num_steps - 1:
vllm/worker/tpu_worker.py
+5 -1
@@ -12,7 +12,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
12 from vllm.logger import init_logger
13 from vllm.model_executor import set_random_seed
14 from vllm.sequence import ExecuteModelRequest
15 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
15 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, bind_kv_cache, get_dtype_size
16 from vllm.worker.tpu_model_runner import ExecutionMode, TPUModelRunner
17 from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
18 LoraNotSupportedWorkerBase, WorkerBase,
@@ -108,6 +108,8 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
108 torch.tensor([], dtype=torch.float32,
109 device=self.device))
110 for _ in range(num_layers)]
111 + bind_kv_cache(self.compilation_config.static_forward_context,
112 + [kv_caches])
113 self.model_runner._dummy_run(
114 batch_size=1,
115 seq_len=self.scheduler_config.max_num_batched_tokens,
@@ -170,6 +172,8 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
172 device="cpu")
173 cpu_v_cache = torch.zeros_like(cpu_k_cache)
174 self.cpu_cache.append((cpu_k_cache, cpu_v_cache))
175 + bind_kv_cache(self.compilation_config.static_forward_context,
176 + [self.tpu_cache])
177 self._warmup_model()
178
179 def _warmup_model(self) -> None:
vllm/worker/xpu_model_runner.py
+12 -9
@@ -12,6 +12,7 @@ import torch.nn as nn
12 from vllm.attention import get_attn_backend
13 from vllm.config import VllmConfig
14 from vllm.distributed import get_pp_group
15 +from vllm.forward_context import set_forward_context
16 from vllm.inputs import INPUT_REGISTRY, InputRegistry
17 from vllm.logger import init_logger
18 from vllm.model_executor import SamplingMetadataCache
@@ -562,15 +563,17 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
563 if (self.observability_config is not None
564 and self.observability_config.collect_model_forward_time):
565 model_forward_start_time = time.time()
565 -
566 - hidden_or_intermediate_states = model_executable(
567 - input_ids=model_input.input_tokens,
568 - positions=model_input.input_positions,
569 - kv_caches=kv_caches,
570 - attn_metadata=model_input.attn_metadata,
571 - intermediate_tensors=intermediate_tensors,
572 - **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
573 - device=self.device))
566 + with set_forward_context(model_input.attn_metadata, self.vllm_config,
567 + model_input.virtual_engine):
568 + hidden_or_intermediate_states = model_executable(
569 + input_ids=model_input.input_tokens,
570 + positions=model_input.input_positions,
571 + kv_caches=kv_caches,
572 + attn_metadata=model_input.attn_metadata,
573 + intermediate_tensors=intermediate_tensors,
574 + **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs
575 + or {},
576 + device=self.device))
577 # Compute the logits in the last pipeline stage.
578 if not get_pp_group().is_last_rank:
579 return hidden_or_intermediate_states