A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[torch.compile] use empty tensor instead of None for profiling (#8875)
youkaichao committed
Sep 27, 2024 at 08:11 UTC
a9b15c606fea67a072416ea0ea115261a2756058
15 files changed
+84
-32
tests/kernels/test_encoder_decoder_attn.py
+6
-2
@@ -136,7 +136,9 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
136
)
137
if test_pt.num_blocks is None or test_pt.num_heads is None:
138
# Caller does not require a KV cache
139
- return TestResources(scale, attn_backend, attn, None)
139
+ return TestResources(
140
+ scale, attn_backend, attn,
141
+ torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
142
143
# Construct KV cache
144
kv_cache = make_kv_cache(test_pt.num_blocks,
@@ -620,7 +622,9 @@ def _run_encoder_attention_test(
622
return attn.forward(packed_qkv.query,
623
packed_qkv.key,
624
packed_qkv.value,
623
- None,
625
+ torch.tensor([],
626
+ dtype=torch.float32,
627
+ device=packed_qkv.query.device),
628
attn_metadata,
629
attn_type=attn_type)
630
vllm/attention/backends/blocksparse_attn.py
+4
-2
@@ -357,6 +357,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
357
key: shape = [num_tokens, num_kv_heads * head_size]
358
value: shape = [num_tokens, num_kv_heads * head_size]
359
kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
360
+ NOTE: kv_cache will be an empty tensor with shape [0]
361
+ for profiling run.
362
attn_metadata: Metadata for attention.
363
Returns:
364
shape = [num_tokens, num_heads * head_size]
@@ -373,7 +375,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
375
key = key.view(-1, self.num_kv_heads, self.head_size)
376
value = value.view(-1, self.num_kv_heads, self.head_size)
377
376
- if kv_cache is not None:
378
+ if kv_cache.numel() > 0:
379
key_cache, value_cache = PagedAttention.split_kv_cache(
380
kv_cache, self.num_kv_heads, self.head_size)
381
@@ -399,7 +401,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
401
# When block_tables are not filled, it means q and k are the
402
# prompt, and they have the same length.
403
402
- assert kv_cache is None \
404
+ assert kv_cache.numel() == 0 \
405
or prefill_meta.block_tables is None \
406
or prefill_meta.block_tables.numel() == 0, \
407
"Does not support prefix-enabled attention."
vllm/attention/backends/flash_attn.py
+4
-2
@@ -665,6 +665,8 @@ class FlashAttentionImpl(AttentionImpl):
665
key: shape = [num_tokens, num_kv_heads * head_size]
666
value: shape = [num_tokens, num_kv_heads * head_size]
667
kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
668
+ NOTE: kv_cache will be an empty tensor with shape [0]
669
+ for profiling run.
670
attn_metadata: Metadata for attention.
671
Returns:
672
shape = [num_tokens, num_heads * head_size]
@@ -685,7 +687,7 @@ class FlashAttentionImpl(AttentionImpl):
687
key = key.view(-1, self.num_kv_heads, self.head_size)
688
value = value.view(-1, self.num_kv_heads, self.head_size)
689
688
- if kv_cache is not None:
690
+ if kv_cache.numel() > 0:
691
key_cache = kv_cache[0]
692
value_cache = kv_cache[1]
693
@@ -722,7 +724,7 @@ class FlashAttentionImpl(AttentionImpl):
724
725
if prefill_meta := attn_metadata.prefill_metadata:
726
# Prompt run.
725
- if (kv_cache is None or prefill_meta.block_tables is None
727
+ if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
728
or prefill_meta.block_tables.numel() == 0):
729
# normal attention
730
# When block_tables are not filled, it means q and k are the
vllm/attention/backends/flashinfer.py
+3
-3
@@ -746,7 +746,7 @@ class FlashInferImpl(AttentionImpl):
746
query: torch.Tensor,
747
key: torch.Tensor,
748
value: torch.Tensor,
749
- kv_cache: Optional[torch.Tensor],
749
+ kv_cache: torch.Tensor,
750
attn_metadata: FlashInferMetadata,
751
k_scale: float = 1.0,
752
v_scale: float = 1.0,
@@ -770,7 +770,7 @@ class FlashInferImpl(AttentionImpl):
770
if attn_metadata.num_decode_tokens > 0:
771
assert attn_metadata.num_prefill_tokens == 0, (
772
"Chunked prefill is not supported with flashinfer yet.")
773
- if kv_cache is not None:
773
+ if kv_cache.numel() > 0:
774
# Use the same reshape and cache kernel as flash attention.
775
ops.reshape_and_cache_flash(
776
key,
@@ -796,7 +796,7 @@ class FlashInferImpl(AttentionImpl):
796
# when kv_cache is not provided.
797
# This happens when vllm runs the profiling to
798
# determine the number of blocks.
799
- if kv_cache is None:
799
+ if kv_cache.numel() == 0:
800
output = torch.ops.vllm.flash_attn_varlen_func(
801
q=query,
802
k=key,
vllm/attention/backends/ipex_attn.py
+6
-3
@@ -167,7 +167,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
167
query: torch.Tensor,
168
key: torch.Tensor,
169
value: torch.Tensor,
170
- kv_cache: Optional[torch.Tensor],
170
+ kv_cache: torch.Tensor,
171
attn_metadata: IpexAttnMetadata, # type: ignore
172
k_scale: float = 1.0,
173
v_scale: float = 1.0,
@@ -180,6 +180,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
180
key: shape = [num_tokens, num_kv_heads * head_size]
181
value: shape = [num_tokens, num_kv_heads * head_size]
182
kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
183
+ NOTE: kv_cache will be an empty tensor with shape [0]
184
+ for profiling run.
185
attn_metadata: Metadata for attention.
186
Returns:
187
shape = [num_tokens, num_heads * head_size]
@@ -196,7 +198,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
198
key = key.view(-1, self.num_kv_heads, self.head_size)
199
value = value.view(-1, self.num_kv_heads, self.head_size)
200
199
- if kv_cache is not None:
201
+ if kv_cache.numel() > 0:
202
key_cache, value_cache = self.split_kv_cache(
203
kv_cache, self.num_kv_heads, self.head_size)
204
ipex_ops.reshape_and_cache(
@@ -212,7 +214,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
214
215
if attn_metadata.is_prompt:
216
assert attn_metadata.seq_lens is not None
215
- if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
217
+ if (kv_cache.numel() == 0
218
+ or attn_metadata.block_tables.numel() == 0):
219
if self.num_kv_heads != self.num_heads:
220
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
221
value = value.repeat_interleave(self.num_queries_per_kv,
vllm/attention/backends/pallas.py
+7
-5
@@ -143,7 +143,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
143
query: torch.Tensor,
144
key: torch.Tensor,
145
value: torch.Tensor,
146
- kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
146
+ kv_cache: Tuple[torch.Tensor, torch.Tensor],
147
attn_metadata: PallasMetadata,
148
k_scale: float = 1.0,
149
v_scale: float = 1.0,
@@ -155,8 +155,10 @@ class PallasAttentionBackendImpl(AttentionImpl):
155
query: shape = [batch_size, seq_len, num_heads * head_size]
156
key: shape = [batch_size, seq_len, num_kv_heads * head_size]
157
value: shape = [batch_size, seq_len, num_kv_heads * head_size]
158
- key_cache = [num_kv_heads, num_blocks, block_size, head_size]
159
- value_cache = [num_kv_heads, num_blocks, block_size, head_size]
158
+ kv_cache[0] = [num_kv_heads, num_blocks, block_size, head_size]
159
+ kv_cache[1] = [num_kv_heads, num_blocks, block_size, head_size]
160
+ NOTE: kv_cache[0] and kv_cache[1] will be an empty tensor
161
+ with shape [0] for profiling run.
162
attn_metadata: Metadata for attention.
163
Returns:
164
shape = [batch_size, seq_len, num_heads * head_size]
@@ -173,7 +175,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
175
value = value.view(batch_size, seq_len, self.num_kv_heads,
176
self.head_size)
177
176
- if kv_cache[0] is not None:
178
+ if kv_cache[0].numel() > 0:
179
slot_mapping = attn_metadata.slot_mapping
180
key_cache, value_cache = kv_cache
181
write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping)
@@ -205,7 +207,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
207
output = output.permute(0, 2, 1, 3)
208
else:
209
# Decoding run.
208
- assert kv_cache is not None
210
+ assert kv_cache[0].numel() > 0
211
212
pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
213
if self.megacore_mode == "batch" and batch_size % 2 != 0:
vllm/attention/backends/rocm_flash_attn.py
+4
-2
@@ -396,6 +396,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
396
key: shape = [num_tokens, num_kv_heads * head_size]
397
value: shape = [num_tokens, num_kv_heads * head_size]
398
kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
399
+ NOTE: kv_cache will be an empty tensor with shape [0]
400
+ for profiling run.
401
attn_metadata: Metadata for attention.
402
Returns:
403
shape = [num_tokens, num_heads * head_size]
@@ -412,7 +414,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
414
key = key.view(-1, self.num_kv_heads, self.head_size)
415
value = value.view(-1, self.num_kv_heads, self.head_size)
416
415
- if kv_cache is not None:
417
+ if kv_cache.numel() > 0:
418
key_cache, value_cache = PagedAttention.split_kv_cache(
419
kv_cache, self.num_kv_heads, self.head_size)
420
@@ -449,7 +451,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
451
if prefill_meta := attn_metadata.prefill_metadata:
452
# Prompt run.
453
assert prefill_meta.seq_lens is not None
452
- if kv_cache is None or prefill_meta.block_tables.numel() == 0:
454
+ if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0:
455
# triton attention
456
# When block_tables are not filled, it means q and k are the
457
# prompt, and they have the same length.
vllm/attention/backends/torch_sdpa.py
+6
-3
@@ -151,7 +151,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
151
query: torch.Tensor,
152
key: torch.Tensor,
153
value: torch.Tensor,
154
- kv_cache: Optional[torch.Tensor],
154
+ kv_cache: torch.Tensor,
155
attn_metadata: TorchSDPAMetadata, # type: ignore
156
k_scale: float = 1.0,
157
v_scale: float = 1.0,
@@ -164,6 +164,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
164
key: shape = [num_tokens, num_kv_heads * head_size]
165
value: shape = [num_tokens, num_kv_heads * head_size]
166
kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
167
+ NOTE: kv_cache will be an empty tensor with shape [0]
168
+ for profiling run.
169
attn_metadata: Metadata for attention.
170
Returns:
171
shape = [num_tokens, num_heads * head_size]
@@ -180,7 +182,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
182
key = key.view(-1, self.num_kv_heads, self.head_size)
183
value = value.view(-1, self.num_kv_heads, self.head_size)
184
183
- if kv_cache is not None:
185
+ if kv_cache.numel() > 0:
186
key_cache, value_cache = PagedAttention.split_kv_cache(
187
kv_cache, self.num_kv_heads, self.head_size)
188
PagedAttention.write_to_paged_cache(key, value, key_cache,
@@ -191,7 +193,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
193
194
if attn_metadata.is_prompt:
195
assert attn_metadata.seq_lens is not None
194
- if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
196
+ if (kv_cache.numel() == 0
197
+ or attn_metadata.block_tables.numel() == 0):
198
if self.num_kv_heads != self.num_heads:
199
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
200
value = value.repeat_interleave(self.num_queries_per_kv,
vllm/attention/backends/xformers.py
+5
-3
@@ -445,7 +445,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
445
query: torch.Tensor,
446
key: Optional[torch.Tensor],
447
value: Optional[torch.Tensor],
448
- kv_cache: Optional[torch.Tensor],
448
+ kv_cache: torch.Tensor,
449
attn_metadata: "XFormersMetadata",
450
k_scale: float = 1.0,
451
v_scale: float = 1.0,
@@ -489,6 +489,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
489
key: shape = [num_tokens, num_kv_heads * head_size]
490
value: shape = [num_tokens, num_kv_heads * head_size]
491
kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
492
+ NOTE: kv_cache will be an empty tensor with shape [0]
493
+ for profiling run.
494
attn_metadata: Metadata for attention.
495
attn_type: Select attention type, between encoder attention,
496
decoder self-attention, or encoder/decoder cross-
@@ -522,7 +524,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
524
# which KV cache memory-mapping & which
525
# seqlen datastructures we utilize
526
525
- if (attn_type != AttentionType.ENCODER and kv_cache is not None):
527
+ if (attn_type != AttentionType.ENCODER and kv_cache.numel() > 0):
528
# KV-cache during decoder-self- or
529
# encoder-decoder-cross-attention, but not
530
# during encoder attention.
@@ -588,7 +590,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
590
591
if prefill_meta := attn_metadata.prefill_metadata:
592
# Prompt run.
591
- if kv_cache is None or prefill_meta.block_tables.numel() == 0:
593
+ if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0:
594
# normal attention.
595
# block tables are empty if the prompt does not have a cached
596
# prefix.
vllm/worker/embedding_model_runner.py
+7
-1
@@ -97,7 +97,13 @@ class EmbeddingModelRunner(
97
model_executable = self.model
98
99
num_layers = self.model_config.get_num_layers(self.parallel_config)
100
- kv_caches = [None] * num_layers
100
+ # use an empty tensor instead of `None`` to force Dynamo to pass
101
+ # it by reference, rather by specializing on the value ``None``.
102
+ # the `dtype` argument does not matter, and we use `float32` as
103
+ # a placeholder (it has wide hardware support).
104
+ kv_caches = [
105
+ torch.tensor([], dtype=torch.float32, device=self.device)
106
+ ] * num_layers
107
108
execute_model_kwargs = {
109
"input_ids":
vllm/worker/enc_dec_model_runner.py
+7
-1
@@ -340,7 +340,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
340
341
# Run the model with the dummy inputs.
342
num_layers = self.model_config.get_num_layers(self.parallel_config)
343
- kv_caches = [None] * num_layers
343
+ # use an empty tensor instead of `None`` to force Dynamo to pass
344
+ # it by reference, rather by specializing on the value ``None``.
345
+ # the `dtype` argument does not matter, and we use `float32` as
346
+ # a placeholder (it has wide hardware support).
347
+ kv_caches = [
348
+ torch.tensor([], dtype=torch.float32, device=self.device)
349
+ ] * num_layers
350
finished_requests_ids = [seq.request_id for seq in seqs]
351
model_input = self.prepare_model_input(
352
seqs, finished_requests_ids=finished_requests_ids)
vllm/worker/model_runner.py
+7
-1
@@ -1223,7 +1223,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1223
1224
# Run the model with the dummy inputs.
1225
num_layers = self.model_config.get_num_layers(self.parallel_config)
1226
- kv_caches = [None] * num_layers
1226
+ # use an empty tensor instead of `None`` to force Dynamo to pass
1227
+ # it by reference, rather by specializing on the value ``None``.
1228
+ # the `dtype` argument does not matter, and we use `float32` as
1229
+ # a placeholder (it has wide hardware support).
1230
+ kv_caches = [
1231
+ torch.tensor([], dtype=torch.float32, device=self.device)
1232
+ ] * num_layers
1233
finished_requests_ids = [seq.request_id for seq in seqs]
1234
model_input = self.prepare_model_input(
1235
seqs, finished_requests_ids=finished_requests_ids)
vllm/worker/tpu_model_runner.py
+2
-2
@@ -714,7 +714,7 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
714
t: torch.Tensor,
715
p: torch.Tensor,
716
num_samples: int,
717
- kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
717
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
718
) -> torch.Tensor:
719
"""Executes the forward pass of the model and samples the next token.
720
@@ -745,7 +745,7 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
745
)
746
747
# Skip this in memory profiling at initialization.
748
- if kv_caches[0][0] is not None:
748
+ if kv_caches[0][0].numel() > 0:
749
# index_copy_(slot_mapping) only works when the inserted dimension
750
# is 0. However, the KV cache in the Pallas backend has the shape
751
# [num_kv_heads, num_blocks, block_size, head_size]. To make it
vllm/worker/tpu_worker.py
+9
-1
@@ -115,7 +115,15 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
115
head_size = self.model_config.get_head_size()
116
num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
117
118
- kv_caches = [(None, None) for _ in range(num_layers)]
118
+ # use an empty tensor instead of `None`` to force Dynamo to pass
119
+ # it by reference, rather by specializing on the value ``None``.
120
+ # the `dtype` argument does not matter, and we use `float32` as
121
+ # a placeholder (it has wide hardware support).
122
+ kv_caches = [(torch.tensor([], dtype=torch.float32,
123
+ device=self.device),
124
+ torch.tensor([], dtype=torch.float32,
125
+ device=self.device))
126
+ for _ in range(num_layers)]
127
self.model_runner._dummy_run(
128
batch_size=1,
129
seq_len=self.scheduler_config.max_num_batched_tokens,
vllm/worker/xpu_model_runner.py
+7
-1
@@ -464,7 +464,13 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
464
465
# Run the model with the dummy inputs.
466
num_layers = self.model_config.get_num_layers(self.parallel_config)
467
- kv_caches = [None] * num_layers
467
+ # use an empty tensor instead of `None`` to force Dynamo to pass
468
+ # it by reference, rather by specializing on the value ``None``.
469
+ # the `dtype` argument does not matter, and we use `float32` as
470
+ # a placeholder (it has wide hardware support).
471
+ kv_caches = [
472
+ torch.tensor([], dtype=torch.float32, device=self.device)
473
+ ] * num_layers
474
finished_requests_ids = [seq.request_id for seq in seqs]
475
model_input = self.prepare_model_input(
476
seqs, finished_requests_ids=finished_requests_ids)