@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a9b15c606f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[torch.compile] use empty tensor instead of None for profiling (#8875)

youkaichao committed Sep 27, 2024 at 08:11 UTC a9b15c606fea67a072416ea0ea115261a2756058
15 files changed +84 -32
tests/kernels/test_encoder_decoder_attn.py
+6 -2
@@ -136,7 +136,9 @@ def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
136 )
137 if test_pt.num_blocks is None or test_pt.num_heads is None:
138 # Caller does not require a KV cache
139 - return TestResources(scale, attn_backend, attn, None)
139 + return TestResources(
140 + scale, attn_backend, attn,
141 + torch.tensor([], dtype=torch.float32, device=CUDA_DEVICE))
142
143 # Construct KV cache
144 kv_cache = make_kv_cache(test_pt.num_blocks,
@@ -620,7 +622,9 @@ def _run_encoder_attention_test(
622 return attn.forward(packed_qkv.query,
623 packed_qkv.key,
624 packed_qkv.value,
623 - None,
625 + torch.tensor([],
626 + dtype=torch.float32,
627 + device=packed_qkv.query.device),
628 attn_metadata,
629 attn_type=attn_type)
630
vllm/attention/backends/blocksparse_attn.py
+4 -2
@@ -357,6 +357,8 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
357 key: shape = [num_tokens, num_kv_heads * head_size]
358 value: shape = [num_tokens, num_kv_heads * head_size]
359 kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
360 + NOTE: kv_cache will be an empty tensor with shape [0]
361 + for profiling run.
362 attn_metadata: Metadata for attention.
363 Returns:
364 shape = [num_tokens, num_heads * head_size]
@@ -373,7 +375,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
375 key = key.view(-1, self.num_kv_heads, self.head_size)
376 value = value.view(-1, self.num_kv_heads, self.head_size)
377
376 - if kv_cache is not None:
378 + if kv_cache.numel() > 0:
379 key_cache, value_cache = PagedAttention.split_kv_cache(
380 kv_cache, self.num_kv_heads, self.head_size)
381
@@ -399,7 +401,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
401 # When block_tables are not filled, it means q and k are the
402 # prompt, and they have the same length.
403
402 - assert kv_cache is None \
404 + assert kv_cache.numel() == 0 \
405 or prefill_meta.block_tables is None \
406 or prefill_meta.block_tables.numel() == 0, \
407 "Does not support prefix-enabled attention."
vllm/attention/backends/flash_attn.py
+4 -2
@@ -665,6 +665,8 @@ class FlashAttentionImpl(AttentionImpl):
665 key: shape = [num_tokens, num_kv_heads * head_size]
666 value: shape = [num_tokens, num_kv_heads * head_size]
667 kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
668 + NOTE: kv_cache will be an empty tensor with shape [0]
669 + for profiling run.
670 attn_metadata: Metadata for attention.
671 Returns:
672 shape = [num_tokens, num_heads * head_size]
@@ -685,7 +687,7 @@ class FlashAttentionImpl(AttentionImpl):
687 key = key.view(-1, self.num_kv_heads, self.head_size)
688 value = value.view(-1, self.num_kv_heads, self.head_size)
689
688 - if kv_cache is not None:
690 + if kv_cache.numel() > 0:
691 key_cache = kv_cache[0]
692 value_cache = kv_cache[1]
693
@@ -722,7 +724,7 @@ class FlashAttentionImpl(AttentionImpl):
724
725 if prefill_meta := attn_metadata.prefill_metadata:
726 # Prompt run.
725 - if (kv_cache is None or prefill_meta.block_tables is None
727 + if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
728 or prefill_meta.block_tables.numel() == 0):
729 # normal attention
730 # When block_tables are not filled, it means q and k are the
vllm/attention/backends/flashinfer.py
+3 -3
@@ -746,7 +746,7 @@ class FlashInferImpl(AttentionImpl):
746 query: torch.Tensor,
747 key: torch.Tensor,
748 value: torch.Tensor,
749 - kv_cache: Optional[torch.Tensor],
749 + kv_cache: torch.Tensor,
750 attn_metadata: FlashInferMetadata,
751 k_scale: float = 1.0,
752 v_scale: float = 1.0,
@@ -770,7 +770,7 @@ class FlashInferImpl(AttentionImpl):
770 if attn_metadata.num_decode_tokens > 0:
771 assert attn_metadata.num_prefill_tokens == 0, (
772 "Chunked prefill is not supported with flashinfer yet.")
773 - if kv_cache is not None:
773 + if kv_cache.numel() > 0:
774 # Use the same reshape and cache kernel as flash attention.
775 ops.reshape_and_cache_flash(
776 key,
@@ -796,7 +796,7 @@ class FlashInferImpl(AttentionImpl):
796 # when kv_cache is not provided.
797 # This happens when vllm runs the profiling to
798 # determine the number of blocks.
799 - if kv_cache is None:
799 + if kv_cache.numel() == 0:
800 output = torch.ops.vllm.flash_attn_varlen_func(
801 q=query,
802 k=key,
vllm/attention/backends/ipex_attn.py
+6 -3
@@ -167,7 +167,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
167 query: torch.Tensor,
168 key: torch.Tensor,
169 value: torch.Tensor,
170 - kv_cache: Optional[torch.Tensor],
170 + kv_cache: torch.Tensor,
171 attn_metadata: IpexAttnMetadata, # type: ignore
172 k_scale: float = 1.0,
173 v_scale: float = 1.0,
@@ -180,6 +180,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
180 key: shape = [num_tokens, num_kv_heads * head_size]
181 value: shape = [num_tokens, num_kv_heads * head_size]
182 kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
183 + NOTE: kv_cache will be an empty tensor with shape [0]
184 + for profiling run.
185 attn_metadata: Metadata for attention.
186 Returns:
187 shape = [num_tokens, num_heads * head_size]
@@ -196,7 +198,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
198 key = key.view(-1, self.num_kv_heads, self.head_size)
199 value = value.view(-1, self.num_kv_heads, self.head_size)
200
199 - if kv_cache is not None:
201 + if kv_cache.numel() > 0:
202 key_cache, value_cache = self.split_kv_cache(
203 kv_cache, self.num_kv_heads, self.head_size)
204 ipex_ops.reshape_and_cache(
@@ -212,7 +214,8 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
214
215 if attn_metadata.is_prompt:
216 assert attn_metadata.seq_lens is not None
215 - if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
217 + if (kv_cache.numel() == 0
218 + or attn_metadata.block_tables.numel() == 0):
219 if self.num_kv_heads != self.num_heads:
220 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
221 value = value.repeat_interleave(self.num_queries_per_kv,
vllm/attention/backends/pallas.py
+7 -5
@@ -143,7 +143,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
143 query: torch.Tensor,
144 key: torch.Tensor,
145 value: torch.Tensor,
146 - kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
146 + kv_cache: Tuple[torch.Tensor, torch.Tensor],
147 attn_metadata: PallasMetadata,
148 k_scale: float = 1.0,
149 v_scale: float = 1.0,
@@ -155,8 +155,10 @@ class PallasAttentionBackendImpl(AttentionImpl):
155 query: shape = [batch_size, seq_len, num_heads * head_size]
156 key: shape = [batch_size, seq_len, num_kv_heads * head_size]
157 value: shape = [batch_size, seq_len, num_kv_heads * head_size]
158 - key_cache = [num_kv_heads, num_blocks, block_size, head_size]
159 - value_cache = [num_kv_heads, num_blocks, block_size, head_size]
158 + kv_cache[0] = [num_kv_heads, num_blocks, block_size, head_size]
159 + kv_cache[1] = [num_kv_heads, num_blocks, block_size, head_size]
160 + NOTE: kv_cache[0] and kv_cache[1] will be an empty tensor
161 + with shape [0] for profiling run.
162 attn_metadata: Metadata for attention.
163 Returns:
164 shape = [batch_size, seq_len, num_heads * head_size]
@@ -173,7 +175,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
175 value = value.view(batch_size, seq_len, self.num_kv_heads,
176 self.head_size)
177
176 - if kv_cache[0] is not None:
178 + if kv_cache[0].numel() > 0:
179 slot_mapping = attn_metadata.slot_mapping
180 key_cache, value_cache = kv_cache
181 write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping)
@@ -205,7 +207,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
207 output = output.permute(0, 2, 1, 3)
208 else:
209 # Decoding run.
208 - assert kv_cache is not None
210 + assert kv_cache[0].numel() > 0
211
212 pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
213 if self.megacore_mode == "batch" and batch_size % 2 != 0:
vllm/attention/backends/rocm_flash_attn.py
+4 -2
@@ -396,6 +396,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
396 key: shape = [num_tokens, num_kv_heads * head_size]
397 value: shape = [num_tokens, num_kv_heads * head_size]
398 kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
399 + NOTE: kv_cache will be an empty tensor with shape [0]
400 + for profiling run.
401 attn_metadata: Metadata for attention.
402 Returns:
403 shape = [num_tokens, num_heads * head_size]
@@ -412,7 +414,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
414 key = key.view(-1, self.num_kv_heads, self.head_size)
415 value = value.view(-1, self.num_kv_heads, self.head_size)
416
415 - if kv_cache is not None:
417 + if kv_cache.numel() > 0:
418 key_cache, value_cache = PagedAttention.split_kv_cache(
419 kv_cache, self.num_kv_heads, self.head_size)
420
@@ -449,7 +451,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
451 if prefill_meta := attn_metadata.prefill_metadata:
452 # Prompt run.
453 assert prefill_meta.seq_lens is not None
452 - if kv_cache is None or prefill_meta.block_tables.numel() == 0:
454 + if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0:
455 # triton attention
456 # When block_tables are not filled, it means q and k are the
457 # prompt, and they have the same length.
vllm/attention/backends/torch_sdpa.py
+6 -3
@@ -151,7 +151,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
151 query: torch.Tensor,
152 key: torch.Tensor,
153 value: torch.Tensor,
154 - kv_cache: Optional[torch.Tensor],
154 + kv_cache: torch.Tensor,
155 attn_metadata: TorchSDPAMetadata, # type: ignore
156 k_scale: float = 1.0,
157 v_scale: float = 1.0,
@@ -164,6 +164,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
164 key: shape = [num_tokens, num_kv_heads * head_size]
165 value: shape = [num_tokens, num_kv_heads * head_size]
166 kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
167 + NOTE: kv_cache will be an empty tensor with shape [0]
168 + for profiling run.
169 attn_metadata: Metadata for attention.
170 Returns:
171 shape = [num_tokens, num_heads * head_size]
@@ -180,7 +182,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
182 key = key.view(-1, self.num_kv_heads, self.head_size)
183 value = value.view(-1, self.num_kv_heads, self.head_size)
184
183 - if kv_cache is not None:
185 + if kv_cache.numel() > 0:
186 key_cache, value_cache = PagedAttention.split_kv_cache(
187 kv_cache, self.num_kv_heads, self.head_size)
188 PagedAttention.write_to_paged_cache(key, value, key_cache,
@@ -191,7 +193,8 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
193
194 if attn_metadata.is_prompt:
195 assert attn_metadata.seq_lens is not None
194 - if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
196 + if (kv_cache.numel() == 0
197 + or attn_metadata.block_tables.numel() == 0):
198 if self.num_kv_heads != self.num_heads:
199 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
200 value = value.repeat_interleave(self.num_queries_per_kv,
vllm/attention/backends/xformers.py
+5 -3
@@ -445,7 +445,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
445 query: torch.Tensor,
446 key: Optional[torch.Tensor],
447 value: Optional[torch.Tensor],
448 - kv_cache: Optional[torch.Tensor],
448 + kv_cache: torch.Tensor,
449 attn_metadata: "XFormersMetadata",
450 k_scale: float = 1.0,
451 v_scale: float = 1.0,
@@ -489,6 +489,8 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
489 key: shape = [num_tokens, num_kv_heads * head_size]
490 value: shape = [num_tokens, num_kv_heads * head_size]
491 kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
492 + NOTE: kv_cache will be an empty tensor with shape [0]
493 + for profiling run.
494 attn_metadata: Metadata for attention.
495 attn_type: Select attention type, between encoder attention,
496 decoder self-attention, or encoder/decoder cross-
@@ -522,7 +524,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
524 # which KV cache memory-mapping & which
525 # seqlen datastructures we utilize
526
525 - if (attn_type != AttentionType.ENCODER and kv_cache is not None):
527 + if (attn_type != AttentionType.ENCODER and kv_cache.numel() > 0):
528 # KV-cache during decoder-self- or
529 # encoder-decoder-cross-attention, but not
530 # during encoder attention.
@@ -588,7 +590,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
590
591 if prefill_meta := attn_metadata.prefill_metadata:
592 # Prompt run.
591 - if kv_cache is None or prefill_meta.block_tables.numel() == 0:
593 + if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0:
594 # normal attention.
595 # block tables are empty if the prompt does not have a cached
596 # prefix.
vllm/worker/embedding_model_runner.py
+7 -1
@@ -97,7 +97,13 @@ class EmbeddingModelRunner(
97 model_executable = self.model
98
99 num_layers = self.model_config.get_num_layers(self.parallel_config)
100 - kv_caches = [None] * num_layers
100 + # use an empty tensor instead of `None`` to force Dynamo to pass
101 + # it by reference, rather by specializing on the value ``None``.
102 + # the `dtype` argument does not matter, and we use `float32` as
103 + # a placeholder (it has wide hardware support).
104 + kv_caches = [
105 + torch.tensor([], dtype=torch.float32, device=self.device)
106 + ] * num_layers
107
108 execute_model_kwargs = {
109 "input_ids":
vllm/worker/enc_dec_model_runner.py
+7 -1
@@ -340,7 +340,13 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
340
341 # Run the model with the dummy inputs.
342 num_layers = self.model_config.get_num_layers(self.parallel_config)
343 - kv_caches = [None] * num_layers
343 + # use an empty tensor instead of `None`` to force Dynamo to pass
344 + # it by reference, rather by specializing on the value ``None``.
345 + # the `dtype` argument does not matter, and we use `float32` as
346 + # a placeholder (it has wide hardware support).
347 + kv_caches = [
348 + torch.tensor([], dtype=torch.float32, device=self.device)
349 + ] * num_layers
350 finished_requests_ids = [seq.request_id for seq in seqs]
351 model_input = self.prepare_model_input(
352 seqs, finished_requests_ids=finished_requests_ids)
vllm/worker/model_runner.py
+7 -1
@@ -1223,7 +1223,13 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1223
1224 # Run the model with the dummy inputs.
1225 num_layers = self.model_config.get_num_layers(self.parallel_config)
1226 - kv_caches = [None] * num_layers
1226 + # use an empty tensor instead of `None`` to force Dynamo to pass
1227 + # it by reference, rather by specializing on the value ``None``.
1228 + # the `dtype` argument does not matter, and we use `float32` as
1229 + # a placeholder (it has wide hardware support).
1230 + kv_caches = [
1231 + torch.tensor([], dtype=torch.float32, device=self.device)
1232 + ] * num_layers
1233 finished_requests_ids = [seq.request_id for seq in seqs]
1234 model_input = self.prepare_model_input(
1235 seqs, finished_requests_ids=finished_requests_ids)
vllm/worker/tpu_model_runner.py
+2 -2
@@ -714,7 +714,7 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
714 t: torch.Tensor,
715 p: torch.Tensor,
716 num_samples: int,
717 - kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
717 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
718 ) -> torch.Tensor:
719 """Executes the forward pass of the model and samples the next token.
720
@@ -745,7 +745,7 @@ class ModelWrapper(TorchCompileWrapperWithCustomDispatcher):
745 )
746
747 # Skip this in memory profiling at initialization.
748 - if kv_caches[0][0] is not None:
748 + if kv_caches[0][0].numel() > 0:
749 # index_copy_(slot_mapping) only works when the inserted dimension
750 # is 0. However, the KV cache in the Pallas backend has the shape
751 # [num_kv_heads, num_blocks, block_size, head_size]. To make it
vllm/worker/tpu_worker.py
+9 -1
@@ -115,7 +115,15 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
115 head_size = self.model_config.get_head_size()
116 num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
117
118 - kv_caches = [(None, None) for _ in range(num_layers)]
118 + # use an empty tensor instead of `None`` to force Dynamo to pass
119 + # it by reference, rather by specializing on the value ``None``.
120 + # the `dtype` argument does not matter, and we use `float32` as
121 + # a placeholder (it has wide hardware support).
122 + kv_caches = [(torch.tensor([], dtype=torch.float32,
123 + device=self.device),
124 + torch.tensor([], dtype=torch.float32,
125 + device=self.device))
126 + for _ in range(num_layers)]
127 self.model_runner._dummy_run(
128 batch_size=1,
129 seq_len=self.scheduler_config.max_num_batched_tokens,
vllm/worker/xpu_model_runner.py
+7 -1
@@ -464,7 +464,13 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
464
465 # Run the model with the dummy inputs.
466 num_layers = self.model_config.get_num_layers(self.parallel_config)
467 - kv_caches = [None] * num_layers
467 + # use an empty tensor instead of `None`` to force Dynamo to pass
468 + # it by reference, rather by specializing on the value ``None``.
469 + # the `dtype` argument does not matter, and we use `float32` as
470 + # a placeholder (it has wide hardware support).
471 + kv_caches = [
472 + torch.tensor([], dtype=torch.float32, device=self.device)
473 + ] * num_layers
474 finished_requests_ids = [seq.request_id for seq in seqs]
475 model_input = self.prepare_model_input(
476 seqs, finished_requests_ids=finished_requests_ids)