A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[TPU] Fix TPU SMEM OOM by Pallas paged attention kernel (#9350)
Woosuk Kwon committed
Oct 14, 2024 at 15:02 UTC
473e7b3606e9b95b39c7da46cce00a33c069dc00
2 files changed
+89
-21
vllm/attention/backends/pallas.py
+80
-21
@@ -208,35 +208,54 @@ class PallasAttentionBackendImpl(AttentionImpl):
208
else:
209
# Decoding run.
210
assert kv_cache[0].numel() > 0
211
-
211
+ query = query.squeeze(dim=1)
212
pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
213
- if self.megacore_mode == "batch" and batch_size % 2 != 0:
214
- megacore_mode = None
215
- else:
216
- megacore_mode = self.megacore_mode
217
-
218
- # NOTE(woosuk): A temporary workaround to avoid the error:
219
- # "xla::paged_attention() Expected a value of type 'str' for
220
- # argument 'megacore_mode' but instead found type 'NoneType'."
221
- if megacore_mode is not None:
222
- output = torch.ops.xla.paged_attention(
223
- query.squeeze(dim=1),
213
+
214
+ assert attn_metadata.block_tables is not None
215
+ assert attn_metadata.context_lens is not None
216
+ # NOTE(woosuk): The PagedAttention Pallas kernel stores the entire
217
+ # block table in SMEM. Therefore, if the block table is too large,
218
+ # the kernel compilation will fail. To avoid this, we split the
219
+ # batch dimension into smaller chunks and run the kernel multiple
220
+ # times.
221
+ MAX_SMEM_USAGE = 512 * 1024
222
+ size_per_seq = 4 * attn_metadata.block_tables.shape[1]
223
+ max_num_seq = MAX_SMEM_USAGE // size_per_seq
224
+
225
+ if batch_size <= max_num_seq:
226
+ output = paged_attention(
227
+ query,
228
key_cache,
229
value_cache,
230
attn_metadata.context_lens,
231
attn_metadata.block_tables,
232
pages_per_compute_block,
229
- megacore_mode=megacore_mode,
233
+ self.megacore_mode,
234
)
235
else:
232
- output = torch.ops.xla.paged_attention(
233
- query.squeeze(dim=1),
234
- key_cache,
235
- value_cache,
236
- attn_metadata.context_lens,
237
- attn_metadata.block_tables,
238
- pages_per_compute_block,
239
- )
236
+ chunk_size = max_num_seq
237
+ # Make sure the chunk size is a multiple of 2.
238
+ chunk_size = chunk_size // 2 * 2
239
+ num_chunks = (batch_size + chunk_size - 1) // chunk_size
240
+
241
+ output = torch.empty_like(query)
242
+ for chunk_idx in range(num_chunks):
243
+ chunk_start = chunk_idx * chunk_size
244
+ chunk_end = chunk_start + chunk_size
245
+ # NOTE(woosuk): We skip this line because it causes Dynamo
246
+ # compilation error. Instead, we rely on the slice operation
247
+ # to handle the out-of-bound case.
248
+ # chunk_end = min(chunk_end, batch_size)
249
+ chunk_output = paged_attention(
250
+ query[chunk_start:chunk_end],
251
+ key_cache,
252
+ value_cache,
253
+ attn_metadata.context_lens[chunk_start:chunk_end],
254
+ attn_metadata.block_tables[chunk_start:chunk_end],
255
+ pages_per_compute_block,
256
+ self.megacore_mode,
257
+ )
258
+ output[chunk_start:chunk_end] = chunk_output
259
260
# Reshape the output tensor.
261
return output.reshape(batch_size, seq_len, hidden_size)
@@ -258,3 +277,43 @@ def write_to_kv_cache(
277
value_cache = value_cache.flatten(0, 2)
278
key_cache.index_copy_(0, slot_mapping, key)
279
value_cache.index_copy_(0, slot_mapping, value)
280
+
281
+
282
+def paged_attention(
283
+ query: torch.Tensor,
284
+ key_cache: torch.Tensor,
285
+ value_cache: torch.Tensor,
286
+ context_lens: torch.Tensor,
287
+ block_tables: torch.Tensor,
288
+ pages_per_compute_block: int,
289
+ megacore_mode: Optional[str],
290
+) -> torch.Tensor:
291
+ batch_size = query.shape[0]
292
+ if megacore_mode == "batch" and batch_size % 2 != 0:
293
+ megacore_mode = None
294
+ else:
295
+ megacore_mode = megacore_mode
296
+
297
+ # NOTE(woosuk): A temporary workaround to avoid the error:
298
+ # "xla::paged_attention() Expected a value of type 'str' for
299
+ # argument 'megacore_mode' but instead found type 'NoneType'."
300
+ if megacore_mode is not None:
301
+ output = torch.ops.xla.paged_attention(
302
+ query,
303
+ key_cache,
304
+ value_cache,
305
+ context_lens,
306
+ block_tables,
307
+ pages_per_compute_block,
308
+ megacore_mode=megacore_mode,
309
+ )
310
+ else:
311
+ output = torch.ops.xla.paged_attention(
312
+ query,
313
+ key_cache,
314
+ value_cache,
315
+ context_lens,
316
+ block_tables,
317
+ pages_per_compute_block,
318
+ )
319
+ return output
vllm/worker/tpu_model_runner.py
+9
@@ -123,6 +123,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
123
)
124
self.cached_step_outputs: List[torch.Tensor] = []
125
126
+ smem_size = 512 * 1024
127
+ block_table_size = 4 * self.block_tables.size
128
+ if block_table_size >= smem_size:
129
+ logger.warning(
130
+ "The max_model_len (%d) is too large. This may degrade the "
131
+ "performance due to the insufficient smem size. Consider "
132
+ "setting --max-model-len to a smaller value.",
133
+ self.model_config.max_model_len)
134
+
135
def load_model(self) -> None:
136
self.device = self.device_config.device
137