@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 473e7b3606
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[TPU] Fix TPU SMEM OOM by Pallas paged attention kernel (#9350)

Woosuk Kwon committed Oct 14, 2024 at 15:02 UTC 473e7b3606e9b95b39c7da46cce00a33c069dc00
2 files changed +89 -21
vllm/attention/backends/pallas.py
+80 -21
@@ -208,35 +208,54 @@ class PallasAttentionBackendImpl(AttentionImpl):
208 else:
209 # Decoding run.
210 assert kv_cache[0].numel() > 0
211 -
211 + query = query.squeeze(dim=1)
212 pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
213 - if self.megacore_mode == "batch" and batch_size % 2 != 0:
214 - megacore_mode = None
215 - else:
216 - megacore_mode = self.megacore_mode
217 -
218 - # NOTE(woosuk): A temporary workaround to avoid the error:
219 - # "xla::paged_attention() Expected a value of type 'str' for
220 - # argument 'megacore_mode' but instead found type 'NoneType'."
221 - if megacore_mode is not None:
222 - output = torch.ops.xla.paged_attention(
223 - query.squeeze(dim=1),
213 +
214 + assert attn_metadata.block_tables is not None
215 + assert attn_metadata.context_lens is not None
216 + # NOTE(woosuk): The PagedAttention Pallas kernel stores the entire
217 + # block table in SMEM. Therefore, if the block table is too large,
218 + # the kernel compilation will fail. To avoid this, we split the
219 + # batch dimension into smaller chunks and run the kernel multiple
220 + # times.
221 + MAX_SMEM_USAGE = 512 * 1024
222 + size_per_seq = 4 * attn_metadata.block_tables.shape[1]
223 + max_num_seq = MAX_SMEM_USAGE // size_per_seq
224 +
225 + if batch_size <= max_num_seq:
226 + output = paged_attention(
227 + query,
228 key_cache,
229 value_cache,
230 attn_metadata.context_lens,
231 attn_metadata.block_tables,
232 pages_per_compute_block,
229 - megacore_mode=megacore_mode,
233 + self.megacore_mode,
234 )
235 else:
232 - output = torch.ops.xla.paged_attention(
233 - query.squeeze(dim=1),
234 - key_cache,
235 - value_cache,
236 - attn_metadata.context_lens,
237 - attn_metadata.block_tables,
238 - pages_per_compute_block,
239 - )
236 + chunk_size = max_num_seq
237 + # Make sure the chunk size is a multiple of 2.
238 + chunk_size = chunk_size // 2 * 2
239 + num_chunks = (batch_size + chunk_size - 1) // chunk_size
240 +
241 + output = torch.empty_like(query)
242 + for chunk_idx in range(num_chunks):
243 + chunk_start = chunk_idx * chunk_size
244 + chunk_end = chunk_start + chunk_size
245 + # NOTE(woosuk): We skip this line because it causes Dynamo
246 + # compilation error. Instead, we rely on the slice operation
247 + # to handle the out-of-bound case.
248 + # chunk_end = min(chunk_end, batch_size)
249 + chunk_output = paged_attention(
250 + query[chunk_start:chunk_end],
251 + key_cache,
252 + value_cache,
253 + attn_metadata.context_lens[chunk_start:chunk_end],
254 + attn_metadata.block_tables[chunk_start:chunk_end],
255 + pages_per_compute_block,
256 + self.megacore_mode,
257 + )
258 + output[chunk_start:chunk_end] = chunk_output
259
260 # Reshape the output tensor.
261 return output.reshape(batch_size, seq_len, hidden_size)
@@ -258,3 +277,43 @@ def write_to_kv_cache(
277 value_cache = value_cache.flatten(0, 2)
278 key_cache.index_copy_(0, slot_mapping, key)
279 value_cache.index_copy_(0, slot_mapping, value)
280 +
281 +
282 +def paged_attention(
283 + query: torch.Tensor,
284 + key_cache: torch.Tensor,
285 + value_cache: torch.Tensor,
286 + context_lens: torch.Tensor,
287 + block_tables: torch.Tensor,
288 + pages_per_compute_block: int,
289 + megacore_mode: Optional[str],
290 +) -> torch.Tensor:
291 + batch_size = query.shape[0]
292 + if megacore_mode == "batch" and batch_size % 2 != 0:
293 + megacore_mode = None
294 + else:
295 + megacore_mode = megacore_mode
296 +
297 + # NOTE(woosuk): A temporary workaround to avoid the error:
298 + # "xla::paged_attention() Expected a value of type 'str' for
299 + # argument 'megacore_mode' but instead found type 'NoneType'."
300 + if megacore_mode is not None:
301 + output = torch.ops.xla.paged_attention(
302 + query,
303 + key_cache,
304 + value_cache,
305 + context_lens,
306 + block_tables,
307 + pages_per_compute_block,
308 + megacore_mode=megacore_mode,
309 + )
310 + else:
311 + output = torch.ops.xla.paged_attention(
312 + query,
313 + key_cache,
314 + value_cache,
315 + context_lens,
316 + block_tables,
317 + pages_per_compute_block,
318 + )
319 + return output
vllm/worker/tpu_model_runner.py
+9
@@ -123,6 +123,15 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
123 )
124 self.cached_step_outputs: List[torch.Tensor] = []
125
126 + smem_size = 512 * 1024
127 + block_table_size = 4 * self.block_tables.size
128 + if block_table_size >= smem_size:
129 + logger.warning(
130 + "The max_model_len (%d) is too large. This may degrade the "
131 + "performance due to the insufficient smem size. Consider "
132 + "setting --max-model-len to a smaller value.",
133 + self.model_config.max_model_len)
134 +
135 def load_model(self) -> None:
136 self.device = self.device_config.device
137