@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9738b84a08
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Force paged attention v2 for long contexts (#1510)

Antoni Baum committed Nov 1, 2023 at 16:24 UTC 9738b84a08957eb828669e8af27337ee722e8fdc
2 files changed +4 -29
vllm/model_executor/layers/attention.py
+3 -1
@@ -156,7 +156,9 @@ class PagedAttention(nn.Module):
156 # sequences or heads is large, we use V1 since there is enough work
157 # to parallelize.
158 # TODO(woosuk): Tune this heuristic.
159 - use_v1 = max_num_partitions == 1 or num_seqs * num_heads > 512
159 + # For context len > 8192, use V2 kernel to avoid shared memory shortage.
160 + use_v1 = input_metadata.max_context_len <= 8192 and (
161 + max_num_partitions == 1 or num_seqs * num_heads > 512)
162 if use_v1:
163 # Run PagedAttention V1.
164 attention_ops.paged_attention_v1(
vllm/worker/worker.py
+1 -28
@@ -13,7 +13,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
13 from vllm.sampling_params import SamplingParams, SamplingType
14 from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15 from vllm.worker.cache_engine import CacheEngine
16 -from vllm.utils import get_gpu_memory, get_max_shared_memory_bytes
16 +from vllm.utils import get_gpu_memory
17
18
19 class Worker:
@@ -141,13 +141,6 @@ class Worker:
141 self.block_size = cache_config.block_size
142 self.sliding_window = cache_config.sliding_window
143
144 - if self.sliding_window is None:
145 - max_seq_len = self.scheduler_config.max_model_len
146 - else:
147 - max_seq_len = min(self.scheduler_config.max_model_len,
148 - self.sliding_window)
149 - _check_if_can_support_max_seq_len(max_seq_len, self.block_size)
150 -
144 self.cache_engine = CacheEngine(self.cache_config, self.model_config,
145 self.parallel_config)
146 self.cache_events = self.cache_engine.events
@@ -421,26 +414,6 @@ def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
414 return x + [pad] * (max_len - len(x))
415
416
424 -def _check_if_can_support_max_seq_len(max_seq_len: int,
425 - block_size: int) -> None:
426 - # Follows the logic in
427 - # attention_kernels.cu::single_query_cached_kv_attention_launcher
428 - max_shared_mem = get_max_shared_memory_bytes()
429 - float32_bytes = torch.finfo(torch.float).bits // 8
430 - padded_max_seq_len = (
431 - (max_seq_len + block_size - 1) / block_size) * block_size
432 - # padded_max_seq_len + extra buffer
433 - required_shared_mem = (padded_max_seq_len + 512) * float32_bytes
434 - if padded_max_seq_len * float32_bytes > max_shared_mem:
435 - raise RuntimeError(
436 - f"vLLM cannot currently support max_model_len={max_seq_len} "
437 - f"with block_size={block_size} on GPU with compute "
438 - f"capability {torch.cuda.get_device_capability()} "
439 - f"(required shared memory {required_shared_mem} > "
440 - f"available shared memory {max_shared_mem}). "
441 - "This will be fixed in a future release.")
442 -
443 -
417 def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
418 # Check if the GPU supports the dtype.
419 if torch_dtype == torch.bfloat16: