A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Force paged attention v2 for long contexts (#1510)
Antoni Baum committed
Nov 1, 2023 at 16:24 UTC
9738b84a08957eb828669e8af27337ee722e8fdc
2 files changed
+4
-29
vllm/model_executor/layers/attention.py
+3
-1
@@ -156,7 +156,9 @@ class PagedAttention(nn.Module):
156
# sequences or heads is large, we use V1 since there is enough work
157
# to parallelize.
158
# TODO(woosuk): Tune this heuristic.
159
- use_v1 = max_num_partitions == 1 or num_seqs * num_heads > 512
159
+ # For context len > 8192, use V2 kernel to avoid shared memory shortage.
160
+ use_v1 = input_metadata.max_context_len <= 8192 and (
161
+ max_num_partitions == 1 or num_seqs * num_heads > 512)
162
if use_v1:
163
# Run PagedAttention V1.
164
attention_ops.paged_attention_v1(
vllm/worker/worker.py
+1
-28
@@ -13,7 +13,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
13
from vllm.sampling_params import SamplingParams, SamplingType
14
from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
15
from vllm.worker.cache_engine import CacheEngine
16
-from vllm.utils import get_gpu_memory, get_max_shared_memory_bytes
16
+from vllm.utils import get_gpu_memory
17
18
19
class Worker:
@@ -141,13 +141,6 @@ class Worker:
141
self.block_size = cache_config.block_size
142
self.sliding_window = cache_config.sliding_window
143
144
- if self.sliding_window is None:
145
- max_seq_len = self.scheduler_config.max_model_len
146
- else:
147
- max_seq_len = min(self.scheduler_config.max_model_len,
148
- self.sliding_window)
149
- _check_if_can_support_max_seq_len(max_seq_len, self.block_size)
150
-
144
self.cache_engine = CacheEngine(self.cache_config, self.model_config,
145
self.parallel_config)
146
self.cache_events = self.cache_engine.events
@@ -421,26 +414,6 @@ def _pad_to_max(x: List[int], max_len: int, pad: int) -> List[int]:
414
return x + [pad] * (max_len - len(x))
415
416
424
-def _check_if_can_support_max_seq_len(max_seq_len: int,
425
- block_size: int) -> None:
426
- # Follows the logic in
427
- # attention_kernels.cu::single_query_cached_kv_attention_launcher
428
- max_shared_mem = get_max_shared_memory_bytes()
429
- float32_bytes = torch.finfo(torch.float).bits // 8
430
- padded_max_seq_len = (
431
- (max_seq_len + block_size - 1) / block_size) * block_size
432
- # padded_max_seq_len + extra buffer
433
- required_shared_mem = (padded_max_seq_len + 512) * float32_bytes
434
- if padded_max_seq_len * float32_bytes > max_shared_mem:
435
- raise RuntimeError(
436
- f"vLLM cannot currently support max_model_len={max_seq_len} "
437
- f"with block_size={block_size} on GPU with compute "
438
- f"capability {torch.cuda.get_device_capability()} "
439
- f"(required shared memory {required_shared_mem} > "
440
- f"available shared memory {max_shared_mem}). "
441
- "This will be fixed in a future release.")
442
-
443
-
417
def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
418
# Check if the GPU supports the dtype.
419
if torch_dtype == torch.bfloat16: