A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Doc] Consistent naming of attention backends (#9498)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Oct 21, 2024 at 16:29 UTC
496e991da82467874092e0be589071b971a63ab7
14 files changed
+23
-19
vllm/attention/backends/flash_attn.py
+1
-1
@@ -32,7 +32,7 @@ class FlashAttentionBackend(AttentionBackend):
32
33
@staticmethod
34
def get_name() -> str:
35
- return "flash-attn"
35
+ return "FLASH_ATTN"
36
37
@staticmethod
38
def get_impl_cls() -> Type["FlashAttentionImpl"]:
vllm/attention/backends/flashinfer.py
+1
-1
@@ -40,7 +40,7 @@ class FlashInferBackend(AttentionBackend):
40
41
@staticmethod
42
def get_name() -> str:
43
- return "flashinfer"
43
+ return "FLASHINFER"
44
45
@staticmethod
46
def get_impl_cls() -> Type["FlashInferImpl"]:
vllm/attention/backends/ipex_attn.py
+1
-1
@@ -19,7 +19,7 @@ class IpexAttnBackend(AttentionBackend):
19
20
@staticmethod
21
def get_name() -> str:
22
- return "ipex-attn"
22
+ return "IPEX"
23
24
@staticmethod
25
def get_impl_cls() -> Type["IpexAttnBackendImpl"]:
vllm/attention/backends/openvino.py
+1
-1
@@ -38,7 +38,7 @@ class OpenVINOAttentionBackend(AttentionBackend):
38
39
@staticmethod
40
def get_name() -> str:
41
- return "openvino"
41
+ return "OPENVINO"
42
43
@staticmethod
44
def get_impl_cls():
vllm/attention/backends/pallas.py
+4
@@ -11,6 +11,10 @@ from vllm.attention.backends.utils import CommonAttentionState
11
12
class PallasAttentionBackend(AttentionBackend):
13
14
+ @staticmethod
15
+ def get_name() -> str:
16
+ return "PALLAS"
17
+
18
@staticmethod
19
def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:
20
return PallasAttentionBackendImpl
vllm/attention/backends/placeholder_attn.py
+1
-1
@@ -20,7 +20,7 @@ class PlaceholderAttentionBackend(AttentionBackend):
20
21
@staticmethod
22
def get_name() -> str:
23
- return "placeholder-attn"
23
+ return "NO_ATTENTION"
24
25
@staticmethod
26
def get_impl_cls() -> Type["PlaceholderAttentionImpl"]:
vllm/attention/backends/rocm_flash_attn.py
+1
-1
@@ -28,7 +28,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
28
29
@staticmethod
30
def get_name() -> str:
31
- return "rocm-flash-attn"
31
+ return "ROCM_FLASH"
32
33
@staticmethod
34
def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]:
vllm/attention/backends/torch_sdpa.py
+1
-1
@@ -25,7 +25,7 @@ class TorchSDPABackend(AttentionBackend):
25
26
@staticmethod
27
def get_name() -> str:
28
- return "torch-sdpa"
28
+ return "TORCH_SDPA"
29
30
@staticmethod
31
def get_impl_cls() -> Type["TorchSDPABackendImpl"]:
vllm/attention/backends/utils.py
+6
-6
@@ -317,8 +317,8 @@ class CommonAttentionState(AttentionState):
317
if is_encoder_decoder_model:
318
# The encoder decoder model works only with XFormers backend.
319
# Assert the same.
320
- assert self.runner.attn_backend.get_name() == "xformers", \
321
- f"Expected attn_backend name to be 'xformers', but "\
320
+ assert self.runner.attn_backend.get_name() == "XFORMERS", \
321
+ f"Expected attn_backend name to be 'XFORMERS', but "\
322
f" got '{self.runner.attn_backend.get_name()}'"
323
self._update_captured_metadata_for_enc_dec_model(
324
batch_size=batch_size, attn_metadata=attn_metadata)
@@ -337,8 +337,8 @@ class CommonAttentionState(AttentionState):
337
if is_encoder_decoder_model:
338
# The encoder decoder model works only with XFormers backend.
339
# Assert the same.
340
- assert self.runner.attn_backend.get_name() == "xformers", \
341
- f"Expected attn_backend name to be 'xformers', but "\
340
+ assert self.runner.attn_backend.get_name() == "XFORMERS", \
341
+ f"Expected attn_backend name to be 'XFORMERS', but "\
342
f" got '{self.runner.attn_backend.get_name()}'"
343
self._add_additonal_input_buffers_for_enc_dec_model(
344
attn_metadata=attn_metadata, input_buffers=input_buffers)
@@ -356,8 +356,8 @@ class CommonAttentionState(AttentionState):
356
if is_encoder_decoder_model:
357
# The encoder decoder model works only with XFormers backend.
358
# Assert the same.
359
- assert self.runner.attn_backend.get_name() == "xformers", \
360
- f"Expected attn_backend name to be 'xformers', but "\
359
+ assert self.runner.attn_backend.get_name() == "XFORMERS", \
360
+ f"Expected attn_backend name to be 'XFORMERS', but "\
361
f" got '{self.runner.attn_backend.get_name()}'"
362
self._prepare_input_buffers_for_enc_dec_model(
363
attn_metadata, input_buffers)
vllm/attention/backends/xformers.py
+1
-1
@@ -24,7 +24,7 @@ class XFormersBackend(AttentionBackend):
24
25
@staticmethod
26
def get_name() -> str:
27
- return "xformers"
27
+ return "XFORMERS"
28
29
@staticmethod
30
def get_impl_cls() -> Type["XFormersImpl"]:
vllm/spec_decode/draft_model_runner.py
+1
-1
@@ -179,7 +179,7 @@ class TP1DraftModelRunner(ModelRunner):
179
return False
180
181
# TODO: Add support for other attn backends
182
- if self.attn_backend.get_name() != "flash-attn":
182
+ if self.attn_backend.get_name() != "FLASH_ATTN":
183
return False
184
185
# TODO: Add support for LORA
vllm/spec_decode/spec_decode_worker.py
+1
-1
@@ -184,7 +184,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
184
185
if not disable_mqa_scorer:
186
if scorer_worker.model_runner.attn_backend.get_name(
187
- ) != "flash-attn":
187
+ ) != "FLASH_ATTN":
188
disable_mqa_scorer = True
189
logger.info(
190
"[Speculative Decoding] Disabling MQA scorer as the "
vllm/worker/model_runner.py
+1
-1
@@ -1855,7 +1855,7 @@ class CUDAGraphRunner(nn.Module):
1855
self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
1856
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1857
1858
- if self.backend_name != "placeholder-attn":
1858
+ if self.backend_name != "NO_ATTENTION":
1859
self.input_buffers["slot_mapping"].copy_(
1860
attn_metadata.slot_mapping, non_blocking=True)
1861
vllm/worker/multi_step_model_runner.py
+2
-2
@@ -29,8 +29,8 @@ if TYPE_CHECKING:
29
30
logger = init_logger(__name__)
31
32
-MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "rocm-flash-attn", "flashinfer"]
33
-MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["flash-attn"]
32
+MULTI_STEP_ATTENTION_BACKENDS = ["FLASH_ATTN", "ROCM_FLASH", "FLASHINFER"]
33
+MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["FLASH_ATTN"]
34
35
def _get_supported_attention_backends(chunked_prefill_enabled: bool) \
36
-> List[str]: