@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 496e991da8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Doc] Consistent naming of attention backends (#9498)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Oct 21, 2024 at 16:29 UTC 496e991da82467874092e0be589071b971a63ab7
14 files changed +23 -19
vllm/attention/backends/flash_attn.py
+1 -1
@@ -32,7 +32,7 @@ class FlashAttentionBackend(AttentionBackend):
32
33 @staticmethod
34 def get_name() -> str:
35 - return "flash-attn"
35 + return "FLASH_ATTN"
36
37 @staticmethod
38 def get_impl_cls() -> Type["FlashAttentionImpl"]:
vllm/attention/backends/flashinfer.py
+1 -1
@@ -40,7 +40,7 @@ class FlashInferBackend(AttentionBackend):
40
41 @staticmethod
42 def get_name() -> str:
43 - return "flashinfer"
43 + return "FLASHINFER"
44
45 @staticmethod
46 def get_impl_cls() -> Type["FlashInferImpl"]:
vllm/attention/backends/ipex_attn.py
+1 -1
@@ -19,7 +19,7 @@ class IpexAttnBackend(AttentionBackend):
19
20 @staticmethod
21 def get_name() -> str:
22 - return "ipex-attn"
22 + return "IPEX"
23
24 @staticmethod
25 def get_impl_cls() -> Type["IpexAttnBackendImpl"]:
vllm/attention/backends/openvino.py
+1 -1
@@ -38,7 +38,7 @@ class OpenVINOAttentionBackend(AttentionBackend):
38
39 @staticmethod
40 def get_name() -> str:
41 - return "openvino"
41 + return "OPENVINO"
42
43 @staticmethod
44 def get_impl_cls():
vllm/attention/backends/pallas.py
+4
@@ -11,6 +11,10 @@ from vllm.attention.backends.utils import CommonAttentionState
11
12 class PallasAttentionBackend(AttentionBackend):
13
14 + @staticmethod
15 + def get_name() -> str:
16 + return "PALLAS"
17 +
18 @staticmethod
19 def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:
20 return PallasAttentionBackendImpl
vllm/attention/backends/placeholder_attn.py
+1 -1
@@ -20,7 +20,7 @@ class PlaceholderAttentionBackend(AttentionBackend):
20
21 @staticmethod
22 def get_name() -> str:
23 - return "placeholder-attn"
23 + return "NO_ATTENTION"
24
25 @staticmethod
26 def get_impl_cls() -> Type["PlaceholderAttentionImpl"]:
vllm/attention/backends/rocm_flash_attn.py
+1 -1
@@ -28,7 +28,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
28
29 @staticmethod
30 def get_name() -> str:
31 - return "rocm-flash-attn"
31 + return "ROCM_FLASH"
32
33 @staticmethod
34 def get_impl_cls() -> Type["ROCmFlashAttentionImpl"]:
vllm/attention/backends/torch_sdpa.py
+1 -1
@@ -25,7 +25,7 @@ class TorchSDPABackend(AttentionBackend):
25
26 @staticmethod
27 def get_name() -> str:
28 - return "torch-sdpa"
28 + return "TORCH_SDPA"
29
30 @staticmethod
31 def get_impl_cls() -> Type["TorchSDPABackendImpl"]:
vllm/attention/backends/utils.py
+6 -6
@@ -317,8 +317,8 @@ class CommonAttentionState(AttentionState):
317 if is_encoder_decoder_model:
318 # The encoder decoder model works only with XFormers backend.
319 # Assert the same.
320 - assert self.runner.attn_backend.get_name() == "xformers", \
321 - f"Expected attn_backend name to be 'xformers', but "\
320 + assert self.runner.attn_backend.get_name() == "XFORMERS", \
321 + f"Expected attn_backend name to be 'XFORMERS', but "\
322 f" got '{self.runner.attn_backend.get_name()}'"
323 self._update_captured_metadata_for_enc_dec_model(
324 batch_size=batch_size, attn_metadata=attn_metadata)
@@ -337,8 +337,8 @@ class CommonAttentionState(AttentionState):
337 if is_encoder_decoder_model:
338 # The encoder decoder model works only with XFormers backend.
339 # Assert the same.
340 - assert self.runner.attn_backend.get_name() == "xformers", \
341 - f"Expected attn_backend name to be 'xformers', but "\
340 + assert self.runner.attn_backend.get_name() == "XFORMERS", \
341 + f"Expected attn_backend name to be 'XFORMERS', but "\
342 f" got '{self.runner.attn_backend.get_name()}'"
343 self._add_additonal_input_buffers_for_enc_dec_model(
344 attn_metadata=attn_metadata, input_buffers=input_buffers)
@@ -356,8 +356,8 @@ class CommonAttentionState(AttentionState):
356 if is_encoder_decoder_model:
357 # The encoder decoder model works only with XFormers backend.
358 # Assert the same.
359 - assert self.runner.attn_backend.get_name() == "xformers", \
360 - f"Expected attn_backend name to be 'xformers', but "\
359 + assert self.runner.attn_backend.get_name() == "XFORMERS", \
360 + f"Expected attn_backend name to be 'XFORMERS', but "\
361 f" got '{self.runner.attn_backend.get_name()}'"
362 self._prepare_input_buffers_for_enc_dec_model(
363 attn_metadata, input_buffers)
vllm/attention/backends/xformers.py
+1 -1
@@ -24,7 +24,7 @@ class XFormersBackend(AttentionBackend):
24
25 @staticmethod
26 def get_name() -> str:
27 - return "xformers"
27 + return "XFORMERS"
28
29 @staticmethod
30 def get_impl_cls() -> Type["XFormersImpl"]:
vllm/spec_decode/draft_model_runner.py
+1 -1
@@ -179,7 +179,7 @@ class TP1DraftModelRunner(ModelRunner):
179 return False
180
181 # TODO: Add support for other attn backends
182 - if self.attn_backend.get_name() != "flash-attn":
182 + if self.attn_backend.get_name() != "FLASH_ATTN":
183 return False
184
185 # TODO: Add support for LORA
vllm/spec_decode/spec_decode_worker.py
+1 -1
@@ -184,7 +184,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
184
185 if not disable_mqa_scorer:
186 if scorer_worker.model_runner.attn_backend.get_name(
187 - ) != "flash-attn":
187 + ) != "FLASH_ATTN":
188 disable_mqa_scorer = True
189 logger.info(
190 "[Speculative Decoding] Disabling MQA scorer as the "
vllm/worker/model_runner.py
+1 -1
@@ -1855,7 +1855,7 @@ class CUDAGraphRunner(nn.Module):
1855 self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
1856 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1857
1858 - if self.backend_name != "placeholder-attn":
1858 + if self.backend_name != "NO_ATTENTION":
1859 self.input_buffers["slot_mapping"].copy_(
1860 attn_metadata.slot_mapping, non_blocking=True)
1861
vllm/worker/multi_step_model_runner.py
+2 -2
@@ -29,8 +29,8 @@ if TYPE_CHECKING:
29
30 logger = init_logger(__name__)
31
32 -MULTI_STEP_ATTENTION_BACKENDS = ["flash-attn", "rocm-flash-attn", "flashinfer"]
33 -MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["flash-attn"]
32 +MULTI_STEP_ATTENTION_BACKENDS = ["FLASH_ATTN", "ROCM_FLASH", "FLASHINFER"]
33 +MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["FLASH_ATTN"]
34
35 def _get_supported_attention_backends(chunked_prefill_enabled: bool) \
36 -> List[str]: