@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 96e0c9cbbd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] directly register custom op (#9896)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Oct 31, 2024 at 21:56 UTC 96e0c9cbbd65ad0b8ad20611b90bcc86a8559aae
9 files changed +192 -67
tests/compile/piecewise/test_simple.py
+16 -4
@@ -6,18 +6,22 @@ import os
6
7 import torch
8 from torch import nn
9 +from torch.library import Library
10
11 from vllm.compilation.compile_context import set_compile_context
12 from vllm.compilation.counter import compilation_counter
13 from vllm.compilation.decorators import support_torch_compile
14 from vllm.compilation.levels import CompilationLevel
15 +from vllm.utils import direct_register_custom_op
16
17 os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
18
19 global_counter = 0
20
21 +# create a library to hold the custom op
22 +silly_lib = Library("silly", "FRAGMENT") # noqa
23 +
24
20 -@torch.library.custom_op("silly::attention", mutates_args=["out"])
25 def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
26 out: torch.Tensor) -> None:
27 global global_counter
@@ -27,12 +31,20 @@ def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
31 out[0] += 1
32
33
30 -@silly_attention.register_fake
31 -def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
32 - out: torch.Tensor) -> None:
34 +def silly_attention_fake(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
35 + out: torch.Tensor) -> None:
36 return
37
38
39 +direct_register_custom_op(
40 + op_name="attention",
41 + op_func=silly_attention,
42 + mutates_args=["out"],
43 + fake_impl=silly_attention_fake,
44 + target_lib=silly_lib,
45 +)
46 +
47 +
48 @support_torch_compile
49 class SillyModel(nn.Module):
50
tests/compile/piecewise/test_toy_llama.py
+16 -4
@@ -8,6 +8,7 @@ from typing import Optional, Tuple
8
9 import torch
10 from torch import nn
11 +from torch.library import Library
12
13 from vllm.compilation.compile_context import set_compile_context
14 from vllm.compilation.config import CompilationConfig
@@ -15,9 +16,12 @@ from vllm.compilation.counter import compilation_counter
16 from vllm.compilation.decorators import support_torch_compile
17 from vllm.compilation.levels import CompilationLevel
18 from vllm.plugins import set_compilation_config
19 +from vllm.utils import direct_register_custom_op
20 +
21 +# create a library to hold the custom op
22 +silly_lib = Library("silly", "FRAGMENT") # noqa
23
24
20 -@torch.library.custom_op("silly::attention", mutates_args=["out"])
25 def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
26 out: torch.Tensor) -> None:
27 out.copy_(q)
@@ -25,12 +29,20 @@ def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
29 out += v
30
31
28 -@silly_attention.register_fake
29 -def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
30 - out: torch.Tensor) -> None:
32 +def silly_attention_fake(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
33 + out: torch.Tensor) -> None:
34 return
35
36
37 +direct_register_custom_op(
38 + op_name="attention",
39 + op_func=silly_attention,
40 + mutates_args=["out"],
41 + fake_impl=silly_attention_fake,
42 + target_lib=silly_lib,
43 +)
44 +
45 +
46 @dataclass
47 class LlamaConfig:
48 hidden_size: int = 128
vllm/attention/backends/flash_attn.py
+11 -5
@@ -14,7 +14,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
14 compute_slot_mapping_start_idx,
15 is_block_tables_empty)
16 from vllm.forward_context import get_forward_context
17 -from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17 +from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
18 + make_tensor_with_pad)
19
20 if TYPE_CHECKING:
21 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -595,8 +596,6 @@ class FlashAttentionImpl(AttentionImpl):
596 return output
597
598
598 -@torch.library.custom_op("vllm::unified_flash_attention",
599 - mutates_args=["kv_cache"])
599 def unified_flash_attention(
600 query: torch.Tensor,
601 key: torch.Tensor,
@@ -755,8 +754,7 @@ def unified_flash_attention(
754 return output.view(num_tokens, hidden_size)
755
756
758 -@unified_flash_attention.register_fake
759 -def _(
757 +def unified_flash_attention_fake(
758 query: torch.Tensor,
759 key: torch.Tensor,
760 value: torch.Tensor,
@@ -773,3 +771,11 @@ def _(
771 logits_soft_cap: Optional[float] = None,
772 ) -> torch.Tensor:
773 return torch.empty_like(query)
774 +
775 +
776 +direct_register_custom_op(
777 + op_name="unified_flash_attention",
778 + op_func=unified_flash_attention,
779 + mutates_args=["kv_cache"],
780 + fake_impl=unified_flash_attention_fake,
781 +)
vllm/attention/backends/flashinfer.py
+11 -6
@@ -28,8 +28,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
28 is_block_tables_empty)
29 from vllm.attention.ops.paged_attn import PagedAttention
30 from vllm.forward_context import get_forward_context
31 -from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
32 - make_tensor_with_pad)
31 +from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
32 + get_kv_cache_torch_dtype, make_tensor_with_pad)
33
34 if TYPE_CHECKING:
35 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -785,8 +785,6 @@ class FlashInferImpl(AttentionImpl):
785 )
786
787
788 -@torch.library.custom_op("vllm::unified_flash_infer",
789 - mutates_args=["kv_cache"])
788 def unified_flash_infer(
789 query: torch.Tensor,
790 key: torch.Tensor,
@@ -906,8 +904,7 @@ def unified_flash_infer(
904 return output.view(num_tokens, hidden_size)
905
906
909 -@unified_flash_infer.register_fake
910 -def _(
907 +def unified_flash_infer_fake(
908 query: torch.Tensor,
909 key: torch.Tensor,
910 value: torch.Tensor,
@@ -924,3 +921,11 @@ def _(
921 logits_soft_cap: Optional[float] = None,
922 ) -> torch.Tensor:
923 return torch.empty_like(query).contiguous()
924 +
925 +
926 +direct_register_custom_op(
927 + op_name="unified_flash_infer",
928 + op_func=unified_flash_infer,
929 + mutates_args=["kv_cache"],
930 + fake_impl=unified_flash_infer_fake,
931 +)
vllm/distributed/parallel_state.py
+23 -11
@@ -37,7 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37 import vllm.envs as envs
38 from vllm.logger import init_logger
39 from vllm.platforms import current_platform
40 -from vllm.utils import supports_custom_op
40 +from vllm.utils import direct_register_custom_op, supports_custom_op
41
42
43 @dataclass
@@ -99,8 +99,6 @@ def _register_group(group: "GroupCoordinator") -> None:
99
100 if supports_custom_op():
101
102 - @torch.library.custom_op("vllm::inplace_all_reduce",
103 - mutates_args=["tensor"])
102 def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None:
103 assert group_name in _groups, f"Group {group_name} is not found."
104 group = _groups[group_name]()
@@ -108,11 +106,16 @@ if supports_custom_op():
106 raise ValueError(f"Group {group_name} is destroyed.")
107 group._all_reduce_in_place(tensor)
108
111 - @inplace_all_reduce.register_fake
112 - def _(tensor: torch.Tensor, group_name: str) -> None:
109 + def inplace_all_reduce_fake(tensor: torch.Tensor, group_name: str) -> None:
110 return
111
115 - @torch.library.custom_op("vllm::outplace_all_reduce", mutates_args=[])
112 + direct_register_custom_op(
113 + op_name="inplace_all_reduce",
114 + op_func=inplace_all_reduce,
115 + mutates_args=["tensor"],
116 + fake_impl=inplace_all_reduce_fake,
117 + )
118 +
119 def outplace_all_reduce(tensor: torch.Tensor,
120 group_name: str) -> torch.Tensor:
121 assert group_name in _groups, f"Group {group_name} is not found."
@@ -121,10 +124,17 @@ if supports_custom_op():
124 raise ValueError(f"Group {group_name} is destroyed.")
125 return group._all_reduce_out_place(tensor)
126
124 - @outplace_all_reduce.register_fake
125 - def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
127 + def outplace_all_reduce_fake(tensor: torch.Tensor,
128 + group_name: str) -> torch.Tensor:
129 return torch.empty_like(tensor)
130
131 + direct_register_custom_op(
132 + op_name="outplace_all_reduce",
133 + op_func=outplace_all_reduce,
134 + mutates_args=[],
135 + fake_impl=outplace_all_reduce_fake,
136 + )
137 +
138
139 class GroupCoordinator:
140 """
@@ -338,6 +348,11 @@ class GroupCoordinator:
348 if self.world_size == 1:
349 return input_
350
351 + if input_.is_cpu:
352 + import intel_extension_for_pytorch as ipex
353 + ipex.distributed.all_reduce(input_, group=self.device_group)
354 + return input_
355 +
356 if not supports_custom_op():
357 self._all_reduce_in_place(input_)
358 return input_
@@ -369,9 +384,6 @@ class GroupCoordinator:
384 pynccl_comm = self.pynccl_comm
385 if (pynccl_comm is not None and not pynccl_comm.disabled):
386 pynccl_comm.all_reduce(input_)
372 - elif input_.is_cpu:
373 - import intel_extension_for_pytorch as ipex
374 - ipex.distributed.all_reduce(input_, group=self.device_group)
387 else:
388 torch.distributed.all_reduce(input_, group=self.device_group)
389
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+19 -6
@@ -8,6 +8,7 @@ from vllm import _custom_ops as ops
8 from vllm.model_executor.layers.fused_moe.fused_moe import (
9 fused_topk, moe_align_block_size, try_get_optimal_moe_config)
10 from vllm.scalar_type import scalar_types
11 +from vllm.utils import direct_register_custom_op
12
13
14 def get_scalar_type(num_bits: int, has_zp: bool):
@@ -18,7 +19,6 @@ def get_scalar_type(num_bits: int, has_zp: bool):
19 return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
20
21
21 -@torch.library.custom_op("vllm::single_marlin_moe", mutates_args=[])
22 def single_marlin_moe(
23 hidden_states: torch.Tensor,
24 w: torch.Tensor,
@@ -119,8 +119,7 @@ def single_marlin_moe(
119 return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
120
121
122 -@single_marlin_moe.register_fake
123 -def _(
122 +def single_marlin_moe_fake(
123 hidden_states: torch.Tensor,
124 w: torch.Tensor,
125 scales: torch.Tensor,
@@ -136,7 +135,14 @@ def _(
135 return torch.empty_like(hidden_states)
136
137
139 -@torch.library.custom_op("vllm::fused_marlin_moe", mutates_args=[])
138 +direct_register_custom_op(
139 + op_name="single_marlin_moe",
140 + op_func=single_marlin_moe,
141 + mutates_args=[],
142 + fake_impl=single_marlin_moe_fake,
143 +)
144 +
145 +
146 def fused_marlin_moe(
147 hidden_states: torch.Tensor,
148 w1: torch.Tensor,
@@ -324,8 +330,7 @@ def fused_marlin_moe(
330 dim=1)
331
332
327 -@fused_marlin_moe.register_fake
328 -def _(
333 +def fused_marlin_moe_fake(
334 hidden_states: torch.Tensor,
335 w1: torch.Tensor,
336 w2: torch.Tensor,
@@ -344,3 +349,11 @@ def _(
349 is_k_full: bool = True,
350 ) -> torch.Tensor:
351 return torch.empty_like(hidden_states)
352 +
353 +
354 +direct_register_custom_op(
355 + op_name="fused_marlin_moe",
356 + op_func=fused_marlin_moe,
357 + mutates_args=[],
358 + fake_impl=fused_marlin_moe_fake,
359 +)
vllm/model_executor/layers/fused_moe/fused_moe.py
+41 -27
@@ -12,6 +12,7 @@ import vllm.envs as envs
12 from vllm import _custom_ops as ops
13 from vllm.logger import init_logger
14 from vllm.platforms import current_platform
15 +from vllm.utils import direct_register_custom_op
16
17 logger = init_logger(__name__)
18
@@ -466,8 +467,6 @@ def get_config_dtype_str(dtype: torch.dtype,
467 return None
468
469
469 -@torch.library.custom_op("vllm::inplace_fused_experts",
470 - mutates_args=["hidden_states"])
470 def inplace_fused_experts(hidden_states: torch.Tensor,
471 w1: torch.Tensor,
472 w2: torch.Tensor,
@@ -484,22 +483,29 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
483 a1_scale, a2_scale)
484
485
487 -@inplace_fused_experts.register_fake
488 -def _(hidden_states: torch.Tensor,
489 - w1: torch.Tensor,
490 - w2: torch.Tensor,
491 - topk_weights: torch.Tensor,
492 - topk_ids: torch.Tensor,
493 - use_fp8_w8a8: bool = False,
494 - use_int8_w8a16: bool = False,
495 - w1_scale: Optional[torch.Tensor] = None,
496 - w2_scale: Optional[torch.Tensor] = None,
497 - a1_scale: Optional[torch.Tensor] = None,
498 - a2_scale: Optional[torch.Tensor] = None) -> None:
486 +def inplace_fused_experts_fake(
487 + hidden_states: torch.Tensor,
488 + w1: torch.Tensor,
489 + w2: torch.Tensor,
490 + topk_weights: torch.Tensor,
491 + topk_ids: torch.Tensor,
492 + use_fp8_w8a8: bool = False,
493 + use_int8_w8a16: bool = False,
494 + w1_scale: Optional[torch.Tensor] = None,
495 + w2_scale: Optional[torch.Tensor] = None,
496 + a1_scale: Optional[torch.Tensor] = None,
497 + a2_scale: Optional[torch.Tensor] = None) -> None:
498 pass
499
500
502 -@torch.library.custom_op("vllm::outplace_fused_experts", mutates_args=[])
501 +direct_register_custom_op(
502 + op_name="inplace_fused_experts",
503 + op_func=inplace_fused_experts,
504 + mutates_args=["hidden_states"],
505 + fake_impl=inplace_fused_experts_fake,
506 +)
507 +
508 +
509 def outplace_fused_experts(
510 hidden_states: torch.Tensor,
511 w1: torch.Tensor,
@@ -517,21 +523,29 @@ def outplace_fused_experts(
523 w2_scale, a1_scale, a2_scale)
524
525
520 -@outplace_fused_experts.register_fake
521 -def _(hidden_states: torch.Tensor,
522 - w1: torch.Tensor,
523 - w2: torch.Tensor,
524 - topk_weights: torch.Tensor,
525 - topk_ids: torch.Tensor,
526 - use_fp8_w8a8: bool = False,
527 - use_int8_w8a16: bool = False,
528 - w1_scale: Optional[torch.Tensor] = None,
529 - w2_scale: Optional[torch.Tensor] = None,
530 - a1_scale: Optional[torch.Tensor] = None,
531 - a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
526 +def outplace_fused_experts_fake(
527 + hidden_states: torch.Tensor,
528 + w1: torch.Tensor,
529 + w2: torch.Tensor,
530 + topk_weights: torch.Tensor,
531 + topk_ids: torch.Tensor,
532 + use_fp8_w8a8: bool = False,
533 + use_int8_w8a16: bool = False,
534 + w1_scale: Optional[torch.Tensor] = None,
535 + w2_scale: Optional[torch.Tensor] = None,
536 + a1_scale: Optional[torch.Tensor] = None,
537 + a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
538 return torch.empty_like(hidden_states)
539
540
541 +direct_register_custom_op(
542 + op_name="outplace_fused_experts",
543 + op_func=outplace_fused_experts,
544 + mutates_args=[],
545 + fake_impl=outplace_fused_experts_fake,
546 +)
547 +
548 +
549 def fused_experts(hidden_states: torch.Tensor,
550 w1: torch.Tensor,
551 w2: torch.Tensor,
vllm/utils.py
+45
@@ -32,6 +32,7 @@ import torch
32 import torch.types
33 import yaml
34 from packaging.version import Version
35 +from torch.library import Library
36 from typing_extensions import ParamSpec, TypeIs, assert_never
37
38 import vllm.envs as envs
@@ -1512,3 +1513,47 @@ def weak_ref_tensors(
1513 if isinstance(tensors, tuple):
1514 return tuple(weak_ref_tensor(t) for t in tensors)
1515 raise ValueError("Invalid type for tensors")
1516 +
1517 +
1518 +def is_in_doc_build() -> bool:
1519 + try:
1520 + from sphinx.ext.autodoc.mock import _MockModule
1521 + return isinstance(torch, _MockModule)
1522 + except ModuleNotFoundError:
1523 + return False
1524 +
1525 +
1526 +# create a library to hold the custom op
1527 +vllm_lib = Library("vllm", "FRAGMENT") # noqa
1528 +
1529 +
1530 +def direct_register_custom_op(
1531 + op_name: str,
1532 + op_func: Callable,
1533 + mutates_args: List[str],
1534 + fake_impl: Optional[Callable] = None,
1535 + target_lib: Optional[Library] = None,
1536 +):
1537 + """
1538 + `torch.library.custom_op` can have significant overhead because it
1539 + needs to consider complicated dispatching logic. This function
1540 + directly registers a custom op and dispatches it to the CUDA backend.
1541 + See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5
1542 + for more details.
1543 +
1544 + By default, the custom op is registered to the vLLM library. If you
1545 + want to register it to a different library, you can pass the library
1546 + object to the `target_lib` argument.
1547 +
1548 + IMPORTANT: the lifetime of the operator is tied to the lifetime of the
1549 + library object. If you want to bind the operator to a different library,
1550 + make sure the library object is alive when the operator is used.
1551 + """
1552 + if is_in_doc_build():
1553 + return
1554 + schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
1555 + my_lib = target_lib or vllm_lib
1556 + my_lib.define(op_name + schema_str)
1557 + my_lib.impl(op_name, op_func, "CUDA")
1558 + if fake_impl is not None:
1559 + my_lib._register_fake(op_name, fake_impl)
vllm/v1/attention/backends/flash_attn.py
+10 -4
@@ -7,6 +7,7 @@ import torch
7 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 AttentionMetadata, AttentionType)
9 from vllm.forward_context import get_forward_context
10 +from vllm.utils import direct_register_custom_op
11 from vllm.vllm_flash_attn import flash_attn_varlen_func
12
13
@@ -152,8 +153,6 @@ class FlashAttentionImpl(AttentionImpl):
153 return output
154
155
155 -@torch.library.custom_op("vllm::unified_flash_attention",
156 - mutates_args=["kv_cache"])
156 def unified_flash_attention(
157 query: torch.Tensor,
158 key: torch.Tensor,
@@ -217,8 +216,7 @@ def unified_flash_attention(
216 return output.view(num_tokens, hidden_size)
217
218
220 -@unified_flash_attention.register_fake
221 -def _(
219 +def unified_flash_attention_fake(
220 query: torch.Tensor,
221 key: torch.Tensor,
222 value: torch.Tensor,
@@ -235,3 +233,11 @@ def _(
233 logits_soft_cap: Optional[float] = None,
234 ) -> torch.Tensor:
235 return torch.empty_like(query)
236 +
237 +
238 +direct_register_custom_op(
239 + op_name="unified_flash_attention",
240 + op_func=unified_flash_attention,
241 + mutates_args=["kv_cache"],
242 + fake_impl=unified_flash_attention_fake,
243 +)