A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] directly register custom op (#9896)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Oct 31, 2024 at 21:56 UTC
96e0c9cbbd65ad0b8ad20611b90bcc86a8559aae
9 files changed
+192
-67
tests/compile/piecewise/test_simple.py
+16
-4
@@ -6,18 +6,22 @@ import os
6
7
import torch
8
from torch import nn
9
+from torch.library import Library
10
11
from vllm.compilation.compile_context import set_compile_context
12
from vllm.compilation.counter import compilation_counter
13
from vllm.compilation.decorators import support_torch_compile
14
from vllm.compilation.levels import CompilationLevel
15
+from vllm.utils import direct_register_custom_op
16
17
os.environ["VLLM_TORCH_COMPILE_LEVEL"] = str(CompilationLevel.PIECEWISE)
18
19
global_counter = 0
20
21
+# create a library to hold the custom op
22
+silly_lib = Library("silly", "FRAGMENT") # noqa
23
+
24
20
-@torch.library.custom_op("silly::attention", mutates_args=["out"])
25
def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
26
out: torch.Tensor) -> None:
27
global global_counter
@@ -27,12 +31,20 @@ def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
31
out[0] += 1
32
33
30
-@silly_attention.register_fake
31
-def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
32
- out: torch.Tensor) -> None:
34
+def silly_attention_fake(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
35
+ out: torch.Tensor) -> None:
36
return
37
38
39
+direct_register_custom_op(
40
+ op_name="attention",
41
+ op_func=silly_attention,
42
+ mutates_args=["out"],
43
+ fake_impl=silly_attention_fake,
44
+ target_lib=silly_lib,
45
+)
46
+
47
+
48
@support_torch_compile
49
class SillyModel(nn.Module):
50
tests/compile/piecewise/test_toy_llama.py
+16
-4
@@ -8,6 +8,7 @@ from typing import Optional, Tuple
8
9
import torch
10
from torch import nn
11
+from torch.library import Library
12
13
from vllm.compilation.compile_context import set_compile_context
14
from vllm.compilation.config import CompilationConfig
@@ -15,9 +16,12 @@ from vllm.compilation.counter import compilation_counter
16
from vllm.compilation.decorators import support_torch_compile
17
from vllm.compilation.levels import CompilationLevel
18
from vllm.plugins import set_compilation_config
19
+from vllm.utils import direct_register_custom_op
20
+
21
+# create a library to hold the custom op
22
+silly_lib = Library("silly", "FRAGMENT") # noqa
23
24
20
-@torch.library.custom_op("silly::attention", mutates_args=["out"])
25
def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
26
out: torch.Tensor) -> None:
27
out.copy_(q)
@@ -25,12 +29,20 @@ def silly_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
29
out += v
30
31
28
-@silly_attention.register_fake
29
-def _(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
30
- out: torch.Tensor) -> None:
32
+def silly_attention_fake(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
33
+ out: torch.Tensor) -> None:
34
return
35
36
37
+direct_register_custom_op(
38
+ op_name="attention",
39
+ op_func=silly_attention,
40
+ mutates_args=["out"],
41
+ fake_impl=silly_attention_fake,
42
+ target_lib=silly_lib,
43
+)
44
+
45
+
46
@dataclass
47
class LlamaConfig:
48
hidden_size: int = 128
vllm/attention/backends/flash_attn.py
+11
-5
@@ -14,7 +14,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
14
compute_slot_mapping_start_idx,
15
is_block_tables_empty)
16
from vllm.forward_context import get_forward_context
17
-from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17
+from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
18
+ make_tensor_with_pad)
19
20
if TYPE_CHECKING:
21
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -595,8 +596,6 @@ class FlashAttentionImpl(AttentionImpl):
596
return output
597
598
598
-@torch.library.custom_op("vllm::unified_flash_attention",
599
- mutates_args=["kv_cache"])
599
def unified_flash_attention(
600
query: torch.Tensor,
601
key: torch.Tensor,
@@ -755,8 +754,7 @@ def unified_flash_attention(
754
return output.view(num_tokens, hidden_size)
755
756
758
-@unified_flash_attention.register_fake
759
-def _(
757
+def unified_flash_attention_fake(
758
query: torch.Tensor,
759
key: torch.Tensor,
760
value: torch.Tensor,
@@ -773,3 +771,11 @@ def _(
771
logits_soft_cap: Optional[float] = None,
772
) -> torch.Tensor:
773
return torch.empty_like(query)
774
+
775
+
776
+direct_register_custom_op(
777
+ op_name="unified_flash_attention",
778
+ op_func=unified_flash_attention,
779
+ mutates_args=["kv_cache"],
780
+ fake_impl=unified_flash_attention_fake,
781
+)
vllm/attention/backends/flashinfer.py
+11
-6
@@ -28,8 +28,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
28
is_block_tables_empty)
29
from vllm.attention.ops.paged_attn import PagedAttention
30
from vllm.forward_context import get_forward_context
31
-from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
32
- make_tensor_with_pad)
31
+from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
32
+ get_kv_cache_torch_dtype, make_tensor_with_pad)
33
34
if TYPE_CHECKING:
35
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -785,8 +785,6 @@ class FlashInferImpl(AttentionImpl):
785
)
786
787
788
-@torch.library.custom_op("vllm::unified_flash_infer",
789
- mutates_args=["kv_cache"])
788
def unified_flash_infer(
789
query: torch.Tensor,
790
key: torch.Tensor,
@@ -906,8 +904,7 @@ def unified_flash_infer(
904
return output.view(num_tokens, hidden_size)
905
906
909
-@unified_flash_infer.register_fake
910
-def _(
907
+def unified_flash_infer_fake(
908
query: torch.Tensor,
909
key: torch.Tensor,
910
value: torch.Tensor,
@@ -924,3 +921,11 @@ def _(
921
logits_soft_cap: Optional[float] = None,
922
) -> torch.Tensor:
923
return torch.empty_like(query).contiguous()
924
+
925
+
926
+direct_register_custom_op(
927
+ op_name="unified_flash_infer",
928
+ op_func=unified_flash_infer,
929
+ mutates_args=["kv_cache"],
930
+ fake_impl=unified_flash_infer_fake,
931
+)
vllm/distributed/parallel_state.py
+23
-11
@@ -37,7 +37,7 @@ from torch.distributed import Backend, ProcessGroup
37
import vllm.envs as envs
38
from vllm.logger import init_logger
39
from vllm.platforms import current_platform
40
-from vllm.utils import supports_custom_op
40
+from vllm.utils import direct_register_custom_op, supports_custom_op
41
42
43
@dataclass
@@ -99,8 +99,6 @@ def _register_group(group: "GroupCoordinator") -> None:
99
100
if supports_custom_op():
101
102
- @torch.library.custom_op("vllm::inplace_all_reduce",
103
- mutates_args=["tensor"])
102
def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None:
103
assert group_name in _groups, f"Group {group_name} is not found."
104
group = _groups[group_name]()
@@ -108,11 +106,16 @@ if supports_custom_op():
106
raise ValueError(f"Group {group_name} is destroyed.")
107
group._all_reduce_in_place(tensor)
108
111
- @inplace_all_reduce.register_fake
112
- def _(tensor: torch.Tensor, group_name: str) -> None:
109
+ def inplace_all_reduce_fake(tensor: torch.Tensor, group_name: str) -> None:
110
return
111
115
- @torch.library.custom_op("vllm::outplace_all_reduce", mutates_args=[])
112
+ direct_register_custom_op(
113
+ op_name="inplace_all_reduce",
114
+ op_func=inplace_all_reduce,
115
+ mutates_args=["tensor"],
116
+ fake_impl=inplace_all_reduce_fake,
117
+ )
118
+
119
def outplace_all_reduce(tensor: torch.Tensor,
120
group_name: str) -> torch.Tensor:
121
assert group_name in _groups, f"Group {group_name} is not found."
@@ -121,10 +124,17 @@ if supports_custom_op():
124
raise ValueError(f"Group {group_name} is destroyed.")
125
return group._all_reduce_out_place(tensor)
126
124
- @outplace_all_reduce.register_fake
125
- def _(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
127
+ def outplace_all_reduce_fake(tensor: torch.Tensor,
128
+ group_name: str) -> torch.Tensor:
129
return torch.empty_like(tensor)
130
131
+ direct_register_custom_op(
132
+ op_name="outplace_all_reduce",
133
+ op_func=outplace_all_reduce,
134
+ mutates_args=[],
135
+ fake_impl=outplace_all_reduce_fake,
136
+ )
137
+
138
139
class GroupCoordinator:
140
"""
@@ -338,6 +348,11 @@ class GroupCoordinator:
348
if self.world_size == 1:
349
return input_
350
351
+ if input_.is_cpu:
352
+ import intel_extension_for_pytorch as ipex
353
+ ipex.distributed.all_reduce(input_, group=self.device_group)
354
+ return input_
355
+
356
if not supports_custom_op():
357
self._all_reduce_in_place(input_)
358
return input_
@@ -369,9 +384,6 @@ class GroupCoordinator:
384
pynccl_comm = self.pynccl_comm
385
if (pynccl_comm is not None and not pynccl_comm.disabled):
386
pynccl_comm.all_reduce(input_)
372
- elif input_.is_cpu:
373
- import intel_extension_for_pytorch as ipex
374
- ipex.distributed.all_reduce(input_, group=self.device_group)
387
else:
388
torch.distributed.all_reduce(input_, group=self.device_group)
389
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+19
-6
@@ -8,6 +8,7 @@ from vllm import _custom_ops as ops
8
from vllm.model_executor.layers.fused_moe.fused_moe import (
9
fused_topk, moe_align_block_size, try_get_optimal_moe_config)
10
from vllm.scalar_type import scalar_types
11
+from vllm.utils import direct_register_custom_op
12
13
14
def get_scalar_type(num_bits: int, has_zp: bool):
@@ -18,7 +19,6 @@ def get_scalar_type(num_bits: int, has_zp: bool):
19
return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128
20
21
21
-@torch.library.custom_op("vllm::single_marlin_moe", mutates_args=[])
22
def single_marlin_moe(
23
hidden_states: torch.Tensor,
24
w: torch.Tensor,
@@ -119,8 +119,7 @@ def single_marlin_moe(
119
return torch.sum(intermediate_cache.view(*intermediate_cache.shape), dim=1)
120
121
122
-@single_marlin_moe.register_fake
123
-def _(
122
+def single_marlin_moe_fake(
123
hidden_states: torch.Tensor,
124
w: torch.Tensor,
125
scales: torch.Tensor,
@@ -136,7 +135,14 @@ def _(
135
return torch.empty_like(hidden_states)
136
137
139
-@torch.library.custom_op("vllm::fused_marlin_moe", mutates_args=[])
138
+direct_register_custom_op(
139
+ op_name="single_marlin_moe",
140
+ op_func=single_marlin_moe,
141
+ mutates_args=[],
142
+ fake_impl=single_marlin_moe_fake,
143
+)
144
+
145
+
146
def fused_marlin_moe(
147
hidden_states: torch.Tensor,
148
w1: torch.Tensor,
@@ -324,8 +330,7 @@ def fused_marlin_moe(
330
dim=1)
331
332
327
-@fused_marlin_moe.register_fake
328
-def _(
333
+def fused_marlin_moe_fake(
334
hidden_states: torch.Tensor,
335
w1: torch.Tensor,
336
w2: torch.Tensor,
@@ -344,3 +349,11 @@ def _(
349
is_k_full: bool = True,
350
) -> torch.Tensor:
351
return torch.empty_like(hidden_states)
352
+
353
+
354
+direct_register_custom_op(
355
+ op_name="fused_marlin_moe",
356
+ op_func=fused_marlin_moe,
357
+ mutates_args=[],
358
+ fake_impl=fused_marlin_moe_fake,
359
+)
vllm/model_executor/layers/fused_moe/fused_moe.py
+41
-27
@@ -12,6 +12,7 @@ import vllm.envs as envs
12
from vllm import _custom_ops as ops
13
from vllm.logger import init_logger
14
from vllm.platforms import current_platform
15
+from vllm.utils import direct_register_custom_op
16
17
logger = init_logger(__name__)
18
@@ -466,8 +467,6 @@ def get_config_dtype_str(dtype: torch.dtype,
467
return None
468
469
469
-@torch.library.custom_op("vllm::inplace_fused_experts",
470
- mutates_args=["hidden_states"])
470
def inplace_fused_experts(hidden_states: torch.Tensor,
471
w1: torch.Tensor,
472
w2: torch.Tensor,
@@ -484,22 +483,29 @@ def inplace_fused_experts(hidden_states: torch.Tensor,
483
a1_scale, a2_scale)
484
485
487
-@inplace_fused_experts.register_fake
488
-def _(hidden_states: torch.Tensor,
489
- w1: torch.Tensor,
490
- w2: torch.Tensor,
491
- topk_weights: torch.Tensor,
492
- topk_ids: torch.Tensor,
493
- use_fp8_w8a8: bool = False,
494
- use_int8_w8a16: bool = False,
495
- w1_scale: Optional[torch.Tensor] = None,
496
- w2_scale: Optional[torch.Tensor] = None,
497
- a1_scale: Optional[torch.Tensor] = None,
498
- a2_scale: Optional[torch.Tensor] = None) -> None:
486
+def inplace_fused_experts_fake(
487
+ hidden_states: torch.Tensor,
488
+ w1: torch.Tensor,
489
+ w2: torch.Tensor,
490
+ topk_weights: torch.Tensor,
491
+ topk_ids: torch.Tensor,
492
+ use_fp8_w8a8: bool = False,
493
+ use_int8_w8a16: bool = False,
494
+ w1_scale: Optional[torch.Tensor] = None,
495
+ w2_scale: Optional[torch.Tensor] = None,
496
+ a1_scale: Optional[torch.Tensor] = None,
497
+ a2_scale: Optional[torch.Tensor] = None) -> None:
498
pass
499
500
502
-@torch.library.custom_op("vllm::outplace_fused_experts", mutates_args=[])
501
+direct_register_custom_op(
502
+ op_name="inplace_fused_experts",
503
+ op_func=inplace_fused_experts,
504
+ mutates_args=["hidden_states"],
505
+ fake_impl=inplace_fused_experts_fake,
506
+)
507
+
508
+
509
def outplace_fused_experts(
510
hidden_states: torch.Tensor,
511
w1: torch.Tensor,
@@ -517,21 +523,29 @@ def outplace_fused_experts(
523
w2_scale, a1_scale, a2_scale)
524
525
520
-@outplace_fused_experts.register_fake
521
-def _(hidden_states: torch.Tensor,
522
- w1: torch.Tensor,
523
- w2: torch.Tensor,
524
- topk_weights: torch.Tensor,
525
- topk_ids: torch.Tensor,
526
- use_fp8_w8a8: bool = False,
527
- use_int8_w8a16: bool = False,
528
- w1_scale: Optional[torch.Tensor] = None,
529
- w2_scale: Optional[torch.Tensor] = None,
530
- a1_scale: Optional[torch.Tensor] = None,
531
- a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
526
+def outplace_fused_experts_fake(
527
+ hidden_states: torch.Tensor,
528
+ w1: torch.Tensor,
529
+ w2: torch.Tensor,
530
+ topk_weights: torch.Tensor,
531
+ topk_ids: torch.Tensor,
532
+ use_fp8_w8a8: bool = False,
533
+ use_int8_w8a16: bool = False,
534
+ w1_scale: Optional[torch.Tensor] = None,
535
+ w2_scale: Optional[torch.Tensor] = None,
536
+ a1_scale: Optional[torch.Tensor] = None,
537
+ a2_scale: Optional[torch.Tensor] = None) -> torch.Tensor:
538
return torch.empty_like(hidden_states)
539
540
541
+direct_register_custom_op(
542
+ op_name="outplace_fused_experts",
543
+ op_func=outplace_fused_experts,
544
+ mutates_args=[],
545
+ fake_impl=outplace_fused_experts_fake,
546
+)
547
+
548
+
549
def fused_experts(hidden_states: torch.Tensor,
550
w1: torch.Tensor,
551
w2: torch.Tensor,
vllm/utils.py
+45
@@ -32,6 +32,7 @@ import torch
32
import torch.types
33
import yaml
34
from packaging.version import Version
35
+from torch.library import Library
36
from typing_extensions import ParamSpec, TypeIs, assert_never
37
38
import vllm.envs as envs
@@ -1512,3 +1513,47 @@ def weak_ref_tensors(
1513
if isinstance(tensors, tuple):
1514
return tuple(weak_ref_tensor(t) for t in tensors)
1515
raise ValueError("Invalid type for tensors")
1516
+
1517
+
1518
+def is_in_doc_build() -> bool:
1519
+ try:
1520
+ from sphinx.ext.autodoc.mock import _MockModule
1521
+ return isinstance(torch, _MockModule)
1522
+ except ModuleNotFoundError:
1523
+ return False
1524
+
1525
+
1526
+# create a library to hold the custom op
1527
+vllm_lib = Library("vllm", "FRAGMENT") # noqa
1528
+
1529
+
1530
+def direct_register_custom_op(
1531
+ op_name: str,
1532
+ op_func: Callable,
1533
+ mutates_args: List[str],
1534
+ fake_impl: Optional[Callable] = None,
1535
+ target_lib: Optional[Library] = None,
1536
+):
1537
+ """
1538
+ `torch.library.custom_op` can have significant overhead because it
1539
+ needs to consider complicated dispatching logic. This function
1540
+ directly registers a custom op and dispatches it to the CUDA backend.
1541
+ See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5
1542
+ for more details.
1543
+
1544
+ By default, the custom op is registered to the vLLM library. If you
1545
+ want to register it to a different library, you can pass the library
1546
+ object to the `target_lib` argument.
1547
+
1548
+ IMPORTANT: the lifetime of the operator is tied to the lifetime of the
1549
+ library object. If you want to bind the operator to a different library,
1550
+ make sure the library object is alive when the operator is used.
1551
+ """
1552
+ if is_in_doc_build():
1553
+ return
1554
+ schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
1555
+ my_lib = target_lib or vllm_lib
1556
+ my_lib.define(op_name + schema_str)
1557
+ my_lib.impl(op_name, op_func, "CUDA")
1558
+ if fake_impl is not None:
1559
+ my_lib._register_fake(op_name, fake_impl)
vllm/v1/attention/backends/flash_attn.py
+10
-4
@@ -7,6 +7,7 @@ import torch
7
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8
AttentionMetadata, AttentionType)
9
from vllm.forward_context import get_forward_context
10
+from vllm.utils import direct_register_custom_op
11
from vllm.vllm_flash_attn import flash_attn_varlen_func
12
13
@@ -152,8 +153,6 @@ class FlashAttentionImpl(AttentionImpl):
153
return output
154
155
155
-@torch.library.custom_op("vllm::unified_flash_attention",
156
- mutates_args=["kv_cache"])
156
def unified_flash_attention(
157
query: torch.Tensor,
158
key: torch.Tensor,
@@ -217,8 +216,7 @@ def unified_flash_attention(
216
return output.view(num_tokens, hidden_size)
217
218
220
-@unified_flash_attention.register_fake
221
-def _(
219
+def unified_flash_attention_fake(
220
query: torch.Tensor,
221
key: torch.Tensor,
222
value: torch.Tensor,
@@ -235,3 +233,11 @@ def _(
233
logits_soft_cap: Optional[float] = None,
234
) -> torch.Tensor:
235
return torch.empty_like(query)
236
+
237
+
238
+direct_register_custom_op(
239
+ op_name="unified_flash_attention",
240
+ op_func=unified_flash_attention,
241
+ mutates_args=["kv_cache"],
242
+ fake_impl=unified_flash_attention_fake,
243
+)