A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[torch.compile] fix cpu broken code (#9947)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 1, 2024 at 23:35 UTC
af7380d83b0d67726a4a6c7a86766423bed6a7a8
1 file changed
+8
-1
vllm/utils.py
+8
-1
@@ -1551,7 +1551,14 @@ def direct_register_custom_op(
1551
"""
1552
if is_in_doc_build():
1553
return
1554
- schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
1554
+ import torch.library
1555
+ if hasattr(torch.library, "infer_schema"):
1556
+ schema_str = torch.library.infer_schema(op_func,
1557
+ mutates_args=mutates_args)
1558
+ else:
1559
+ # for pytorch 2.4
1560
+ import torch._custom_op.impl
1561
+ schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
1562
my_lib = target_lib or vllm_lib
1563
my_lib.define(op_name + schema_str)
1564
my_lib.impl(op_name, op_func, "CUDA")