@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af7380d83b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[torch.compile] fix cpu broken code (#9947)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 1, 2024 at 23:35 UTC af7380d83b0d67726a4a6c7a86766423bed6a7a8
1 file changed +8 -1
vllm/utils.py
+8 -1
@@ -1551,7 +1551,14 @@ def direct_register_custom_op(
1551 """
1552 if is_in_doc_build():
1553 return
1554 - schema_str = torch.library.infer_schema(op_func, mutates_args=mutates_args)
1554 + import torch.library
1555 + if hasattr(torch.library, "infer_schema"):
1556 + schema_str = torch.library.infer_schema(op_func,
1557 + mutates_args=mutates_args)
1558 + else:
1559 + # for pytorch 2.4
1560 + import torch._custom_op.impl
1561 + schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
1562 my_lib = target_lib or vllm_lib
1563 my_lib.define(op_name + schema_str)
1564 my_lib.impl(op_name, op_func, "CUDA")