A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[misc] add early error message for custom ops (#11355)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 19, 2024 at 21:18 UTC
c954f21ac05642c416cbd87861ddebe9af2ae1b4
1 file changed
+11
-1
vllm/utils.py
+11
-1
@@ -1577,8 +1577,18 @@ def direct_register_custom_op(
1577
library object. If you want to bind the operator to a different library,
1578
make sure the library object is alive when the operator is used.
1579
"""
1580
- if is_in_doc_build() or not supports_custom_op():
1580
+ if is_in_doc_build():
1581
return
1582
+
1583
+ if not supports_custom_op():
1584
+ assert not current_platform.is_cuda_alike(), (
1585
+ "cuda platform needs torch>=2.4 to support custom op, "
1586
+ "chances are you are using an old version of pytorch "
1587
+ "or a custom build of pytorch. It is recommended to "
1588
+ "use vLLM in a fresh new environment and let it install "
1589
+ "the required dependencies.")
1590
+ return
1591
+
1592
import torch.library
1593
if hasattr(torch.library, "infer_schema"):
1594
schema_str = torch.library.infer_schema(op_func,