A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[misc] add hint for AttributeError (#5462)
youkaichao committed
Jun 12, 2024 at 14:46 UTC
622d45128c02e5296e1177481c65199754eab396
1 file changed
+46
-2
vllm/_custom_ops.py
+46
-2
@@ -1,13 +1,16 @@
1
import contextlib
2
+import functools
3
from typing import List, Optional, Tuple, Type
4
5
import torch
6
7
+from vllm.logger import init_logger
8
+
9
+logger = init_logger(__name__)
10
+
11
try:
12
import vllm._C
13
except ImportError as e:
9
- from vllm.logger import init_logger
10
- logger = init_logger(__name__)
14
logger.warning("Failed to import from vllm._C with %r", e)
15
16
with contextlib.suppress(ImportError):
@@ -23,6 +26,25 @@ def is_custom_op_supported(op_name: str) -> bool:
26
return op is not None
27
28
29
+def hint_on_error(fn):
30
+
31
+ @functools.wraps(fn)
32
+ def wrapper(*args, **kwargs):
33
+ try:
34
+ return fn(*args, **kwargs)
35
+ except AttributeError as e:
36
+ msg = (
37
+ "Error in calling custom op %s: %s\n"
38
+ "Possibly you have built or installed an obsolete version of vllm.\n"
39
+ "Please try a clean build and install of vllm,"
40
+ "or remove old built files such as vllm/*cpython*.so and build/ ."
41
+ )
42
+ logger.error(msg, fn.__name__, e)
43
+ raise e
44
+
45
+ return wrapper
46
+
47
+
48
# activation ops
49
def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
50
torch.ops._C.silu_and_mul(out, x)
@@ -459,3 +481,25 @@ def dispatch_bgmv_low_level(
481
h_out,
482
y_offset,
483
)
484
+
485
+
486
+# temporary fix for https://github.com/vllm-project/vllm/issues/5456
487
+# TODO: remove this in v0.6.0
488
+names_and_values = globals()
489
+names_and_values_to_update = {}
490
+# prepare variables to avoid dict size change during iteration
491
+k, v, arg = None, None, None
492
+fn_type = type(lambda x: x)
493
+for k, v in names_and_values.items():
494
+ # find functions that are defined in this file and have torch.Tensor
495
+ # in their annotations. `arg == "torch.Tensor"` is used to handle
496
+ # the case when users use `import __annotations__` to turn type
497
+ # hints into strings.
498
+ if isinstance(v, fn_type) \
499
+ and v.__code__.co_filename == __file__ \
500
+ and any(arg is torch.Tensor or arg == "torch.Tensor"
501
+ for arg in v.__annotations__.values()):
502
+ names_and_values_to_update[k] = hint_on_error(v)
503
+
504
+names_and_values.update(names_and_values_to_update)
505
+del names_and_values_to_update, names_and_values, v, k, fn_type