@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 622d45128c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[misc] add hint for AttributeError (#5462)

youkaichao committed Jun 12, 2024 at 14:46 UTC 622d45128c02e5296e1177481c65199754eab396
1 file changed +46 -2
vllm/_custom_ops.py
+46 -2
@@ -1,13 +1,16 @@
1 import contextlib
2 +import functools
3 from typing import List, Optional, Tuple, Type
4
5 import torch
6
7 +from vllm.logger import init_logger
8 +
9 +logger = init_logger(__name__)
10 +
11 try:
12 import vllm._C
13 except ImportError as e:
9 - from vllm.logger import init_logger
10 - logger = init_logger(__name__)
14 logger.warning("Failed to import from vllm._C with %r", e)
15
16 with contextlib.suppress(ImportError):
@@ -23,6 +26,25 @@ def is_custom_op_supported(op_name: str) -> bool:
26 return op is not None
27
28
29 +def hint_on_error(fn):
30 +
31 + @functools.wraps(fn)
32 + def wrapper(*args, **kwargs):
33 + try:
34 + return fn(*args, **kwargs)
35 + except AttributeError as e:
36 + msg = (
37 + "Error in calling custom op %s: %s\n"
38 + "Possibly you have built or installed an obsolete version of vllm.\n"
39 + "Please try a clean build and install of vllm,"
40 + "or remove old built files such as vllm/*cpython*.so and build/ ."
41 + )
42 + logger.error(msg, fn.__name__, e)
43 + raise e
44 +
45 + return wrapper
46 +
47 +
48 # activation ops
49 def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
50 torch.ops._C.silu_and_mul(out, x)
@@ -459,3 +481,25 @@ def dispatch_bgmv_low_level(
481 h_out,
482 y_offset,
483 )
484 +
485 +
486 +# temporary fix for https://github.com/vllm-project/vllm/issues/5456
487 +# TODO: remove this in v0.6.0
488 +names_and_values = globals()
489 +names_and_values_to_update = {}
490 +# prepare variables to avoid dict size change during iteration
491 +k, v, arg = None, None, None
492 +fn_type = type(lambda x: x)
493 +for k, v in names_and_values.items():
494 + # find functions that are defined in this file and have torch.Tensor
495 + # in their annotations. `arg == "torch.Tensor"` is used to handle
496 + # the case when users use `import __annotations__` to turn type
497 + # hints into strings.
498 + if isinstance(v, fn_type) \
499 + and v.__code__.co_filename == __file__ \
500 + and any(arg is torch.Tensor or arg == "torch.Tensor"
501 + for arg in v.__annotations__.values()):
502 + names_and_values_to_update[k] = hint_on_error(v)
503 +
504 +names_and_values.update(names_and_values_to_update)
505 +del names_and_values_to_update, names_and_values, v, k, fn_type