@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6e634f3d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Suppress import custom_ops warning (#7458)

Woosuk Kwon committed Aug 13, 2024 at 00:30 UTC d6e634f3d78649002460758f3964e4df0d39a546
2 files changed +7 -5
vllm/_custom_ops.py
+6 -4
@@ -6,13 +6,15 @@ import torch
6
7 from vllm._core_ext import ScalarType
8 from vllm.logger import init_logger
9 +from vllm.platforms import current_platform
10
11 logger = init_logger(__name__)
12
12 -try:
13 - import vllm._C
14 -except ImportError as e:
15 - logger.warning("Failed to import from vllm._C with %r", e)
13 +if not current_platform.is_tpu():
14 + try:
15 + import vllm._C
16 + except ImportError as e:
17 + logger.warning("Failed to import from vllm._C with %r", e)
18
19 with contextlib.suppress(ImportError):
20 # ruff: noqa: F401
vllm/utils.py
+1 -1
@@ -29,7 +29,6 @@ import torch.types
29 from typing_extensions import ParamSpec, TypeIs, assert_never
30
31 import vllm.envs as envs
32 -from vllm import _custom_ops as ops
32 from vllm.logger import enable_trace_function_call, init_logger
33
34 logger = init_logger(__name__)
@@ -359,6 +358,7 @@ def is_xpu() -> bool:
358 @lru_cache(maxsize=None)
359 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
360 """Returns the maximum shared memory per thread block in bytes."""
361 + from vllm import _custom_ops as ops
362 max_shared_mem = (
363 ops.get_max_shared_memory_per_block_device_attribute(gpu))
364 # value 0 will cause MAX_SEQ_LEN become negative and test_attention.py