A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Neuron] [Bugfix] Fix neuron startup (#9374)
Co-authored-by: Jerzy Zagorski <jzagorsk@amazon.com>
xendo committed
Oct 22, 2024 at 14:51 UTC
9dbcce84a73742805433414ff9000cfe7a5ef1c5
7 files changed
+37
-18
vllm/_custom_ops.py
+2
-1
@@ -26,7 +26,8 @@ with contextlib.suppress(ImportError):
26
import vllm._moe_C # noqa: F401
27
supports_moe_ops = True
28
29
-if TYPE_CHECKING:
29
+# neuron has torch version that doesn't even have impl_abstract
30
+if TYPE_CHECKING or current_platform.is_neuron():
31
32
def register_fake(fn):
33
return lambda name: fn
vllm/config.py
+7
-6
@@ -17,8 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17
get_hf_image_processor_config,
18
get_hf_text_config)
19
from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20
- is_hip, is_neuron, is_openvino, is_xpu,
21
- print_warning_once)
20
+ is_hip, is_openvino, is_xpu, print_warning_once)
21
22
if TYPE_CHECKING:
23
from ray.util.placement_group import PlacementGroup
@@ -215,8 +214,10 @@ class ModelConfig:
214
self.is_attention_free = self._init_attention_free()
215
self.has_inner_state = self._init_has_inner_state()
216
218
- self.override_neuron_config = override_neuron_config if is_neuron(
219
- ) else None
217
+ if current_platform.is_neuron():
218
+ self.override_neuron_config = override_neuron_config
219
+ else:
220
+ self.override_neuron_config = None
221
222
supported_tasks, task = self._resolve_task(task, self.hf_config)
223
self.supported_tasks = supported_tasks
@@ -368,7 +369,7 @@ class ModelConfig:
369
"Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
370
" is not set, enabling VLLM_USE_TRITON_AWQ.")
371
envs.VLLM_USE_TRITON_AWQ = True
371
- if is_neuron(
372
+ if current_platform.is_neuron(
373
) and self.quantization not in neuron_supported_quantization:
374
raise ValueError(
375
f"{self.quantization} quantization is currently not "
@@ -1112,7 +1113,7 @@ class DeviceConfig:
1113
# Automated device type detection
1114
if current_platform.is_cuda_alike():
1115
self.device_type = "cuda"
1115
- elif is_neuron():
1116
+ elif current_platform.is_neuron():
1117
self.device_type = "neuron"
1118
elif is_openvino():
1119
self.device_type = "openvino"
vllm/platforms/__init__.py
+10
@@ -58,6 +58,13 @@ try:
58
except Exception:
59
pass
60
61
+is_neuron = False
62
+try:
63
+ import transformers_neuronx # noqa: F401
64
+ is_neuron = True
65
+except ImportError:
66
+ pass
67
+
68
if is_tpu:
69
# people might install pytorch built with cuda but run on tpu
70
# so we need to check tpu first
@@ -75,6 +82,9 @@ elif is_xpu:
82
elif is_cpu:
83
from .cpu import CpuPlatform
84
current_platform = CpuPlatform()
85
+elif is_neuron:
86
+ from .neuron import NeuronPlatform
87
+ current_platform = NeuronPlatform()
88
else:
89
current_platform = UnspecifiedPlatform()
90
vllm/platforms/interface.py
+4
@@ -10,6 +10,7 @@ class PlatformEnum(enum.Enum):
10
TPU = enum.auto()
11
XPU = enum.auto()
12
CPU = enum.auto()
13
+ NEURON = enum.auto()
14
UNSPECIFIED = enum.auto()
15
16
@@ -48,6 +49,9 @@ class Platform:
49
def is_cpu(self) -> bool:
50
return self._enum == PlatformEnum.CPU
51
52
+ def is_neuron(self) -> bool:
53
+ return self._enum == PlatformEnum.NEURON
54
+
55
def is_cuda_alike(self) -> bool:
56
"""Stateless version of :func:`torch.cuda.is_available`."""
57
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
vllm/platforms/neuron.py
new
+9
@@ -0,0 +1,9 @@
1
+from .interface import Platform, PlatformEnum
2
+
3
+
4
+class NeuronPlatform(Platform):
5
+ _enum = PlatformEnum.NEURON
6
+
7
+ @classmethod
8
+ def get_device_name(cls, device_id: int = 0) -> str:
9
+ return "neuron"
vllm/triton_utils/importing.py
+4
-1
@@ -1,10 +1,13 @@
1
from importlib.util import find_spec
2
3
from vllm.logger import init_logger
4
+from vllm.platforms import current_platform
5
6
logger = init_logger(__name__)
7
7
-HAS_TRITON = find_spec("triton") is not None
8
+# neuron has too old torch
9
+HAS_TRITON = find_spec(
10
+ "triton") is not None and not current_platform.is_neuron()
11
12
if not HAS_TRITON:
13
logger.info("Triton not installed; certain GPU-related functions"
vllm/utils.py
+1
-10
@@ -327,15 +327,6 @@ def is_openvino() -> bool:
327
return False
328
329
330
-@lru_cache(maxsize=None)
331
-def is_neuron() -> bool:
332
- try:
333
- import transformers_neuronx
334
- except ImportError:
335
- transformers_neuronx = None
336
- return transformers_neuronx is not None
337
-
338
-
330
@lru_cache(maxsize=None)
331
def is_xpu() -> bool:
332
from importlib.metadata import PackageNotFoundError, version
@@ -786,7 +777,7 @@ def is_pin_memory_available() -> bool:
777
elif is_xpu():
778
print_warning_once("Pin memory is not supported on XPU.")
779
return False
789
- elif is_neuron():
780
+ elif current_platform.is_neuron():
781
print_warning_once("Pin memory is not supported on Neuron.")
782
return False
783
elif current_platform.is_cpu() or is_openvino():