@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9dbcce84a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Neuron] [Bugfix] Fix neuron startup (#9374)

Co-authored-by: Jerzy Zagorski <jzagorsk@amazon.com>

xendo committed Oct 22, 2024 at 14:51 UTC 9dbcce84a73742805433414ff9000cfe7a5ef1c5
7 files changed +37 -18
vllm/_custom_ops.py
+2 -1
@@ -26,7 +26,8 @@ with contextlib.suppress(ImportError):
26 import vllm._moe_C # noqa: F401
27 supports_moe_ops = True
28
29 -if TYPE_CHECKING:
29 +# neuron has torch version that doesn't even have impl_abstract
30 +if TYPE_CHECKING or current_platform.is_neuron():
31
32 def register_fake(fn):
33 return lambda name: fn
vllm/config.py
+7 -6
@@ -17,8 +17,7 @@ from vllm.transformers_utils.config import (ConfigFormat, get_config,
17 get_hf_image_processor_config,
18 get_hf_text_config)
19 from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
20 - is_hip, is_neuron, is_openvino, is_xpu,
21 - print_warning_once)
20 + is_hip, is_openvino, is_xpu, print_warning_once)
21
22 if TYPE_CHECKING:
23 from ray.util.placement_group import PlacementGroup
@@ -215,8 +214,10 @@ class ModelConfig:
214 self.is_attention_free = self._init_attention_free()
215 self.has_inner_state = self._init_has_inner_state()
216
218 - self.override_neuron_config = override_neuron_config if is_neuron(
219 - ) else None
217 + if current_platform.is_neuron():
218 + self.override_neuron_config = override_neuron_config
219 + else:
220 + self.override_neuron_config = None
221
222 supported_tasks, task = self._resolve_task(task, self.hf_config)
223 self.supported_tasks = supported_tasks
@@ -368,7 +369,7 @@ class ModelConfig:
369 "Using AWQ quantization with ROCm, but VLLM_USE_TRITON_AWQ"
370 " is not set, enabling VLLM_USE_TRITON_AWQ.")
371 envs.VLLM_USE_TRITON_AWQ = True
371 - if is_neuron(
372 + if current_platform.is_neuron(
373 ) and self.quantization not in neuron_supported_quantization:
374 raise ValueError(
375 f"{self.quantization} quantization is currently not "
@@ -1112,7 +1113,7 @@ class DeviceConfig:
1113 # Automated device type detection
1114 if current_platform.is_cuda_alike():
1115 self.device_type = "cuda"
1115 - elif is_neuron():
1116 + elif current_platform.is_neuron():
1117 self.device_type = "neuron"
1118 elif is_openvino():
1119 self.device_type = "openvino"
vllm/platforms/__init__.py
+10
@@ -58,6 +58,13 @@ try:
58 except Exception:
59 pass
60
61 +is_neuron = False
62 +try:
63 + import transformers_neuronx # noqa: F401
64 + is_neuron = True
65 +except ImportError:
66 + pass
67 +
68 if is_tpu:
69 # people might install pytorch built with cuda but run on tpu
70 # so we need to check tpu first
@@ -75,6 +82,9 @@ elif is_xpu:
82 elif is_cpu:
83 from .cpu import CpuPlatform
84 current_platform = CpuPlatform()
85 +elif is_neuron:
86 + from .neuron import NeuronPlatform
87 + current_platform = NeuronPlatform()
88 else:
89 current_platform = UnspecifiedPlatform()
90
vllm/platforms/interface.py
+4
@@ -10,6 +10,7 @@ class PlatformEnum(enum.Enum):
10 TPU = enum.auto()
11 XPU = enum.auto()
12 CPU = enum.auto()
13 + NEURON = enum.auto()
14 UNSPECIFIED = enum.auto()
15
16
@@ -48,6 +49,9 @@ class Platform:
49 def is_cpu(self) -> bool:
50 return self._enum == PlatformEnum.CPU
51
52 + def is_neuron(self) -> bool:
53 + return self._enum == PlatformEnum.NEURON
54 +
55 def is_cuda_alike(self) -> bool:
56 """Stateless version of :func:`torch.cuda.is_available`."""
57 return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
vllm/platforms/neuron.py new
+9
@@ -0,0 +1,9 @@
1 +from .interface import Platform, PlatformEnum
2 +
3 +
4 +class NeuronPlatform(Platform):
5 + _enum = PlatformEnum.NEURON
6 +
7 + @classmethod
8 + def get_device_name(cls, device_id: int = 0) -> str:
9 + return "neuron"
vllm/triton_utils/importing.py
+4 -1
@@ -1,10 +1,13 @@
1 from importlib.util import find_spec
2
3 from vllm.logger import init_logger
4 +from vllm.platforms import current_platform
5
6 logger = init_logger(__name__)
7
7 -HAS_TRITON = find_spec("triton") is not None
8 +# neuron has too old torch
9 +HAS_TRITON = find_spec(
10 + "triton") is not None and not current_platform.is_neuron()
11
12 if not HAS_TRITON:
13 logger.info("Triton not installed; certain GPU-related functions"
vllm/utils.py
+1 -10
@@ -327,15 +327,6 @@ def is_openvino() -> bool:
327 return False
328
329
330 -@lru_cache(maxsize=None)
331 -def is_neuron() -> bool:
332 - try:
333 - import transformers_neuronx
334 - except ImportError:
335 - transformers_neuronx = None
336 - return transformers_neuronx is not None
337 -
338 -
330 @lru_cache(maxsize=None)
331 def is_xpu() -> bool:
332 from importlib.metadata import PackageNotFoundError, version
@@ -786,7 +777,7 @@ def is_pin_memory_available() -> bool:
777 elif is_xpu():
778 print_warning_once("Pin memory is not supported on XPU.")
779 return False
789 - elif is_neuron():
780 + elif current_platform.is_neuron():
781 print_warning_once("Pin memory is not supported on Neuron.")
782 return False
783 elif current_platform.is_cpu() or is_openvino():