A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Docker] bump up neuron sdk v2.21 (#11593)
Signed-off-by: Liangfu Chen <liangfc@amazon.com>
Liangfu Chen committed
Dec 29, 2024 at 21:46 UTC
628ec6c17b8121517e8f303b64567573036cdb38
4 files changed
+6
-8
Dockerfile.neuron
+3
-3
@@ -1,6 +1,6 @@
1
# default base image
2
# https://gallery.ecr.aws/neuron/pytorch-inference-neuronx
3
-ARG BASE_IMAGE="public.ecr.aws/neuron/pytorch-inference-neuronx:2.1.2-neuronx-py310-sdk2.20.2-ubuntu20.04"
3
+ARG BASE_IMAGE="public.ecr.aws/neuron/pytorch-inference-neuronx:2.5.1-neuronx-py310-sdk2.21.0-ubuntu22.04"
4
5
FROM $BASE_IMAGE
6
@@ -22,9 +22,9 @@ WORKDIR ${APP_MOUNT}/vllm
22
23
RUN python3 -m pip install --upgrade pip
24
RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
25
-RUN python3 -m pip install sentencepiece transformers==4.36.2 -U
25
+RUN python3 -m pip install sentencepiece transformers==4.45.2 -U
26
RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
27
-RUN python3 -m pip install --pre neuronx-cc==2.15.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
27
+RUN python3 -m pip install neuronx-cc==2.16.345.0 --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
28
29
COPY . .
30
ARG GIT_REPO_CHECK=0
requirements-neuron.txt
+2
-2
@@ -2,6 +2,6 @@
2
-r requirements-common.txt
3
4
# Dependencies for Neuron devices
5
-transformers-neuronx >= 0.12.0
6
-torch-neuronx >= 2.1.2
5
+transformers-neuronx >= 0.13.0
6
+torch-neuronx >= 2.5.0
7
neuronx-cc
vllm/_custom_ops.py
+1
-2
@@ -23,8 +23,7 @@ with contextlib.suppress(ImportError):
23
import vllm._moe_C # noqa: F401
24
supports_moe_ops = True
25
26
-# neuron has torch version that doesn't even have impl_abstract
27
-if TYPE_CHECKING or current_platform.is_neuron():
26
+if TYPE_CHECKING:
27
28
def register_fake(fn):
29
return lambda name: fn
vllm/triton_utils/importing.py
-1
@@ -8,7 +8,6 @@ logger = init_logger(__name__)
8
HAS_TRITON = (
9
find_spec("triton") is not None
10
and not current_platform.is_xpu() # Not compatible
11
- and not current_platform.is_neuron() # neuron has too old torch
11
)
12
13
if not HAS_TRITON: