@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 628ec6c17b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Docker] bump up neuron sdk v2.21 (#11593)

Signed-off-by: Liangfu Chen <liangfc@amazon.com>

Liangfu Chen committed Dec 29, 2024 at 21:46 UTC 628ec6c17b8121517e8f303b64567573036cdb38
4 files changed +6 -8
Dockerfile.neuron
+3 -3
@@ -1,6 +1,6 @@
1 # default base image
2 # https://gallery.ecr.aws/neuron/pytorch-inference-neuronx
3 -ARG BASE_IMAGE="public.ecr.aws/neuron/pytorch-inference-neuronx:2.1.2-neuronx-py310-sdk2.20.2-ubuntu20.04"
3 +ARG BASE_IMAGE="public.ecr.aws/neuron/pytorch-inference-neuronx:2.5.1-neuronx-py310-sdk2.21.0-ubuntu22.04"
4
5 FROM $BASE_IMAGE
6
@@ -22,9 +22,9 @@ WORKDIR ${APP_MOUNT}/vllm
22
23 RUN python3 -m pip install --upgrade pip
24 RUN python3 -m pip install --no-cache-dir fastapi ninja tokenizers pandas
25 -RUN python3 -m pip install sentencepiece transformers==4.36.2 -U
25 +RUN python3 -m pip install sentencepiece transformers==4.45.2 -U
26 RUN python3 -m pip install transformers-neuronx --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
27 -RUN python3 -m pip install --pre neuronx-cc==2.15.* --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
27 +RUN python3 -m pip install neuronx-cc==2.16.345.0 --extra-index-url=https://pip.repos.neuron.amazonaws.com -U
28
29 COPY . .
30 ARG GIT_REPO_CHECK=0
requirements-neuron.txt
+2 -2
@@ -2,6 +2,6 @@
2 -r requirements-common.txt
3
4 # Dependencies for Neuron devices
5 -transformers-neuronx >= 0.12.0
6 -torch-neuronx >= 2.1.2
5 +transformers-neuronx >= 0.13.0
6 +torch-neuronx >= 2.5.0
7 neuronx-cc
vllm/_custom_ops.py
+1 -2
@@ -23,8 +23,7 @@ with contextlib.suppress(ImportError):
23 import vllm._moe_C # noqa: F401
24 supports_moe_ops = True
25
26 -# neuron has torch version that doesn't even have impl_abstract
27 -if TYPE_CHECKING or current_platform.is_neuron():
26 +if TYPE_CHECKING:
27
28 def register_fake(fn):
29 return lambda name: fn
vllm/triton_utils/importing.py
-1
@@ -8,7 +8,6 @@ logger = init_logger(__name__)
8 HAS_TRITON = (
9 find_spec("triton") is not None
10 and not current_platform.is_xpu() # Not compatible
11 - and not current_platform.is_neuron() # neuron has too old torch
11 )
12
13 if not HAS_TRITON: