@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c4f59afc3

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 12:15 UTC

Update PyTorch to 2.7.0 (#16859)

Huy Do committed Apr 29, 2025 at 19:08 UTC 2c4f59afc3d50fda805c4ad94c9d9be168cded0b

18 files changed +102 -77

.buildkite/release-pipeline.yaml

+5 -5

@@ -1,20 +1,20 @@

1 steps:

2 - - label: "Build wheel - CUDA 12.4"

2 + - label: "Build wheel - CUDA 12.8"

3 agents:

4 queue: cpu_queue_postmerge

5 commands:

6 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."

6 + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."

7 - "mkdir artifacts"

8 - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"

9 - "bash .buildkite/scripts/upload-wheels.sh"

10 env:

11 DOCKER_BUILDKIT: "1"

13 - - label: "Build wheel - CUDA 12.1"

13 + - label: "Build wheel - CUDA 12.6"

14 agents:

15 queue: cpu_queue_postmerge

16 commands:

17 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.1.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."

17 + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.6.3 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."

18 - "mkdir artifacts"

19 - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"

20 - "bash .buildkite/scripts/upload-wheels.sh"

@@ -48,7 +48,7 @@ steps:

48 queue: cpu_queue_postmerge

49 commands:

50 - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"

51 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.4.0 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain -f docker/Dockerfile ."

51 + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.8.1 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT --target vllm-openai --progress plain -f docker/Dockerfile ."

52 - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"

54 - label: "Build and publish TPU release image"

.buildkite/scripts/upload-wheels.sh

+9 -9

@@ -50,11 +50,11 @@ aws s3 cp "$normal_wheel" "s3://vllm-wheels/$BUILDKITE_COMMIT/"

50 if [[ $normal_wheel == *"cu118"* ]]; then

51 # if $normal_wheel matches cu118, do not upload the index.html

52 echo "Skipping index files for cu118 wheels"

53 -elif [[ $normal_wheel == *"cu121"* ]]; then

54 - # if $normal_wheel matches cu121, do not upload the index.html

55 - echo "Skipping index files for cu121 wheels"

53 +elif [[ $normal_wheel == *"cu126"* ]]; then

54 + # if $normal_wheel matches cu126, do not upload the index.html

55 + echo "Skipping index files for cu126 wheels"

56 else

57 - # only upload index.html for cu124 wheels (default wheels)

57 + # only upload index.html for cu128 wheels (default wheels)

58 aws s3 cp index.html "s3://vllm-wheels/$BUILDKITE_COMMIT/vllm/index.html"

59 aws s3 cp "s3://vllm-wheels/nightly/index.html" "s3://vllm-wheels/$BUILDKITE_COMMIT/index.html"

60 fi

@@ -66,12 +66,12 @@ aws s3 cp "$normal_wheel" "s3://vllm-wheels/nightly/"

66 if [[ $normal_wheel == *"cu118"* ]]; then

67 # if $normal_wheel matches cu118, do not upload the index.html

68 echo "Skipping index files for cu118 wheels"

69 -elif [[ $normal_wheel == *"cu121"* ]]; then

70 - # if $normal_wheel matches cu121, do not upload the index.html

71 - echo "Skipping index files for cu121 wheels"

69 +elif [[ $normal_wheel == *"cu126"* ]]; then

70 + # if $normal_wheel matches cu126, do not upload the index.html

71 + echo "Skipping index files for cu126 wheels"

72 else

73 - # only upload index.html for cu124 wheels (default wheels)

73 + # only upload index.html for cu128 wheels (default wheels)

74 aws s3 cp index.html "s3://vllm-wheels/nightly/vllm/index.html"

75 fi

77 -aws s3 cp "$wheel" "s3://vllm-wheels/$version/"

\ No newline at end of file

77 +aws s3 cp "$wheel" "s3://vllm-wheels/$version/"

.buildkite/test-pipeline.yaml

+1 -1

@@ -455,7 +455,7 @@ steps:

455 - tests/models/encoder_decoder/language

456 commands:

457 # Install causal-conv1d for plamo2 models here, as it is not compatible with pip-compile.

458 - - pip install causal-conv1d

458 + - pip install 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.0.post8'

459 - pytest -v -s models/decoder_only/language -m 'core_model or quant_model'

460 - pytest -v -s models/embedding/language -m core_model

461

.github/workflows/lint-and-deploy.yaml

+2 -2

@@ -66,7 +66,7 @@ jobs:

66 export AWS_SECRET_ACCESS_KEY=minioadmin

67 sleep 30 && kubectl -n ns-vllm logs -f "$(kubectl -n ns-vllm get pods | awk '/deployment/ {print $1;exit}')" &

68 helm install --wait --wait-for-jobs --timeout 5m0s --debug --create-namespace --namespace=ns-vllm test-vllm examples/online_serving/chart-helm -f examples/online_serving/chart-helm/values.yaml --set secrets.s3endpoint=http://minio:9000 --set secrets.s3bucketname=testbucket --set secrets.s3accesskeyid=$AWS_ACCESS_KEY_ID --set secrets.s3accesskey=$AWS_SECRET_ACCESS_KEY --set resources.requests.cpu=1 --set resources.requests.memory=4Gi --set resources.limits.cpu=2 --set resources.limits.memory=5Gi --set image.env[0].name=VLLM_CPU_KVCACHE_SPACE --set image.env[1].name=VLLM_LOGGING_LEVEL --set-string image.env[0].value="1" --set-string image.env[1].value="DEBUG" --set-string extraInit.s3modelpath="opt-125m/" --set-string 'resources.limits.nvidia\.com/gpu=0' --set-string 'resources.requests.nvidia\.com/gpu=0' --set-string image.repository="vllm-cpu-env"

69 -

69 +

70 - name: curl test

71 run: |

72 kubectl -n ns-vllm port-forward service/test-vllm-service 8001:80 &

@@ -79,4 +79,4 @@ jobs:

79 "max_tokens": 7,

80 "temperature": 0

81 }'):$CODE"

82 - echo "$CODE"

\ No newline at end of file

82 + echo "$CODE"

.pre-commit-config.yaml

+1 -1

@@ -46,7 +46,7 @@ repos:

46 rev: 0.6.17

47 hooks:

48 - id: pip-compile

49 - args: [requirements/test.in, -o, requirements/test.txt]

49 + args: [requirements/test.in, -o, requirements/test.txt, --index-strategy, unsafe-best-match]

50 files: ^requirements/test\.(in|txt)$

51 - repo: local

52 hooks:

CMakeLists.txt

+2 -2

@@ -46,8 +46,8 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1

46 # requirements.txt files and should be kept consistent. The ROCm torch

47 # versions are derived from docker/Dockerfile.rocm

48 #

49 -set(TORCH_SUPPORTED_VERSION_CUDA "2.6.0")

50 -set(TORCH_SUPPORTED_VERSION_ROCM "2.6.0")

49 +set(TORCH_SUPPORTED_VERSION_CUDA "2.7.0")

50 +set(TORCH_SUPPORTED_VERSION_ROCM "2.7.0")

52 #

53 # Try to find python package with an executable that exactly matches

docker/Dockerfile

+32 -14

@@ -5,11 +5,11 @@

5 # docs/source/contributing/dockerfile/dockerfile.md and

6 # docs/source/assets/contributing/dockerfile-stages-dependency.png

8 -ARG CUDA_VERSION=12.4.1

8 +ARG CUDA_VERSION=12.8.1

9 #################### BASE BUILD IMAGE ####################

10 # prepare basic build environment

11 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 AS base

12 -ARG CUDA_VERSION=12.4.1

12 +ARG CUDA_VERSION=12.8.1

13 ARG PYTHON_VERSION=3.12

14 ARG TARGETPLATFORM

15 ENV DEBIAN_FRONTEND=noninteractive

@@ -37,6 +37,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \

37 # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out

38 # Reference: https://github.com/astral-sh/uv/pull/1694

39 ENV UV_HTTP_TIMEOUT=500

40 +ENV UV_INDEX_STRATEGY="unsafe-best-match"

42 # Upgrade to GCC 10 to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519

43 # as it was causing spam when compiling the CUTLASS kernels

@@ -69,7 +70,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \

70 COPY requirements/common.txt requirements/common.txt

71 COPY requirements/cuda.txt requirements/cuda.txt

72 RUN --mount=type=cache,target=/root/.cache/uv \

72 - uv pip install --system -r requirements/cuda.txt

73 + uv pip install --system -r requirements/cuda.txt \

74 + --extra-index-url https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

76 # cuda arch list used by torch

77 # can be useful for both `dev` and `test`

@@ -92,9 +94,11 @@ COPY requirements/build.txt requirements/build.txt

94 # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out

95 # Reference: https://github.com/astral-sh/uv/pull/1694

96 ENV UV_HTTP_TIMEOUT=500

97 +ENV UV_INDEX_STRATEGY="unsafe-best-match"

99 RUN --mount=type=cache,target=/root/.cache/uv \

97 - uv pip install --system -r requirements/build.txt

100 + uv pip install --system -r requirements/build.txt \

101 + --extra-index-url https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

102

103 COPY . .

104 ARG GIT_REPO_CHECK=0

@@ -161,22 +165,25 @@ FROM base as dev

165 # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out

166 # Reference: https://github.com/astral-sh/uv/pull/1694

167 ENV UV_HTTP_TIMEOUT=500

168 +ENV UV_INDEX_STRATEGY="unsafe-best-match"

169 +

170 +# Workaround for #17068

171 +RUN --mount=type=cache,target=/root/.cache/uv \

172 + uv pip install --system --no-build-isolation "git+https://github.com/state-spaces/mamba@v2.2.4"

173

174 COPY requirements/lint.txt requirements/lint.txt

175 COPY requirements/test.txt requirements/test.txt

176 COPY requirements/dev.txt requirements/dev.txt

168 -# Workaround for #17068

169 -RUN --mount=type=cache,target=/root/.cache/uv \

170 - uv pip install --system mamba-ssm==2.2.4 --no-build-isolation

177 RUN --mount=type=cache,target=/root/.cache/uv \

172 - uv pip install --system -r requirements/dev.txt

178 + uv pip install --system -r requirements/dev.txt \

179 + --extra-index-url https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

180 #################### DEV IMAGE ####################

181

182 #################### vLLM installation IMAGE ####################

183 # image with vLLM installed

184 # TODO: Restore to base image after FlashInfer AOT wheel fixed

185 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 AS vllm-base

179 -ARG CUDA_VERSION=12.4.1

186 +ARG CUDA_VERSION=12.8.1

187 ARG PYTHON_VERSION=3.12

188 WORKDIR /vllm-workspace

189 ENV DEBIAN_FRONTEND=noninteractive

@@ -209,6 +216,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \

216 # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out

217 # Reference: https://github.com/astral-sh/uv/pull/1694

218 ENV UV_HTTP_TIMEOUT=500

219 +ENV UV_INDEX_STRATEGY="unsafe-best-match"

220

221 # Workaround for https://github.com/openai/triton/issues/2507 and

222 # https://github.com/pytorch/pytorch/issues/107960 -- hopefully

@@ -229,7 +237,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \

237 # Install vllm wheel first, so that torch etc will be installed.

238 RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \

239 --mount=type=cache,target=/root/.cache/uv \

232 - uv pip install --system dist/*.whl --verbose

240 + uv pip install --system dist/*.whl --verbose \

241 + --extra-index-url https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

242

243 # If we need to build FlashInfer wheel before its release:

244 # $ export FLASHINFER_ENABLE_AOT=1

@@ -246,19 +255,26 @@ RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist

255 RUN --mount=type=cache,target=/root/.cache/uv \

256 . /etc/environment && \

257 if [ "$TARGETPLATFORM" != "linux/arm64" ]; then \

249 - uv pip install --system https://github.com/flashinfer-ai/flashinfer/releases/download/v0.2.1.post2/flashinfer_python-0.2.1.post2+cu124torch2.6-cp38-abi3-linux_x86_64.whl ; \

258 + # TESTING: install FlashInfer from source to test 2.7.0 final RC

259 + FLASHINFER_ENABLE_AOT=1 TORCH_CUDA_ARCH_LIST='7.5 8.0 8.6 8.9 9.0+PTX' \

260 + uv pip install --system --no-build-isolation "git+https://github.com/flashinfer-ai/flashinfer@v0.2.2.post1" ; \

261 fi

262 COPY examples examples

263 COPY benchmarks benchmarks

264 COPY ./vllm/collect_env.py .

265

266 +RUN --mount=type=cache,target=/root/.cache/uv \

267 +. /etc/environment && \

268 +uv pip list

269 +

270 # Although we build Flashinfer with AOT mode, there's still

271 # some issues w.r.t. JIT compilation. Therefore we need to

272 # install build dependencies for JIT compilation.

273 # TODO: Remove this once FlashInfer AOT wheel is fixed

274 COPY requirements/build.txt requirements/build.txt

275 RUN --mount=type=cache,target=/root/.cache/uv \

261 - uv pip install --system -r requirements/build.txt

276 + uv pip install --system -r requirements/build.txt \

277 + --extra-index-url https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

278

279 #################### vLLM installation IMAGE ####################

280

@@ -272,11 +288,13 @@ ADD . /vllm-workspace/

288 # This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out

289 # Reference: https://github.com/astral-sh/uv/pull/1694

290 ENV UV_HTTP_TIMEOUT=500

291 +ENV UV_INDEX_STRATEGY="unsafe-best-match"

292

276 -# install development dependencies (for testing)

293 # Workaround for #17068

294 RUN --mount=type=cache,target=/root/.cache/uv \

279 - uv pip install --system mamba-ssm==2.2.4 --no-build-isolation

295 + uv pip install --system --no-build-isolation "git+https://github.com/state-spaces/mamba@v2.2.4"

296 +

297 +# install development dependencies (for testing)

298 RUN --mount=type=cache,target=/root/.cache/uv \

299 uv pip install --system -r requirements/dev.txt

300

docs/source/getting_started/installation/gpu/cuda.inc.md

+3 -3

@@ -1,6 +1,6 @@

1 # Installation

3 -vLLM contains pre-compiled C++ and CUDA (12.1) binaries.

3 +vLLM contains pre-compiled C++ and CUDA (12.6) binaries.

5 ## Requirements

@@ -23,12 +23,12 @@ Therefore, it is recommended to install vLLM with a **fresh new** environment. I

23 You can install vLLM using either `pip` or `uv pip`:

25 ```console

26 -# Install vLLM with CUDA 12.4.

26 +# Install vLLM with CUDA 12.6.

27 pip install vllm # If you are using pip.

28 uv pip install vllm # If you are using uv.

29 ```

31 -As of now, vLLM's binaries are compiled with CUDA 12.4 and public PyTorch release versions by default. We also provide vLLM binaries compiled with CUDA 12.1, 11.8, and public PyTorch release versions:

31 +As of now, vLLM's binaries are compiled with CUDA 12.6 and public PyTorch release versions by default. We also provide vLLM binaries compiled with CUDA 12.8, 11.8, and public PyTorch release versions:

33 ```console

34 # Install vLLM with CUDA 11.8.

examples/online_serving/chart-helm/values.yaml

+1 -1

@@ -8,7 +8,7 @@ image:

8 # -- Image tag

9 tag: "latest"

10 # -- Container launch command

11 - command: ["vllm", "serve", "/data/", "--served-model-name", "opt-125m", "--dtype", "bfloat16", "--host", "0.0.0.0", "--port", "8000"]

11 + command: ["vllm", "serve", "/data/", "--served-model-name", "opt-125m", "--dtype", "float32", "--block-size", "16", "--host", "0.0.0.0", "--port", "8000"]

13 # -- Container port

14 containerPort: 8000

pyproject.toml

+1 -1

@@ -6,7 +6,7 @@ requires = [

6 "packaging",

7 "setuptools>=61",

8 "setuptools-scm>=8.0",

9 - "torch == 2.6.0",

9 + "torch == 2.7.0",

10 "wheel",

11 "jinja2",

12 ]

requirements/build.txt

+1 -1

@@ -4,6 +4,6 @@ ninja

4 packaging

5 setuptools>=61

6 setuptools-scm>=8

7 -torch==2.6.0

7 +torch==2.7.0

8 wheel

9 jinja2>=3.1.6

requirements/cpu.txt

+6 -5

@@ -2,18 +2,19 @@

2 -r common.txt

4 # Dependencies for CPUs

5 -torch==2.6.0+cpu; platform_machine == "x86_64"

6 -torch==2.6.0; platform_system == "Darwin"

7 -torch==2.6.0; platform_machine == "ppc64le" or platform_machine == "aarch64"

5 +--extra-index-url https://download.pytorch.org/whl/cpu

6 +torch==2.7.0+cpu; platform_machine == "x86_64"

7 +torch==2.7.0; platform_system == "Darwin"

8 +torch==2.7.0; platform_machine == "ppc64le" or platform_machine == "aarch64"

9 torch==2.7.0.dev20250304; platform_machine == "s390x"

11 # required for the image processor of minicpm-o-2_6, this must be updated alongside torch

12 torchaudio; platform_machine != "ppc64le" and platform_machine != "s390x"

12 -torchaudio==2.6.0; platform_machine == "ppc64le"

13 +torchaudio==2.7.0; platform_machine == "ppc64le"

15 # required for the image processor of phi3v, this must be updated alongside torch

16 torchvision; platform_machine != "ppc64le" and platform_machine != "s390x"

16 -torchvision==0.21.0; platform_machine == "ppc64le"

17 +torchvision==0.22.0; platform_machine == "ppc64le"

18 datasets # for benchmark scripts

20 # cpu cannot use triton 3.3.0

requirements/cuda.txt

+5 -4

@@ -6,8 +6,9 @@ numba == 0.61.2; python_version > '3.9'

7 # Dependencies for NVIDIA GPUs

8 ray[cgraph]>=2.43.0, !=2.44.* # Ray Compiled Graph, required for pipeline parallelism in V1.

9 -torch==2.6.0

10 -torchaudio==2.6.0

9 +torch==2.7.0

10 +torchaudio==2.7.0

11 # These must be updated alongside torch

12 -torchvision==0.21.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version

13 -xformers==0.0.29.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.6.0

12 +torchvision==0.22.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version

13 +# https://github.com/facebookresearch/xformers/releases/tag/v0.0.30

14 +xformers==0.0.30; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch >= 2.7

requirements/rocm-build.txt

+3 -3

@@ -2,9 +2,9 @@

2 -r common.txt

4 --extra-index-url https://download.pytorch.org/whl/rocm6.2.4

5 -torch==2.6.0

6 -torchvision==0.21.0

7 -torchaudio==2.6.0

5 +torch==2.7.0

6 +torchvision==0.22.0

7 +torchaudio==2.7.0

9 triton==3.2

10 cmake>=3.26,<4

requirements/test.in

+3 -3

@@ -23,9 +23,9 @@ sentence-transformers # required for embedding tests

23 soundfile # required for audio tests

24 jiwer # required for audio tests

25 timm # required for internvl test

26 -torch==2.6.0

27 -torchaudio==2.6.0

28 -torchvision==0.21.0

26 +torch==2.7.0

27 +torchaudio==2.7.0

28 +torchvision==0.22.0

29 transformers_stream_generator # required for qwen-vl test

30 mamba_ssm # required for plamo2 test

31 matplotlib # required for qwen-vl test

requirements/test.txt

+24 -20

@@ -1,5 +1,5 @@

1 # This file was autogenerated by uv via the following command:

2 -# uv pip compile requirements/test.in -o requirements/test.txt

2 +# uv pip compile requirements/test.in -o requirements/test.txt --index-strategy unsafe-best-match

3 absl-py==2.1.0

4 # via rouge-score

5 accelerate==1.0.1

@@ -274,7 +274,7 @@ mamba-ssm==2.2.4

274 # via -r requirements/test.in

275 markdown-it-py==3.0.0

276 # via rich

277 -markupsafe==3.0.2

277 +markupsafe==3.0.1

278 # via

279 # jinja2

280 # werkzeug

@@ -355,39 +355,42 @@ numpy==1.26.4

355 # transformers

356 # tritonclient

357 # vocos

358 -nvidia-cublas-cu12==12.4.5.8

358 +nvidia-cublas-cu12==12.6.4.1

359 # via

360 # nvidia-cudnn-cu12

361 # nvidia-cusolver-cu12

362 # torch

363 -nvidia-cuda-cupti-cu12==12.4.127

363 +nvidia-cuda-cupti-cu12==12.6.80

364 # via torch

365 -nvidia-cuda-nvrtc-cu12==12.4.127

365 +nvidia-cuda-nvrtc-cu12==12.6.77

366 # via torch

367 -nvidia-cuda-runtime-cu12==12.4.127

367 +nvidia-cuda-runtime-cu12==12.6.77

368 # via torch

369 -nvidia-cudnn-cu12==9.1.0.70

369 +nvidia-cudnn-cu12==9.5.1.17

370 # via torch

371 -nvidia-cufft-cu12==11.2.1.3

371 +nvidia-cufft-cu12==11.3.0.4

372 # via torch

373 -nvidia-curand-cu12==10.3.5.147

373 +nvidia-cufile-cu12==1.11.1.6

374 # via torch

375 -nvidia-cusolver-cu12==11.6.1.9

375 +nvidia-curand-cu12==10.3.7.77

376 # via torch

377 -nvidia-cusparse-cu12==12.3.1.170

377 +nvidia-cusolver-cu12==11.7.1.2

378 + # via torch

379 +nvidia-cusparse-cu12==12.5.4.2

380 # via

381 # nvidia-cusolver-cu12

382 # torch

381 -nvidia-cusparselt-cu12==0.6.2

383 +nvidia-cusparselt-cu12==0.6.3

384 # via torch

383 -nvidia-nccl-cu12==2.21.5

385 +nvidia-nccl-cu12==2.26.2

386 # via torch

385 -nvidia-nvjitlink-cu12==12.4.127

387 +nvidia-nvjitlink-cu12==12.6.85

388 # via

389 + # nvidia-cufft-cu12

390 # nvidia-cusolver-cu12

391 # nvidia-cusparse-cu12

392 # torch

390 -nvidia-nvtx-cu12==12.4.127

393 +nvidia-nvtx-cu12==12.6.77

394 # via torch

395 opencv-python-headless==4.11.0.86

396 # via

@@ -634,6 +637,7 @@ setuptools==75.8.0

637 # mamba-ssm

638 # pytablewriter

639 # torch

640 + # triton

641 shellingham==1.5.4

642 # via typer

643 six==1.16.0

@@ -664,7 +668,7 @@ starlette-testclient==0.4.1

668 # via schemathesis

669 statsmodels==0.14.4

670 # via genai-perf

667 -sympy==1.13.1

671 +sympy==1.13.3

672 # via

673 # einx

674 # torch

@@ -696,7 +700,7 @@ tomli==2.2.1

700 # via schemathesis

701 tomli-w==1.2.0

702 # via schemathesis

699 -torch==2.6.0

703 +torch==2.7.0

704 # via

705 # -r requirements/test.in

706 # accelerate

@@ -714,12 +718,12 @@ torch==2.6.0

718 # torchvision

719 # vector-quantize-pytorch

720 # vocos

717 -torchaudio==2.6.0

721 +torchaudio==2.7.0

722 # via

723 # -r requirements/test.in

724 # encodec

725 # vocos

722 -torchvision==0.21.0

726 +torchvision==0.22.0

727 # via

728 # -r requirements/test.in

729 # timm

@@ -748,7 +752,7 @@ transformers==4.51.3

752 # transformers-stream-generator

753 transformers-stream-generator==0.0.5

754 # via -r requirements/test.in

751 -triton==3.2.0

755 +triton==3.3.0

756 # via torch

757 tritonclient==2.51.0

758 # via

setup.py

+1 -1

@@ -54,7 +54,7 @@ elif (sys.platform.startswith("linux") and torch.version.cuda is None

54 # fallback to cpu

55 VLLM_TARGET_DEVICE = "cpu"

57 -MAIN_CUDA_VERSION = "12.4"

57 +MAIN_CUDA_VERSION = "12.8"

60 def is_sccache_available() -> bool:

vllm/attention/ops/ipex_attn.py

+2 -1

@@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple

5 try:

6 import intel_extension_for_pytorch.llm.modules as ipex_modules

7 _use_ipex = True

8 -except ImportError:

8 +# AttributeError is to handle a bug in ipex https://github.com/intel/intel-extension-for-pytorch/pull/813

9 +except (ImportError, AttributeError):

10 _use_ipex = False

12 import torch