@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e0861bd0b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[CI/Build] Update PyTorch to 2.4.0 (#6951)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Sage Moore committed Aug 1, 2024 at 11:11 UTC 7e0861bd0bb25ea5ceaa3a513da4133fb828b5fe
8 files changed +13 -13
.buildkite/test-pipeline.yaml
+3 -3
@@ -44,7 +44,7 @@ steps:
44 fast_check: true
45 commands:
46 # This flashinfer installation will fail on AMD ROCm, so it is set as optional.
47 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl || true
47 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl || true
48 - pytest -v -s basic_correctness/test_basic_correctness.py
49 - pytest -v -s basic_correctness/test_cpu_offload.py
50 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
@@ -164,7 +164,7 @@ steps:
164 - label: Models Test
165 #mirror_hardwares: [amd]
166 commands:
167 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
167 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
168 - pytest -v -s models -m \"not vlm\"
169
170 - label: Vision Language Models Test
@@ -281,7 +281,7 @@ steps:
281 - pytest -v -s distributed/test_custom_all_reduce.py
282 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
283 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
284 - - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
284 + - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
285 - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
286 - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
287 - pytest -v -s -x lora/test_mixtral.py
.github/workflows/publish.yml
+1 -1
@@ -49,7 +49,7 @@ jobs:
49 matrix:
50 os: ['ubuntu-20.04']
51 python-version: ['3.8', '3.9', '3.10', '3.11']
52 - pytorch-version: ['2.3.1'] # Must be the most recent version that meets requirements-cuda.txt.
52 + pytorch-version: ['2.4.0'] # Must be the most recent version that meets requirements-cuda.txt.
53 cuda-version: ['11.8', '12.1']
54
55 steps:
CMakeLists.txt
+1 -1
@@ -32,7 +32,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
32 # requirements.txt files and should be kept consistent. The ROCm torch
33 # versions are derived from Dockerfile.rocm
34 #
35 -set(TORCH_SUPPORTED_VERSION_CUDA "2.3.1")
35 +set(TORCH_SUPPORTED_VERSION_CUDA "2.4.0")
36 set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
37
38 #
Dockerfile
+1 -1
@@ -192,7 +192,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb
192 python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
193
194 RUN --mount=type=cache,target=/root/.cache/pip \
195 - python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.9/flashinfer-0.0.9+cu121torch2.3-cp310-cp310-linux_x86_64.whl
195 + python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
196 #################### vLLM installation IMAGE ####################
197
198
pyproject.toml
+1 -1
@@ -5,7 +5,7 @@ requires = [
5 "ninja",
6 "packaging",
7 "setuptools >= 49.4.0",
8 - "torch == 2.3.1",
8 + "torch == 2.4.0",
9 "wheel",
10 ]
11 build-backend = "setuptools.build_meta"
requirements-build.txt
+1 -1
@@ -3,5 +3,5 @@ cmake>=3.21
3 ninja
4 packaging
5 setuptools>=49.4.0
6 -torch==2.3.1
6 +torch==2.4.0
7 wheel
requirements-cuda.txt
+4 -4
@@ -4,8 +4,8 @@
4 # Dependencies for NVIDIA GPUs
5 ray >= 2.9
6 nvidia-ml-py # for pynvml package
7 -torch == 2.3.1
7 +torch == 2.4.0
8 # These must be updated alongside torch
9 -torchvision == 0.18.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 -xformers == 0.0.27 # Requires PyTorch 2.3.1
11 -vllm-flash-attn == 2.5.9.post1 # Requires PyTorch 2.3.1
9 +torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 +xformers == 0.0.27.post2 # Requires PyTorch 2.4.0
11 +vllm-flash-attn == 2.6.0 # Requires PyTorch 2.4.0
vllm/model_executor/layers/ops/sample.py
+1 -1
@@ -7,7 +7,7 @@ import triton.language as tl
7 from vllm.model_executor.layers.ops.rand import seeded_uniform
8 from vllm.triton_utils.sample import get_num_triton_sampler_splits
9
10 -_EPS = 1e-6
10 +_EPS: tl.constexpr = 1e-6
11
12
13 def _multi_split_sample(