A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[CI/Build] Update PyTorch to 2.4.0 (#6951)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Sage Moore committed
Aug 1, 2024 at 11:11 UTC
7e0861bd0bb25ea5ceaa3a513da4133fb828b5fe
8 files changed
+13
-13
.buildkite/test-pipeline.yaml
+3
-3
@@ -44,7 +44,7 @@ steps:
44
fast_check: true
45
commands:
46
# This flashinfer installation will fail on AMD ROCm, so it is set as optional.
47
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl || true
47
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl || true
48
- pytest -v -s basic_correctness/test_basic_correctness.py
49
- pytest -v -s basic_correctness/test_cpu_offload.py
50
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_chunked_prefill.py
@@ -164,7 +164,7 @@ steps:
164
- label: Models Test
165
#mirror_hardwares: [amd]
166
commands:
167
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
167
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
168
- pytest -v -s models -m \"not vlm\"
169
170
- label: Vision Language Models Test
@@ -281,7 +281,7 @@ steps:
281
- pytest -v -s distributed/test_custom_all_reduce.py
282
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
283
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
284
- - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.8/flashinfer-0.0.8+cu121torch2.3-cp310-cp310-linux_x86_64.whl
284
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
285
- VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
286
- VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
287
- pytest -v -s -x lora/test_mixtral.py
.github/workflows/publish.yml
+1
-1
@@ -49,7 +49,7 @@ jobs:
49
matrix:
50
os: ['ubuntu-20.04']
51
python-version: ['3.8', '3.9', '3.10', '3.11']
52
- pytorch-version: ['2.3.1'] # Must be the most recent version that meets requirements-cuda.txt.
52
+ pytorch-version: ['2.4.0'] # Must be the most recent version that meets requirements-cuda.txt.
53
cuda-version: ['11.8', '12.1']
54
55
steps:
CMakeLists.txt
+1
-1
@@ -32,7 +32,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
32
# requirements.txt files and should be kept consistent. The ROCm torch
33
# versions are derived from Dockerfile.rocm
34
#
35
-set(TORCH_SUPPORTED_VERSION_CUDA "2.3.1")
35
+set(TORCH_SUPPORTED_VERSION_CUDA "2.4.0")
36
set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
37
38
#
Dockerfile
+1
-1
@@ -192,7 +192,7 @@ RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamb
192
python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
193
194
RUN --mount=type=cache,target=/root/.cache/pip \
195
- python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.9/flashinfer-0.0.9+cu121torch2.3-cp310-cp310-linux_x86_64.whl
195
+ python3 -m pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.1.2/flashinfer-0.1.2+cu121torch2.4-cp310-cp310-linux_x86_64.whl
196
#################### vLLM installation IMAGE ####################
197
198
pyproject.toml
+1
-1
@@ -5,7 +5,7 @@ requires = [
5
"ninja",
6
"packaging",
7
"setuptools >= 49.4.0",
8
- "torch == 2.3.1",
8
+ "torch == 2.4.0",
9
"wheel",
10
]
11
build-backend = "setuptools.build_meta"
requirements-build.txt
+1
-1
@@ -3,5 +3,5 @@ cmake>=3.21
3
ninja
4
packaging
5
setuptools>=49.4.0
6
-torch==2.3.1
6
+torch==2.4.0
7
wheel
requirements-cuda.txt
+4
-4
@@ -4,8 +4,8 @@
4
# Dependencies for NVIDIA GPUs
5
ray >= 2.9
6
nvidia-ml-py # for pynvml package
7
-torch == 2.3.1
7
+torch == 2.4.0
8
# These must be updated alongside torch
9
-torchvision == 0.18.1 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
-xformers == 0.0.27 # Requires PyTorch 2.3.1
11
-vllm-flash-attn == 2.5.9.post1 # Requires PyTorch 2.3.1
9
+torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
+xformers == 0.0.27.post2 # Requires PyTorch 2.4.0
11
+vllm-flash-attn == 2.6.0 # Requires PyTorch 2.4.0
vllm/model_executor/layers/ops/sample.py
+1
-1
@@ -7,7 +7,7 @@ import triton.language as tl
7
from vllm.model_executor.layers.ops.rand import seeded_uniform
8
from vllm.triton_utils.sample import get_num_triton_sampler_splits
9
10
-_EPS = 1e-6
10
+_EPS: tl.constexpr = 1e-6
11
12
13
def _multi_split_sample(