@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06458a0b42
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Upgrade to CUDA 12 (#1527)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Zhuohan Li committed Nov 8, 2023 at 14:17 UTC 06458a0b42449398aa2ba001d9dbaff256159448
5 files changed +15 -7
.github/workflows/publish.yml
+4 -4
@@ -43,14 +43,14 @@ jobs:
43 name: Build Wheel
44 runs-on: ${{ matrix.os }}
45 needs: release
46 -
46 +
47 strategy:
48 fail-fast: false
49 matrix:
50 os: ['ubuntu-20.04']
51 python-version: ['3.8', '3.9', '3.10', '3.11']
52 - pytorch-version: ['2.0.1']
53 - cuda-version: ['11.8'] # Github runner can't build anything older than 11.8
52 + pytorch-version: ['2.1.0']
53 + cuda-version: ['12.1']
54
55 steps:
56 - name: Checkout
@@ -82,7 +82,7 @@ jobs:
82 asset_name=${wheel_name//"linux"/"manylinux1"}
83 echo "wheel_name=${wheel_name}" >> $GITHUB_ENV
84 echo "asset_name=${asset_name}" >> $GITHUB_ENV
85 -
85 +
86 - name: Upload Release Asset
87 uses: actions/upload-release-asset@v1
88 env:
.github/workflows/scripts/build.sh
+3
@@ -11,5 +11,8 @@ LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
11 $python_executable -m pip install wheel packaging
12 $python_executable -m pip install -r requirements.txt
13
14 +# Limit the number of parallel jobs to avoid OOM
15 +export MAX_JOBS=1
16 +
17 # Build
18 $python_executable setup.py bdist_wheel --dist-dir=dist
.github/workflows/scripts/cuda-install.sh
+5
@@ -16,3 +16,8 @@ sudo apt clean
16 # Test nvcc
17 PATH=/usr/local/cuda-$1/bin:${PATH}
18 nvcc --version
19 +
20 +# Log gcc, g++, c++ versions
21 +gcc --version
22 +g++ --version
23 +c++ --version
pyproject.toml
+1 -1
@@ -3,7 +3,7 @@ requires = [
3 "ninja",
4 "packaging",
5 "setuptools",
6 - "torch == 2.0.1",
6 + "torch >= 2.1.0",
7 "wheel",
8 ]
9 build-backend = "setuptools.build_meta"
requirements.txt
+2 -2
@@ -5,9 +5,9 @@ pandas # Required for Ray data.
5 pyarrow # Required for Ray data.
6 sentencepiece # Required for LLaMA tokenizer.
7 numpy
8 -torch == 2.0.1
8 +torch >= 2.1.0
9 transformers >= 4.34.0 # Required for Mistral.
10 -xformers == 0.0.22 # Required for Mistral.
10 +xformers >= 0.0.22.post7 # Required for CUDA 12.1.
11 fastapi
12 uvicorn[standard]
13 pydantic == 1.10.13 # Required for OpenAI server.