A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Upgrade to CUDA 12 (#1527)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Zhuohan Li committed
Nov 8, 2023 at 14:17 UTC
06458a0b42449398aa2ba001d9dbaff256159448
5 files changed
+15
-7
.github/workflows/publish.yml
+4
-4
@@ -43,14 +43,14 @@ jobs:
43
name: Build Wheel
44
runs-on: ${{ matrix.os }}
45
needs: release
46
-
46
+
47
strategy:
48
fail-fast: false
49
matrix:
50
os: ['ubuntu-20.04']
51
python-version: ['3.8', '3.9', '3.10', '3.11']
52
- pytorch-version: ['2.0.1']
53
- cuda-version: ['11.8'] # Github runner can't build anything older than 11.8
52
+ pytorch-version: ['2.1.0']
53
+ cuda-version: ['12.1']
54
55
steps:
56
- name: Checkout
@@ -82,7 +82,7 @@ jobs:
82
asset_name=${wheel_name//"linux"/"manylinux1"}
83
echo "wheel_name=${wheel_name}" >> $GITHUB_ENV
84
echo "asset_name=${asset_name}" >> $GITHUB_ENV
85
-
85
+
86
- name: Upload Release Asset
87
uses: actions/upload-release-asset@v1
88
env:
.github/workflows/scripts/build.sh
+3
@@ -11,5 +11,8 @@ LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
11
$python_executable -m pip install wheel packaging
12
$python_executable -m pip install -r requirements.txt
13
14
+# Limit the number of parallel jobs to avoid OOM
15
+export MAX_JOBS=1
16
+
17
# Build
18
$python_executable setup.py bdist_wheel --dist-dir=dist
.github/workflows/scripts/cuda-install.sh
+5
@@ -16,3 +16,8 @@ sudo apt clean
16
# Test nvcc
17
PATH=/usr/local/cuda-$1/bin:${PATH}
18
nvcc --version
19
+
20
+# Log gcc, g++, c++ versions
21
+gcc --version
22
+g++ --version
23
+c++ --version
pyproject.toml
+1
-1
@@ -3,7 +3,7 @@ requires = [
3
"ninja",
4
"packaging",
5
"setuptools",
6
- "torch == 2.0.1",
6
+ "torch >= 2.1.0",
7
"wheel",
8
]
9
build-backend = "setuptools.build_meta"
requirements.txt
+2
-2
@@ -5,9 +5,9 @@ pandas # Required for Ray data.
5
pyarrow # Required for Ray data.
6
sentencepiece # Required for LLaMA tokenizer.
7
numpy
8
-torch == 2.0.1
8
+torch >= 2.1.0
9
transformers >= 4.34.0 # Required for Mistral.
10
-xformers == 0.0.22 # Required for Mistral.
10
+xformers >= 0.0.22.post7 # Required for CUDA 12.1.
11
fastapi
12
uvicorn[standard]
13
pydantic == 1.10.13 # Required for OpenAI server.