A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] Upgrade to pytorch 2.5 (#9588)
Signed-off-by: Bill Nell <bill@neuralmagic.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
bnellnm committed
Oct 27, 2024 at 05:44 UTC
3cb07a36a20f9af11346650559470d685e9dc711
8 files changed
+48
-25
CMakeLists.txt
+2
-2
@@ -49,7 +49,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
49
# requirements.txt files and should be kept consistent. The ROCm torch
50
# versions are derived from Dockerfile.rocm
51
#
52
-set(TORCH_SUPPORTED_VERSION_CUDA "2.4.0")
52
+set(TORCH_SUPPORTED_VERSION_CUDA "2.5.0")
53
set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
54
55
#
@@ -507,7 +507,7 @@ else()
507
FetchContent_Declare(
508
vllm-flash-attn
509
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
510
- GIT_TAG 013f0c4fc47e6574060879d9734c1df8c5c273bd
510
+ GIT_TAG 5259c586c403a4e4d8bf69973c159b40cc346fb9
511
GIT_PROGRESS TRUE
512
# Don't share the vllm-flash-attn build between build types
513
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
cmake/utils.cmake
+1
-5
@@ -424,11 +424,7 @@ function (define_gpu_extension_target GPU_MOD_NAME)
424
# Don't use `TORCH_LIBRARIES` for CUDA since it pulls in a bunch of
425
# dependencies that are not necessary and may not be installed.
426
if (GPU_LANGUAGE STREQUAL "CUDA")
427
- if ("${CUDA_CUDA_LIB}" STREQUAL "")
428
- set(CUDA_CUDA_LIB "${CUDA_CUDA_LIBRARY}")
429
- endif()
430
- target_link_libraries(${GPU_MOD_NAME} PRIVATE ${CUDA_CUDA_LIB}
431
- ${CUDA_LIBRARIES})
427
+ target_link_libraries(${GPU_MOD_NAME} PRIVATE CUDA::cudart CUDA::cuda_driver)
428
else()
429
target_link_libraries(${GPU_MOD_NAME} PRIVATE ${TORCH_LIBRARIES})
430
endif()
pyproject.toml
+1
-1
@@ -6,7 +6,7 @@ requires = [
6
"packaging",
7
"setuptools>=61",
8
"setuptools-scm>=8.0",
9
- "torch == 2.4.0",
9
+ "torch == 2.5.0",
10
"wheel",
11
"jinja2",
12
]
requirements-build.txt
+1
-1
@@ -4,6 +4,6 @@ ninja
4
packaging
5
setuptools>=61
6
setuptools-scm>=8
7
-torch==2.4.0
7
+torch==2.5.0
8
wheel
9
jinja2
requirements-cuda.txt
+3
-3
@@ -4,7 +4,7 @@
4
# Dependencies for NVIDIA GPUs
5
ray >= 2.9
6
nvidia-ml-py # for pynvml package
7
-torch == 2.4.0
7
+torch == 2.5.0
8
# These must be updated alongside torch
9
-torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
-xformers == 0.0.27.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
9
+torchvision == 0.20 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10
+xformers == 0.0.28.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.5.0
requirements-openvino.txt
+1
-1
@@ -1,7 +1,7 @@
1
# Common dependencies
2
-r requirements-common.txt
3
4
-torch == 2.4.0 # should be aligned with "common" vLLM torch version
4
+torch == 2.5.0 # should be aligned with "common" vLLM torch version
5
openvino >= 2024.4.0 # since 2024.4.0 both CPU and GPU support Paged Attention
6
7
optimum @ git+https://github.com/huggingface/optimum.git@main # latest optimum is used to support latest transformers version
tests/models/decoder_only/language/test_big_models.py
+34
-12
@@ -8,7 +8,7 @@ import pytest
8
9
from vllm.platforms import current_platform
10
11
-from ...utils import check_outputs_equal
11
+from ...utils import check_logprobs_close, check_outputs_equal
12
13
MODELS = [
14
"meta-llama/Llama-2-7b-hf",
@@ -43,18 +43,40 @@ def test_models(
43
dtype: str,
44
max_tokens: int,
45
) -> None:
46
- with hf_runner(model, dtype=dtype) as hf_model:
47
- hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
46
49
- with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
50
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
51
-
52
- check_outputs_equal(
53
- outputs_0_lst=hf_outputs,
54
- outputs_1_lst=vllm_outputs,
55
- name_0="hf",
56
- name_1="vllm",
57
- )
47
+ if model == "openbmb/MiniCPM3-4B":
48
+ # the output becomes slightly different when upgrading to
49
+ # pytorch 2.5 . Changing to logprobs checks instead of exact
50
+ # output checks.
51
+ NUM_LOG_PROBS = 8
52
+ with hf_runner(model, dtype=dtype) as hf_model:
53
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
54
+ example_prompts, max_tokens, NUM_LOG_PROBS)
55
+
56
+ with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
57
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
58
+ example_prompts, max_tokens, NUM_LOG_PROBS)
59
+
60
+ check_logprobs_close(
61
+ outputs_0_lst=hf_outputs,
62
+ outputs_1_lst=vllm_outputs,
63
+ name_0="hf",
64
+ name_1="vllm",
65
+ )
66
+ else:
67
+ with hf_runner(model, dtype=dtype) as hf_model:
68
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69
+
70
+ with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
71
+ vllm_outputs = vllm_model.generate_greedy(example_prompts,
72
+ max_tokens)
73
+
74
+ check_outputs_equal(
75
+ outputs_0_lst=hf_outputs,
76
+ outputs_1_lst=vllm_outputs,
77
+ name_0="hf",
78
+ name_1="vllm",
79
+ )
80
81
82
@pytest.mark.parametrize("model", MODELS)
vllm/platforms/cuda.py
+5
@@ -7,6 +7,7 @@ from functools import lru_cache, wraps
7
from typing import Callable, List, Tuple, TypeVar
8
9
import pynvml
10
+import torch
11
from typing_extensions import ParamSpec
12
13
from vllm.logger import init_logger
@@ -26,6 +27,10 @@ if pynvml.__file__.endswith("__init__.py"):
27
" and cause errors. See https://pypi.org/project/pynvml "
28
"for more information.")
29
30
+# pytorch 2.5 uses cudnn sdpa by default, which will cause crash on some models
31
+# see https://github.com/huggingface/diffusers/issues/9704 for details
32
+torch.backends.cuda.enable_cudnn_sdp(False)
33
+
34
# NVML utils
35
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
36
# all the related functions work on real physical device ids.