@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3cb07a36a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] Upgrade to pytorch 2.5 (#9588)

Signed-off-by: Bill Nell <bill@neuralmagic.com> Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

bnellnm committed Oct 27, 2024 at 05:44 UTC 3cb07a36a20f9af11346650559470d685e9dc711
8 files changed +48 -25
CMakeLists.txt
+2 -2
@@ -49,7 +49,7 @@ set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx11
49 # requirements.txt files and should be kept consistent. The ROCm torch
50 # versions are derived from Dockerfile.rocm
51 #
52 -set(TORCH_SUPPORTED_VERSION_CUDA "2.4.0")
52 +set(TORCH_SUPPORTED_VERSION_CUDA "2.5.0")
53 set(TORCH_SUPPORTED_VERSION_ROCM "2.5.0")
54
55 #
@@ -507,7 +507,7 @@ else()
507 FetchContent_Declare(
508 vllm-flash-attn
509 GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
510 - GIT_TAG 013f0c4fc47e6574060879d9734c1df8c5c273bd
510 + GIT_TAG 5259c586c403a4e4d8bf69973c159b40cc346fb9
511 GIT_PROGRESS TRUE
512 # Don't share the vllm-flash-attn build between build types
513 BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
cmake/utils.cmake
+1 -5
@@ -424,11 +424,7 @@ function (define_gpu_extension_target GPU_MOD_NAME)
424 # Don't use `TORCH_LIBRARIES` for CUDA since it pulls in a bunch of
425 # dependencies that are not necessary and may not be installed.
426 if (GPU_LANGUAGE STREQUAL "CUDA")
427 - if ("${CUDA_CUDA_LIB}" STREQUAL "")
428 - set(CUDA_CUDA_LIB "${CUDA_CUDA_LIBRARY}")
429 - endif()
430 - target_link_libraries(${GPU_MOD_NAME} PRIVATE ${CUDA_CUDA_LIB}
431 - ${CUDA_LIBRARIES})
427 + target_link_libraries(${GPU_MOD_NAME} PRIVATE CUDA::cudart CUDA::cuda_driver)
428 else()
429 target_link_libraries(${GPU_MOD_NAME} PRIVATE ${TORCH_LIBRARIES})
430 endif()
pyproject.toml
+1 -1
@@ -6,7 +6,7 @@ requires = [
6 "packaging",
7 "setuptools>=61",
8 "setuptools-scm>=8.0",
9 - "torch == 2.4.0",
9 + "torch == 2.5.0",
10 "wheel",
11 "jinja2",
12 ]
requirements-build.txt
+1 -1
@@ -4,6 +4,6 @@ ninja
4 packaging
5 setuptools>=61
6 setuptools-scm>=8
7 -torch==2.4.0
7 +torch==2.5.0
8 wheel
9 jinja2
requirements-cuda.txt
+3 -3
@@ -4,7 +4,7 @@
4 # Dependencies for NVIDIA GPUs
5 ray >= 2.9
6 nvidia-ml-py # for pynvml package
7 -torch == 2.4.0
7 +torch == 2.5.0
8 # These must be updated alongside torch
9 -torchvision == 0.19 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 -xformers == 0.0.27.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.4.0
9 +torchvision == 0.20 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
10 +xformers == 0.0.28.post2; platform_system == 'Linux' and platform_machine == 'x86_64' # Requires PyTorch 2.5.0
requirements-openvino.txt
+1 -1
@@ -1,7 +1,7 @@
1 # Common dependencies
2 -r requirements-common.txt
3
4 -torch == 2.4.0 # should be aligned with "common" vLLM torch version
4 +torch == 2.5.0 # should be aligned with "common" vLLM torch version
5 openvino >= 2024.4.0 # since 2024.4.0 both CPU and GPU support Paged Attention
6
7 optimum @ git+https://github.com/huggingface/optimum.git@main # latest optimum is used to support latest transformers version
tests/models/decoder_only/language/test_big_models.py
+34 -12
@@ -8,7 +8,7 @@ import pytest
8
9 from vllm.platforms import current_platform
10
11 -from ...utils import check_outputs_equal
11 +from ...utils import check_logprobs_close, check_outputs_equal
12
13 MODELS = [
14 "meta-llama/Llama-2-7b-hf",
@@ -43,18 +43,40 @@ def test_models(
43 dtype: str,
44 max_tokens: int,
45 ) -> None:
46 - with hf_runner(model, dtype=dtype) as hf_model:
47 - hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
46
49 - with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
50 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
51 -
52 - check_outputs_equal(
53 - outputs_0_lst=hf_outputs,
54 - outputs_1_lst=vllm_outputs,
55 - name_0="hf",
56 - name_1="vllm",
57 - )
47 + if model == "openbmb/MiniCPM3-4B":
48 + # the output becomes slightly different when upgrading to
49 + # pytorch 2.5 . Changing to logprobs checks instead of exact
50 + # output checks.
51 + NUM_LOG_PROBS = 8
52 + with hf_runner(model, dtype=dtype) as hf_model:
53 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
54 + example_prompts, max_tokens, NUM_LOG_PROBS)
55 +
56 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
57 + vllm_outputs = vllm_model.generate_greedy_logprobs(
58 + example_prompts, max_tokens, NUM_LOG_PROBS)
59 +
60 + check_logprobs_close(
61 + outputs_0_lst=hf_outputs,
62 + outputs_1_lst=vllm_outputs,
63 + name_0="hf",
64 + name_1="vllm",
65 + )
66 + else:
67 + with hf_runner(model, dtype=dtype) as hf_model:
68 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
69 +
70 + with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
71 + vllm_outputs = vllm_model.generate_greedy(example_prompts,
72 + max_tokens)
73 +
74 + check_outputs_equal(
75 + outputs_0_lst=hf_outputs,
76 + outputs_1_lst=vllm_outputs,
77 + name_0="hf",
78 + name_1="vllm",
79 + )
80
81
82 @pytest.mark.parametrize("model", MODELS)
vllm/platforms/cuda.py
+5
@@ -7,6 +7,7 @@ from functools import lru_cache, wraps
7 from typing import Callable, List, Tuple, TypeVar
8
9 import pynvml
10 +import torch
11 from typing_extensions import ParamSpec
12
13 from vllm.logger import init_logger
@@ -26,6 +27,10 @@ if pynvml.__file__.endswith("__init__.py"):
27 " and cause errors. See https://pypi.org/project/pynvml "
28 "for more information.")
29
30 +# pytorch 2.5 uses cudnn sdpa by default, which will cause crash on some models
31 +# see https://github.com/huggingface/diffusers/issues/9704 for details
32 +torch.backends.cuda.enable_cudnn_sdp(False)
33 +
34 # NVML utils
35 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
36 # all the related functions work on real physical device ids.