@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d346ec695e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[CI/Build] Consolidate model loader tests and requirements (#25765)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Sep 27, 2025 at 12:45 UTC d346ec695ef5dc74cde338a6bc3857e91c311ab2
20 files changed +48 -90
.buildkite/test-pipeline.yaml
+4 -15
@@ -465,29 +465,18 @@ steps:
465 commands:
466 - pytest -v -s kernels/mamba
467
468 -- label: Tensorizer Test # 14min
469 - timeout_in_minutes: 25
470 - mirror_hardwares: [amdexperimental]
471 - source_file_dependencies:
472 - - vllm/model_executor/model_loader
473 - - tests/tensorizer_loader
474 - - tests/entrypoints/openai/test_tensorizer_entrypoint.py
475 - commands:
476 - - apt-get update && apt-get install -y curl libsodium23
477 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
478 - - pytest -v -s tensorizer_loader
479 - - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py
480 -
481 -- label: Model Executor Test # 7min
482 - timeout_in_minutes: 20
468 +- label: Model Executor Test # ???
469 + timeout_in_minutes: 60
470 mirror_hardwares: [amdexperimental]
471 source_file_dependencies:
472 - vllm/model_executor
473 - tests/model_executor
474 + - tests/entrypoints/openai/test_tensorizer_entrypoint.py
475 commands:
476 - apt-get update && apt-get install -y curl libsodium23
477 - export VLLM_WORKER_MULTIPROC_METHOD=spawn
478 - pytest -v -s model_executor
479 + - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py
480
481 - label: Benchmarks # 11min
482 timeout_in_minutes: 20
.github/mergify.yml
+1 -1
@@ -274,7 +274,7 @@ pull_request_rules:
274 - files~=^vllm/model_executor/model_loader/tensorizer.py
275 - files~=^vllm/model_executor/model_loader/tensorizer_loader.py
276 - files~=^tests/entrypoints/openai/test_tensorizer_entrypoint.py
277 - - files~=^tests/tensorizer_loader/
277 + - files~=^tests/model_executor/model_loader/tensorizer_loader/
278 actions:
279 assign:
280 users:
docker/Dockerfile
+1 -1
@@ -546,7 +546,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
546 else \
547 BITSANDBYTES_VERSION="0.46.1"; \
548 fi; \
549 - uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' boto3 runai-model-streamer runai-model-streamer[s3]
549 + uv pip install --system accelerate hf_transfer modelscope "bitsandbytes>=${BITSANDBYTES_VERSION}" 'timm>=1.0.17' 'runai-model-streamer[s3]>=0.14.0'
550
551 ENV VLLM_USAGE_SOURCE production-docker-image
552
requirements/nightly_torch_test.txt
+1 -2
@@ -43,7 +43,6 @@ tritonclient==2.51.0
43 numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
44 numba == 0.61.2; python_version > '3.9'
45 numpy
46 -runai-model-streamer==0.11.0
47 -runai-model-streamer-s3==0.11.0
46 +runai-model-streamer[s3]==0.14.0
47 fastsafetensors>=0.1.10
48 pydantic>=2.10 # 2.9 leads to error on python 3.10
requirements/rocm.txt
+1 -4
@@ -5,8 +5,6 @@ numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Req
5 numba == 0.61.2; python_version > '3.9'
6
7 # Dependencies for AMD GPUs
8 -boto3
9 -botocore
8 datasets
9 ray[cgraph]>=2.48.0 # Ray Compiled Graph, required for pipeline parallelism in V1.
10 peft
@@ -15,7 +13,6 @@ tensorizer==2.10.1
13 packaging>=24.2
14 setuptools>=77.0.3,<80.0.0
15 setuptools-scm>=8
18 -runai-model-streamer==0.11.0
19 -runai-model-streamer-s3==0.11.0
16 +runai-model-streamer[s3]==0.14.0
17 conch-triton-kernels==1.2.1
18 timm>=1.0.17
\ No newline at end of file
requirements/test.in
+1 -2
@@ -51,8 +51,7 @@ tritonclient==2.51.0
51 numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
52 numba == 0.61.2; python_version > '3.9'
53 numpy
54 -runai-model-streamer==0.11.0
55 -runai-model-streamer-s3==0.11.0
54 +runai-model-streamer[s3]==0.14.0
55 fastsafetensors>=0.1.10
56 pydantic>=2.10 # 2.9 leads to error on python 3.10
57 decord==0.6.0
requirements/test.txt
+6 -4
@@ -72,7 +72,9 @@ blobfile==3.0.0
72 bm25s==0.2.13
73 # via mteb
74 boto3==1.35.57
75 - # via tensorizer
75 + # via
76 + # runai-model-streamer-s3
77 + # tensorizer
78 botocore==1.35.57
79 # via
80 # boto3
@@ -925,10 +927,10 @@ rsa==4.9.1
927 # via google-auth
928 rtree==1.4.0
929 # via torchgeo
928 -runai-model-streamer==0.11.0
929 - # via -r requirements/test.in
930 -runai-model-streamer-s3==0.11.0
930 +runai-model-streamer==0.14.0
931 # via -r requirements/test.in
932 +runai-model-streamer-s3==0.14.0
933 + # via runai-model-streamer
934 s3transfer==0.10.3
935 # via boto3
936 sacrebleu==2.4.3
setup.py
+1 -4
@@ -654,10 +654,7 @@ setup(
654 "bench": ["pandas", "datasets"],
655 "tensorizer": ["tensorizer==2.10.1"],
656 "fastsafetensors": ["fastsafetensors >= 0.1.10"],
657 - "runai": [
658 - "runai-model-streamer >= 0.14.0", "runai-model-streamer-gcs",
659 - "google-cloud-storage", "runai-model-streamer-s3", "boto3"
660 - ],
657 + "runai": ["runai-model-streamer[s3,gcs] >= 0.14.0"],
658 "audio": ["librosa", "soundfile",
659 "mistral_common[audio]"], # Required for audio processing
660 "video": [], # Kept for backwards compatibility
tests/model_executor/conftest.py deleted
-52
@@ -1,52 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -
6 -
7 -@pytest.fixture
8 -def sample_regex():
9 - return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
10 - r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
11 -
12 -
13 -@pytest.fixture
14 -def sample_json_schema():
15 - return {
16 - "type": "object",
17 - "properties": {
18 - "name": {
19 - "type": "string"
20 - },
21 - "age": {
22 - "type": "integer"
23 - },
24 - "skills": {
25 - "type": "array",
26 - "items": {
27 - "type": "string",
28 - "maxLength": 10
29 - },
30 - "minItems": 3
31 - },
32 - "work_history": {
33 - "type": "array",
34 - "items": {
35 - "type": "object",
36 - "properties": {
37 - "company": {
38 - "type": "string"
39 - },
40 - "duration": {
41 - "type": "number"
42 - },
43 - "position": {
44 - "type": "string"
45 - }
46 - },
47 - "required": ["company", "position"]
48 - }
49 - }
50 - },
51 - "required": ["name", "age", "skills", "work_history"]
52 - }
tests/model_executor/model_loader/fastsafetensors_loader/__init__.py renamed
tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py renamed
tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py renamed
tests/model_executor/model_loader/runai_model_streamer/__init__.py renamed
tests/model_executor/model_loader/runai_model_streamer/test_runai_model_streamer_loader.py renamed
tests/model_executor/model_loader/runai_model_streamer/test_runai_utils.py renamed
tests/model_executor/model_loader/runai_model_streamer/test_weight_utils.py renamed
tests/model_executor/model_loader/tensorizer_loader/__init__.py renamed
tests/model_executor/model_loader/tensorizer_loader/conftest.py renamed
tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py renamed
+1 -1
@@ -14,6 +14,7 @@ import pytest
14 import torch
15
16 import vllm.model_executor.model_loader.tensorizer
17 +from tests.utils import VLLM_PATH, RemoteOpenAIServer
18 from vllm import LLM, SamplingParams
19 from vllm.engine.arg_utils import EngineArgs
20 # yapf: disable
@@ -27,7 +28,6 @@ from vllm.model_executor.model_loader.tensorizer_loader import (
28 # yapf: enable
29 from vllm.utils import PlaceholderModule
30
30 -from ..utils import VLLM_PATH, RemoteOpenAIServer
31 from .conftest import DummyExecutor, assert_from_collective_rpc
32
33 try:
vllm/model_executor/model_loader/weight_utils.py
+31 -4
@@ -639,6 +639,19 @@ def runai_safetensors_weights_iterator(
639 yield from tensor_iter
640
641
642 +def _init_loader(
643 + pg: torch.distributed.ProcessGroup,
644 + device: torch.device,
645 + f_list: list[str],
646 + *,
647 + nogds: bool = False,
648 +):
649 + loader = SafeTensorsFileLoader(pg, device, nogds=nogds)
650 + rank_file_map = {i: [f] for i, f in enumerate(f_list)}
651 + loader.add_filenames(rank_file_map)
652 + return loader
653 +
654 +
655 def fastsafetensors_weights_iterator(
656 hf_weights_files: list[str],
657 use_tqdm_on_load: bool,
@@ -656,17 +669,31 @@ def fastsafetensors_weights_iterator(
669 for i in range(0, len(hf_weights_files), pg.size())
670 ]
671
672 + nogds = False
673 +
674 for f_list in tqdm(
675 weight_files_sub_lists,
676 desc="Loading safetensors using Fastsafetensor loader",
677 disable=not enable_tqdm(use_tqdm_on_load),
678 bar_format=_BAR_FORMAT,
679 ):
665 - loader = SafeTensorsFileLoader(pg, device)
666 - rank_file_map = {i: [f] for i, f in enumerate(f_list)}
667 - loader.add_filenames(rank_file_map)
680 + loader = _init_loader(pg, device, f_list, nogds=nogds)
681 try:
669 - fb = loader.copy_files_to_device()
682 + try:
683 + fb = loader.copy_files_to_device()
684 + except RuntimeError as e:
685 + if "gds" not in str(e):
686 + raise
687 +
688 + loader.close()
689 + nogds = True
690 + logger.warning_once(
691 + "GDS not enabled, setting `nogds=True`.\n"
692 + "For more information, see: https://github.com/foundation-model-stack/fastsafetensors?tab=readme-ov-file#basic-api-usages"
693 + )
694 + loader = _init_loader(pg, device, f_list, nogds=nogds)
695 + fb = loader.copy_files_to_device()
696 +
697 try:
698 keys = list(fb.key_to_rank_lidx.keys())
699 for k in keys: