@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2bb0e1a799

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[Bugfix][ROCm] running new process using spawn method for rocm in tests. (#14810)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

vllmellm committed Mar 17, 2025 at 19:33 UTC 2bb0e1a7994d55e964f5bb3acb79f8491fcb92f7

21 files changed +174 -99

tests/basic_correctness/test_cumem.py

+5 -5

@@ -7,10 +7,10 @@ from vllm import LLM, SamplingParams

7 from vllm.device_allocator.cumem import CuMemAllocator

8 from vllm.utils import GiB_bytes

10 -from ..utils import fork_new_process_for_each_test

10 +from ..utils import create_new_process_for_each_test

13 -@fork_new_process_for_each_test

13 +@create_new_process_for_each_test()

14 def test_python_error():

15 """

16 Test if Python error occurs when there's low-level

@@ -36,7 +36,7 @@ def test_python_error():

36 allocator.wake_up()

39 -@fork_new_process_for_each_test

39 +@create_new_process_for_each_test()

40 def test_basic_cumem():

41 # some tensors from default memory pool

42 shape = (1024, 1024)

@@ -69,7 +69,7 @@ def test_basic_cumem():

69 assert torch.allclose(output, torch.ones_like(output) * 3)

72 -@fork_new_process_for_each_test

72 +@create_new_process_for_each_test()

73 def test_cumem_with_cudagraph():

74 allocator = CuMemAllocator.get_instance()

75 with allocator.use_memory_pool():

@@ -114,7 +114,7 @@ def test_cumem_with_cudagraph():

114 assert torch.allclose(y, x + 1)

115

116

117 -@fork_new_process_for_each_test

117 +@create_new_process_for_each_test()

118 @pytest.mark.parametrize(

119 "model, use_v1",

120 [

tests/compile/test_full_graph.py

+2 -2

@@ -12,7 +12,7 @@ from vllm import LLM, SamplingParams

12 from vllm.config import CompilationLevel

13 from vllm.platforms import current_platform

15 -from ..utils import fork_new_process_for_each_test

15 +from ..utils import create_new_process_for_each_test

18 @pytest.fixture(params=None, name="model_info")

@@ -78,7 +78,7 @@ def models_list_fixture(request):

78 [CompilationLevel.DYNAMO_ONCE, CompilationLevel.PIECEWISE],

79 )

80 @pytest.mark.parametrize("model_info", "", indirect=True)

81 -@fork_new_process_for_each_test

81 +@create_new_process_for_each_test()

82 def test_full_graph(

83 monkeypatch: pytest.MonkeyPatch,

84 model_info: tuple[str, dict[str, Any]],

tests/distributed/test_expert_parallel.py

+2 -2

@@ -8,7 +8,7 @@ import pytest

8 from vllm.config import TaskOption

9 from vllm.logger import init_logger

11 -from ..utils import compare_two_settings, fork_new_process_for_each_test

11 +from ..utils import compare_two_settings, create_new_process_for_each_test

13 logger = init_logger("test_expert_parallel")

@@ -209,7 +209,7 @@ def _compare_tp(

209 for params in settings.iter_params(model_name)

210 ],

211 )

212 -@fork_new_process_for_each_test

212 +@create_new_process_for_each_test()

213 def test_ep(

214 model_name: str,

215 parallel_setup: ParallelSetup,

tests/distributed/test_pipeline_parallel.py

+4 -4

@@ -17,7 +17,7 @@ from vllm.config import TaskOption

17 from vllm.logger import init_logger

19 from ..models.registry import HF_EXAMPLE_MODELS

20 -from ..utils import compare_two_settings, fork_new_process_for_each_test

20 +from ..utils import compare_two_settings, create_new_process_for_each_test

22 logger = init_logger("test_pipeline_parallel")

@@ -402,7 +402,7 @@ def _compare_tp(

402 for params in settings.iter_params(model_id) if model_id in TEST_MODELS

403 ],

404 )

405 -@fork_new_process_for_each_test

405 +@create_new_process_for_each_test()

406 def test_tp_language_generation(

407 model_id: str,

408 parallel_setup: ParallelSetup,

@@ -431,7 +431,7 @@ def test_tp_language_generation(

431 for params in settings.iter_params(model_id) if model_id in TEST_MODELS

432 ],

433 )

434 -@fork_new_process_for_each_test

434 +@create_new_process_for_each_test()

435 def test_tp_language_embedding(

436 model_id: str,

437 parallel_setup: ParallelSetup,

@@ -460,7 +460,7 @@ def test_tp_language_embedding(

460 for params in settings.iter_params(model_id) if model_id in TEST_MODELS

461 ],

462 )

463 -@fork_new_process_for_each_test

463 +@create_new_process_for_each_test()

464 def test_tp_multimodal_generation(

465 model_id: str,

466 parallel_setup: ParallelSetup,

tests/distributed/test_pp_cudagraph.py

+2 -2

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING

6 import pytest

8 -from ..utils import compare_two_settings, fork_new_process_for_each_test

8 +from ..utils import compare_two_settings, create_new_process_for_each_test

10 if TYPE_CHECKING:

11 from typing_extensions import LiteralString

@@ -18,7 +18,7 @@ if TYPE_CHECKING:

18 "FLASH_ATTN",

19 "FLASHINFER",

20 ])

21 -@fork_new_process_for_each_test

21 +@create_new_process_for_each_test()

22 def test_pp_cudagraph(

23 monkeypatch: pytest.MonkeyPatch,

24 PP_SIZE: int,

tests/entrypoints/llm/test_collective_rpc.py

+2 -2

@@ -4,12 +4,12 @@ import pytest

5 from vllm import LLM

7 -from ...utils import fork_new_process_for_each_test

7 +from ...utils import create_new_process_for_each_test

10 @pytest.mark.parametrize("tp_size", [1, 2])

11 @pytest.mark.parametrize("backend", ["mp", "ray"])

12 -@fork_new_process_for_each_test

12 +@create_new_process_for_each_test()

13 def test_collective_rpc(tp_size, backend):

14 if tp_size == 1 and backend == "ray":

15 pytest.skip("Skip duplicate test case")

tests/lora/test_chatglm3_tp.py

+4 -5

@@ -3,10 +3,9 @@

3 import pytest

5 import vllm

6 -from tests.utils import fork_new_process_for_each_test

6 from vllm.lora.request import LoRARequest

9 -from ..utils import multi_gpu_test

8 +from ..utils import create_new_process_for_each_test, multi_gpu_test

10 MODEL_PATH = "THUDM/chatglm3-6b"

@@ -55,7 +54,7 @@ def v1(run_with_both_engines_lora):

54 pass

58 -@fork_new_process_for_each_test

57 +@create_new_process_for_each_test()

58 def test_chatglm3_lora(chatglm3_lora_files):

59 llm = vllm.LLM(MODEL_PATH,

60 max_model_len=1024,

@@ -75,7 +74,7 @@ def test_chatglm3_lora(chatglm3_lora_files):

76 @multi_gpu_test(num_gpus=4)

78 -@fork_new_process_for_each_test

77 +@create_new_process_for_each_test()

78 def test_chatglm3_lora_tp4(chatglm3_lora_files):

79 llm = vllm.LLM(MODEL_PATH,

80 max_model_len=1024,

@@ -96,7 +95,7 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):

97 @multi_gpu_test(num_gpus=4)

99 -@fork_new_process_for_each_test

98 +@create_new_process_for_each_test()

99 def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):

100 llm = vllm.LLM(MODEL_PATH,

101 max_model_len=1024,

tests/lora/test_llama_tp.py

+6 -7

@@ -4,10 +4,9 @@ import pytest

4 import ray

6 import vllm

7 -from tests.utils import fork_new_process_for_each_test

7 from vllm.lora.request import LoRARequest

10 -from ..utils import multi_gpu_test

9 +from ..utils import create_new_process_for_each_test, multi_gpu_test

11 MODEL_PATH = "meta-llama/Llama-2-7b-hf"

@@ -82,7 +81,7 @@ def v1(run_with_both_engines_lora):

82 # V1 Test: Failing due to numerics on V1.

83 @pytest.mark.skip_v1

85 -@fork_new_process_for_each_test

84 +@create_new_process_for_each_test()

85 def test_llama_lora(sql_lora_files):

87 llm = vllm.LLM(MODEL_PATH,

@@ -97,7 +96,7 @@ def test_llama_lora(sql_lora_files):

96 # Skipping for v1 as v1 doesn't have a good way to expose the num_gpu_blocks

97 # used by the engine yet.

98 @pytest.mark.skip_v1

100 -@fork_new_process_for_each_test

99 +@create_new_process_for_each_test()

100 def test_llama_lora_warmup(sql_lora_files):

101 """Test that the LLM initialization works with a warmup LORA path and

102 is more conservative"""

@@ -128,7 +127,7 @@ def test_llama_lora_warmup(sql_lora_files):

127 # V1 Test: Failing due to numerics on V1.

128 @pytest.mark.skip_v1

129 @multi_gpu_test(num_gpus=4)

131 -@fork_new_process_for_each_test

130 +@create_new_process_for_each_test()

131 def test_llama_lora_tp4(sql_lora_files):

132

133 llm = vllm.LLM(

@@ -143,7 +142,7 @@ def test_llama_lora_tp4(sql_lora_files):

142

143

144 @multi_gpu_test(num_gpus=4)

146 -@fork_new_process_for_each_test

145 +@create_new_process_for_each_test()

146 def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):

147

148 llm = vllm.LLM(

@@ -159,7 +158,7 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):

158

159

160 @multi_gpu_test(num_gpus=4)

162 -@fork_new_process_for_each_test

161 +@create_new_process_for_each_test()

162 def test_llama_lora_tp4_fully_sharded_enable_bias(sql_lora_files):

163

164 llm = vllm.LLM(

tests/lora/test_minicpmv_tp.py

+5 -4

@@ -3,11 +3,12 @@

3 import pytest

5 import vllm

6 -from tests.utils import fork_new_process_for_each_test

6 from vllm.assets.image import ImageAsset

7 from vllm.lora.request import LoRARequest

8 from vllm.platforms import current_platform

10 +from ..utils import create_new_process_for_each_test

11 +

12 MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"

14 PROMPT_TEMPLATE = (

@@ -57,7 +58,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

58 @pytest.mark.xfail(

59 current_platform.is_rocm(),

60 reason="MiniCPM-V dependency xformers incompatible with ROCm")

60 -@fork_new_process_for_each_test

61 +@create_new_process_for_each_test()

62 def test_minicpmv_lora(minicpmv_lora_files):

63 llm = vllm.LLM(

64 MODEL_PATH,

@@ -80,7 +81,7 @@ def test_minicpmv_lora(minicpmv_lora_files):

81 @pytest.mark.xfail(

82 current_platform.is_rocm(),

83 reason="MiniCPM-V dependency xformers incompatible with ROCm")

83 -@fork_new_process_for_each_test

84 +@create_new_process_for_each_test()

85 def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):

86 llm = vllm.LLM(

87 MODEL_PATH,

@@ -101,7 +102,7 @@ def test_minicpmv_tp4_wo_fully_sharded_loras(minicpmv_lora_files):

102 @pytest.mark.xfail(

103 current_platform.is_rocm(),

104 reason="MiniCPM-V dependency xformers incompatible with ROCm")

104 -@fork_new_process_for_each_test

105 +@create_new_process_for_each_test()

106 def test_minicpmv_tp4_fully_sharded_loras(minicpmv_lora_files):

107 llm = vllm.LLM(

108 MODEL_PATH,

tests/lora/test_transfomers_model.py

+4 -5

@@ -3,10 +3,9 @@

3 import pytest

5 import vllm

6 -from tests.utils import fork_new_process_for_each_test

6 from vllm.lora.request import LoRARequest

9 -from ..utils import multi_gpu_test

8 +from ..utils import create_new_process_for_each_test, multi_gpu_test

10 MODEL_PATH = "ArthurZ/ilama-3.2-1B"

@@ -56,7 +55,7 @@ def v1(run_with_both_engines_lora):

57 @pytest.mark.skip_v1

59 -@fork_new_process_for_each_test

58 +@create_new_process_for_each_test()

59 def test_ilama_lora(ilama_lora_files):

60 llm = vllm.LLM(MODEL_PATH,

61 max_model_len=1024,

@@ -77,7 +76,7 @@ def test_ilama_lora(ilama_lora_files):

77 @pytest.mark.skip_v1

78 @multi_gpu_test(num_gpus=4)

80 -@fork_new_process_for_each_test

79 +@create_new_process_for_each_test()

80 def test_ilama_lora_tp4(ilama_lora_files):

81 llm = vllm.LLM(MODEL_PATH,

82 max_model_len=1024,

@@ -99,7 +98,7 @@ def test_ilama_lora_tp4(ilama_lora_files):

99 @pytest.mark.skip_v1

100 @multi_gpu_test(num_gpus=4)

102 -@fork_new_process_for_each_test

101 +@create_new_process_for_each_test()

102 def test_ilama_lora_tp4_fully_sharded_loras(ilama_lora_files):

103 llm = vllm.LLM(MODEL_PATH,

104 max_model_len=1024,

tests/models/decoder_only/vision_language/test_models.py

+15 -15

@@ -17,7 +17,7 @@ from vllm.utils import identity

18 from ....conftest import (IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets,

19 _VideoAssets)

20 -from ....utils import (fork_new_process_for_each_test, large_gpu_mark,

20 +from ....utils import (create_new_process_for_each_test, large_gpu_mark,

21 multi_gpu_marks)

22 from ...utils import check_outputs_equal

23 from .vlm_utils import custom_inputs, model_utils, runners

@@ -592,7 +592,7 @@ VLM_TEST_SETTINGS = _mark_splits(VLM_TEST_SETTINGS, num_groups=2)

592 get_parametrized_options(

593 VLM_TEST_SETTINGS,

594 test_type=VLMTestType.IMAGE,

595 - fork_new_process_for_each_test=False,

595 + create_new_process_for_each_test=False,

596 ))

597 def test_single_image_models(tmp_path: PosixPath, model_type: str,

598 test_case: ExpandableVLMTestArgs,

@@ -617,7 +617,7 @@ def test_single_image_models(tmp_path: PosixPath, model_type: str,

617 get_parametrized_options(

618 VLM_TEST_SETTINGS,

619 test_type=VLMTestType.MULTI_IMAGE,

620 - fork_new_process_for_each_test=False,

620 + create_new_process_for_each_test=False,

621 ))

622 def test_multi_image_models(tmp_path: PosixPath, model_type: str,

623 test_case: ExpandableVLMTestArgs,

@@ -642,7 +642,7 @@ def test_multi_image_models(tmp_path: PosixPath, model_type: str,

642 get_parametrized_options(

643 VLM_TEST_SETTINGS,

644 test_type=VLMTestType.EMBEDDING,

645 - fork_new_process_for_each_test=False,

645 + create_new_process_for_each_test=False,

646 ))

647 def test_image_embedding_models(model_type: str,

648 test_case: ExpandableVLMTestArgs,

@@ -666,7 +666,7 @@ def test_image_embedding_models(model_type: str,

666 get_parametrized_options(

667 VLM_TEST_SETTINGS,

668 test_type=VLMTestType.VIDEO,

669 - fork_new_process_for_each_test=False,

669 + create_new_process_for_each_test=False,

670 ))

671 def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,

672 hf_runner: type[HfRunner], vllm_runner: type[VllmRunner],

@@ -688,7 +688,7 @@ def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,

688 get_parametrized_options(

689 VLM_TEST_SETTINGS,

690 test_type=VLMTestType.CUSTOM_INPUTS,

691 - fork_new_process_for_each_test=False,

691 + create_new_process_for_each_test=False,

692 ))

693 def test_custom_inputs_models(

694 model_type: str,

@@ -714,9 +714,9 @@ def test_custom_inputs_models(

714 get_parametrized_options(

715 VLM_TEST_SETTINGS,

716 test_type=VLMTestType.IMAGE,

717 - fork_new_process_for_each_test=True,

717 + create_new_process_for_each_test=True,

718 ))

719 -@fork_new_process_for_each_test

719 +@create_new_process_for_each_test()

720 def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,

721 test_case: ExpandableVLMTestArgs,

722 hf_runner: type[HfRunner],

@@ -740,9 +740,9 @@ def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,

740 get_parametrized_options(

741 VLM_TEST_SETTINGS,

742 test_type=VLMTestType.MULTI_IMAGE,

743 - fork_new_process_for_each_test=True,

743 + create_new_process_for_each_test=True,

744 ))

745 -@fork_new_process_for_each_test

745 +@create_new_process_for_each_test()

746 def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,

747 test_case: ExpandableVLMTestArgs,

748 hf_runner: type[HfRunner],

@@ -766,9 +766,9 @@ def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,

766 get_parametrized_options(

767 VLM_TEST_SETTINGS,

768 test_type=VLMTestType.EMBEDDING,

769 - fork_new_process_for_each_test=True,

769 + create_new_process_for_each_test=True,

770 ))

771 -@fork_new_process_for_each_test

771 +@create_new_process_for_each_test()

772 def test_image_embedding_models_heavy(model_type: str,

773 test_case: ExpandableVLMTestArgs,

774 hf_runner: type[HfRunner],

@@ -791,7 +791,7 @@ def test_image_embedding_models_heavy(model_type: str,

791 get_parametrized_options(

792 VLM_TEST_SETTINGS,

793 test_type=VLMTestType.VIDEO,

794 - fork_new_process_for_each_test=True,

794 + create_new_process_for_each_test=True,

795 ))

796 def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

797 hf_runner: type[HfRunner],

@@ -814,9 +814,9 @@ def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,

814 get_parametrized_options(

815 VLM_TEST_SETTINGS,

816 test_type=VLMTestType.CUSTOM_INPUTS,

817 - fork_new_process_for_each_test=True,

817 + create_new_process_for_each_test=True,

818 ))

819 -@fork_new_process_for_each_test

819 +@create_new_process_for_each_test()

820 def test_custom_inputs_models_heavy(

821 model_type: str,

822 test_case: ExpandableVLMTestArgs,

tests/models/decoder_only/vision_language/vlm_utils/case_filtering.py

+6 -6

@@ -13,9 +13,9 @@ from .types import (EMBEDDING_SIZE_FACTORS, ExpandableVLMTestArgs,

13 ImageSizeWrapper, SizeType, VLMTestInfo, VLMTestType)

16 -def get_filtered_test_settings(test_settings: dict[str, VLMTestInfo],

17 - test_type: VLMTestType,

18 - fork_per_test: bool) -> dict[str, VLMTestInfo]:

16 +def get_filtered_test_settings(

17 + test_settings: dict[str, VLMTestInfo], test_type: VLMTestType,

18 + new_proc_per_test: bool) -> dict[str, VLMTestInfo]:

19 """Given the dict of potential test settings to run, return a subdict

20 of tests who have the current test type enabled with the matching val for

21 fork_per_test.

@@ -43,7 +43,7 @@ def get_filtered_test_settings(test_settings: dict[str, VLMTestInfo],

44 # Everything looks okay; keep if this is has correct proc handling

45 if (test_info.distributed_executor_backend

46 - is not None) == fork_per_test:

46 + is not None) == new_proc_per_test:

47 matching_tests[test_name] = test_info

49 return matching_tests

@@ -51,14 +51,14 @@ def get_filtered_test_settings(test_settings: dict[str, VLMTestInfo],

52 def get_parametrized_options(test_settings: dict[str, VLMTestInfo],

53 test_type: VLMTestType,

54 - fork_new_process_for_each_test: bool):

54 + create_new_process_for_each_test: bool):

55 """Converts all of our VLMTestInfo into an expanded list of parameters.

56 This is similar to nesting pytest parametrize calls, but done directly

57 through an itertools product so that each test can set things like

58 size factors etc, while still running in isolated test cases.

59 """

60 matching_tests = get_filtered_test_settings(

61 - test_settings, test_type, fork_new_process_for_each_test)

61 + test_settings, test_type, create_new_process_for_each_test)

63 # Ensure that something is wrapped as an iterable it's not already

64 ensure_wrapped = lambda e: e if isinstance(e, (list, tuple)) else (e, )

tests/models/encoder_decoder/audio_language/test_whisper.py

+2 -2

@@ -10,7 +10,7 @@ import pytest

10 from vllm import LLM, SamplingParams

11 from vllm.assets.audio import AudioAsset

13 -from ....utils import fork_new_process_for_each_test, multi_gpu_test

13 +from ....utils import create_new_process_for_each_test, multi_gpu_test

15 PROMPTS = [

16 {

@@ -119,7 +119,7 @@ def run_test(

119 assert output.outputs[0].text == expected

120

121

122 -@fork_new_process_for_each_test

122 +@create_new_process_for_each_test()

123 @pytest.mark.core_model

124 @pytest.mark.parametrize(

125 "model", ["openai/whisper-small", "openai/whisper-large-v3-turbo"])

tests/models/test_oot_registration.py

+5 -5

@@ -5,10 +5,10 @@ import pytest

5 from vllm import LLM, SamplingParams

6 from vllm.assets.image import ImageAsset

8 -from ..utils import fork_new_process_for_each_test

8 +from ..utils import create_new_process_for_each_test

11 -@fork_new_process_for_each_test

11 +@create_new_process_for_each_test()

12 def test_plugin(

13 monkeypatch: pytest.MonkeyPatch,

14 dummy_opt_path: str,

@@ -24,7 +24,7 @@ def test_plugin(

24 assert (error_msg in str(excinfo.value))

27 -@fork_new_process_for_each_test

27 +@create_new_process_for_each_test()

28 def test_oot_registration_text_generation(

29 monkeypatch: pytest.MonkeyPatch,

30 dummy_opt_path: str,

@@ -44,7 +44,7 @@ def test_oot_registration_text_generation(

44 assert rest == ""

47 -@fork_new_process_for_each_test

47 +@create_new_process_for_each_test()

48 def test_oot_registration_embedding(

49 monkeypatch: pytest.MonkeyPatch,

50 dummy_gemma2_embedding_path: str,

@@ -62,7 +62,7 @@ def test_oot_registration_embedding(

62 image = ImageAsset("cherry_blossom").pil_image.convert("RGB")

65 -@fork_new_process_for_each_test

65 +@create_new_process_for_each_test()

66 def test_oot_registration_multimodal(

67 monkeypatch: pytest.MonkeyPatch,

68 dummy_llava_path: str,

tests/models/test_registry.py

+3 -3

@@ -17,7 +17,7 @@ from vllm.model_executor.models.registry import (_MULTIMODAL_MODELS,

17 ModelRegistry)

18 from vllm.platforms import current_platform

20 -from ..utils import fork_new_process_for_each_test

20 +from ..utils import create_new_process_for_each_test

21 from .registry import HF_EXAMPLE_MODELS

@@ -45,7 +45,7 @@ def test_registry_imports(model_arch):

45 assert supports_multimodal(model_cls)

48 -@fork_new_process_for_each_test

48 +@create_new_process_for_each_test()

49 @pytest.mark.parametrize("model_arch,is_mm,init_cuda,is_ce", [

50 ("LlamaForCausalLM", False, False, False),

51 ("MllamaForConditionalGeneration", True, False, False),

@@ -70,7 +70,7 @@ def test_registry_model_property(model_arch, is_mm, init_cuda, is_ce):

70 stacklevel=2)

73 -@fork_new_process_for_each_test

73 +@create_new_process_for_each_test()

74 @pytest.mark.parametrize("model_arch,is_pp,init_cuda", [

75 ("MLPSpeculatorPreTrainedModel", False, False),

76 ("DeepseekV2ForCausalLM", True, False),

tests/quantization/test_bitsandbytes.py

+7 -6

@@ -10,7 +10,8 @@ import pytest

10 import torch

12 from tests.quantization.utils import is_quant_method_supported

13 -from tests.utils import compare_two_settings, fork_new_process_for_each_test

13 +

14 +from ..utils import compare_two_settings, create_new_process_for_each_test

16 models_4bit_to_test = [

17 ("facebook/opt-125m", "quantize opt model inflight"),

@@ -32,7 +33,7 @@ models_pre_quant_8bit_to_test = [

33 @pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),

34 reason='bitsandbytes is not supported on this GPU type.')

35 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)

35 -@fork_new_process_for_each_test

36 +@create_new_process_for_each_test()

37 def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,

38 model_name, description) -> None:

@@ -45,7 +46,7 @@ def test_load_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,

46 reason='bitsandbytes is not supported on this GPU type.')

47 @pytest.mark.parametrize("model_name, description",

48 models_pre_qaunt_4bit_to_test)

48 -@fork_new_process_for_each_test

49 +@create_new_process_for_each_test()

50 def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,

51 model_name, description) -> None:

@@ -57,7 +58,7 @@ def test_load_pre_quant_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,

58 reason='bitsandbytes is not supported on this GPU type.')

59 @pytest.mark.parametrize("model_name, description",

60 models_pre_quant_8bit_to_test)

60 -@fork_new_process_for_each_test

61 +@create_new_process_for_each_test()

62 def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,

63 model_name, description) -> None:

@@ -70,7 +71,7 @@ def test_load_8bit_bnb_model(hf_runner, vllm_runner, example_prompts,

71 @pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),

72 reason='bitsandbytes is not supported on this GPU type.')

73 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)

73 -@fork_new_process_for_each_test

74 +@create_new_process_for_each_test()

75 def test_load_tp_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,

76 model_name, description) -> None:

@@ -88,7 +89,7 @@ def test_load_tp_4bit_bnb_model(hf_runner, vllm_runner, example_prompts,

89 @pytest.mark.skipif(not is_quant_method_supported("bitsandbytes"),

90 reason='bitsandbytes is not supported on this GPU type.')

91 @pytest.mark.parametrize("model_name, description", models_4bit_to_test)

91 -@fork_new_process_for_each_test

92 +@create_new_process_for_each_test()

93 def test_load_pp_4bit_bnb_model(model_name, description) -> None:

94 common_args = [

95 "--disable-log-stats",

tests/spec_decode/e2e/test_multistep_correctness.py

+13 -13

@@ -42,7 +42,7 @@ from transformers import AutoTokenizer

43 from vllm import SamplingParams

45 -from ...utils import fork_new_process_for_each_test

45 +from ...utils import create_new_process_for_each_test

46 from .conftest import (get_output_from_llm_generator,

47 run_equality_correctness_test)

@@ -82,7 +82,7 @@ from .conftest import (get_output_from_llm_generator,

82 @pytest.mark.parametrize("test_llm_kwargs", [{}])

83 @pytest.mark.parametrize("batch_size", [1, 32])

84 @pytest.mark.parametrize("seed", [1])

85 -@fork_new_process_for_each_test

85 +@create_new_process_for_each_test()

86 def test_spec_decode_e2e_with_detokenization(test_llm_generator,

87 batch_size: int):

88 """Run generation with speculative decoding on a batch. Verify the engine

@@ -170,7 +170,7 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,

170 ])

171 @pytest.mark.parametrize("batch_size", [1])

172 @pytest.mark.parametrize("seed", [1])

173 -@fork_new_process_for_each_test

173 +@create_new_process_for_each_test()

174 def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(

175 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

176 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,

@@ -244,7 +244,7 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(

244 ])

245 @pytest.mark.parametrize("batch_size", [64])

246 @pytest.mark.parametrize("seed", [1])

247 -@fork_new_process_for_each_test

247 +@create_new_process_for_each_test()

248 def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(

249 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

250 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,

@@ -300,7 +300,7 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(

300 ])

301 @pytest.mark.parametrize("batch_size", [32])

302 @pytest.mark.parametrize("seed", [1])

303 -@fork_new_process_for_each_test

303 +@create_new_process_for_each_test()

304 def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(

305 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

306 baseline_llm_kwargs, test_llm_kwargs, batch_size: int,

@@ -356,7 +356,7 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs_diff_output_len(

356 256,

357 ])

358 @pytest.mark.parametrize("seed", [1])

359 -@fork_new_process_for_each_test

359 +@create_new_process_for_each_test()

360 def test_spec_decode_e2e_greedy_correctness_real_model_bs1(

361 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

362 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,

@@ -411,7 +411,7 @@ def test_spec_decode_e2e_greedy_correctness_real_model_bs1(

411 64,

412 ])

413 @pytest.mark.parametrize("seed", [1])

414 -@fork_new_process_for_each_test

414 +@create_new_process_for_each_test()

415 def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(

416 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

417 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,

@@ -469,7 +469,7 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(

469 ])

470 @pytest.mark.parametrize("batch_size", [4])

471 @pytest.mark.parametrize("seed", [1])

472 -@fork_new_process_for_each_test

472 +@create_new_process_for_each_test()

473 def test_spec_decode_e2e_greedy_correctness_with_preemption(

474 vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

475 baseline_llm_kwargs, test_llm_kwargs, batch_size: int, output_len: int,

@@ -534,7 +534,7 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(

534 32,

535 ])

536 @pytest.mark.parametrize("seed", [1])

537 -@fork_new_process_for_each_test

537 +@create_new_process_for_each_test()

538 def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,

539 per_test_common_llm_kwargs,

540 baseline_llm_kwargs, test_llm_kwargs,

@@ -594,7 +594,7 @@ def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,

594 64,

595 ])

596 @pytest.mark.parametrize("seed", [1])

597 -@fork_new_process_for_each_test

597 +@create_new_process_for_each_test()

598 def test_skip_speculation(vllm_runner, common_llm_kwargs,

599 per_test_common_llm_kwargs, baseline_llm_kwargs,

600 test_llm_kwargs, batch_size: int, output_len: int,

@@ -644,7 +644,7 @@ def test_skip_speculation(vllm_runner, common_llm_kwargs,

644 @pytest.mark.parametrize("batch_size", [8])

645 @pytest.mark.parametrize("output_len", [10])

646 @pytest.mark.parametrize("seed", [1])

647 -@fork_new_process_for_each_test

647 +@create_new_process_for_each_test()

648 def test_disable_speculation(vllm_runner, common_llm_kwargs,

649 per_test_common_llm_kwargs, baseline_llm_kwargs,

650 test_llm_kwargs, batch_size: int, output_len: int,

@@ -697,7 +697,7 @@ def test_disable_speculation(vllm_runner, common_llm_kwargs,

697 32,

698 ])

699 @pytest.mark.parametrize("seed", [1])

700 -@fork_new_process_for_each_test

700 +@create_new_process_for_each_test()

701 def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

702 baseline_llm_kwargs, test_llm_kwargs, batch_size: int,

703 output_len: int, seed: int):

@@ -752,7 +752,7 @@ def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,

752 32,

753 ])

754 @pytest.mark.parametrize("seed", [1])

755 -@fork_new_process_for_each_test

755 +@create_new_process_for_each_test()

756 def test_typical_acceptance_sampling(vllm_runner, common_llm_kwargs,

757 per_test_common_llm_kwargs,

758 baseline_llm_kwargs, test_llm_kwargs,

tests/test_utils.py

+2 -2

@@ -16,7 +16,7 @@ from vllm.utils import (FlexibleArgumentParser, MemorySnapshot,

16 deprecate_kwargs, get_open_port, memory_profiling,

17 merge_async_iterators, supports_kw, swap_dict_values)

19 -from .utils import error_on_warning, fork_new_process_for_each_test

19 +from .utils import create_new_process_for_each_test, error_on_warning

22 @pytest.mark.asyncio

@@ -276,7 +276,7 @@ def test_supports_kw(callable,kw_name,requires_kw_only,

276 ) == is_supported

277

278

279 -@fork_new_process_for_each_test

279 +@create_new_process_for_each_test()

280 def test_memory_profiling():

281 # Fake out some model loading + inference memory usage to test profiling

282 # Memory used by other processes will show up as cuda usage outside of torch

tests/utils.py

+77 -3

@@ -7,12 +7,14 @@ import os

7 import signal

8 import subprocess

9 import sys

10 +import tempfile

11 import time

12 import warnings

12 -from contextlib import contextmanager

13 +from contextlib import contextmanager, suppress

14 from pathlib import Path

14 -from typing import Any, Callable, Optional, Union

15 +from typing import Any, Callable, Literal, Optional, Union

17 +import cloudpickle

18 import openai

19 import pytest

20 import requests

@@ -703,6 +705,78 @@ def fork_new_process_for_each_test(

705 return wrapper

706

707

708 +def spawn_new_process_for_each_test(

709 + f: Callable[_P, None]) -> Callable[_P, None]:

710 + """Decorator to spawn a new process for each test function.

711 + """

712 +

713 + @functools.wraps(f)

714 + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:

715 + # Check if we're already in a subprocess

716 + if os.environ.get('RUNNING_IN_SUBPROCESS') == '1':

717 + # If we are, just run the function directly

718 + return f(*args, **kwargs)

719 +

720 + import torch.multiprocessing as mp

721 + with suppress(RuntimeError):

722 + mp.set_start_method('spawn')

723 +

724 + # Get the module

725 + module_name = f.__module__

726 +

727 + # Create a process with environment variable set

728 + env = os.environ.copy()

729 + env['RUNNING_IN_SUBPROCESS'] = '1'

730 +

731 + with tempfile.TemporaryDirectory() as tempdir:

732 + output_filepath = os.path.join(tempdir, "new_process.tmp")

733 +

734 + # `cloudpickle` allows pickling complex functions directly

735 + input_bytes = cloudpickle.dumps((f, output_filepath))

736 +

737 + cmd = [sys.executable, "-m", f"{module_name}"]

738 +

739 + returned = subprocess.run(cmd,

740 + input=input_bytes,

741 + capture_output=True,

742 + env=env)

743 +

744 + # check if the subprocess is successful

745 + try:

746 + returned.check_returncode()

747 + except Exception as e:

748 + # wrap raised exception to provide more information

749 + raise RuntimeError(f"Error raised in subprocess:\n"

750 + f"{returned.stderr.decode()}") from e

751 +

752 + return wrapper

753 +

754 +

755 +def create_new_process_for_each_test(

756 + method: Optional[Literal["spawn", "fork"]] = None

757 +) -> Callable[[Callable[_P, None]], Callable[_P, None]]:

758 + """Creates a decorator that runs each test function in a new process.

759 +

760 + Args:

761 + method: The process creation method. Can be either "spawn" or "fork".

762 + If not specified,

763 + it defaults to "spawn" on ROCm platforms and "fork" otherwise.

764 +

765 + Returns:

766 + A decorator to run test functions in separate processes.

767 + """

768 + if method is None:

769 + method = "spawn" if current_platform.is_rocm() else "fork"

770 +

771 + assert method in ["spawn",

772 + "fork"], "Method must be either 'spawn' or 'fork'"

773 +

774 + if method == "fork":

775 + return fork_new_process_for_each_test

776 +

777 + return spawn_new_process_for_each_test

778 +

779 +

780 def large_gpu_mark(min_gb: int) -> pytest.MarkDecorator:

781 """

782 Get a pytest mark, which skips the test if the GPU doesn't meet

@@ -762,7 +836,7 @@ def multi_gpu_test(*, num_gpus: int):

836 marks = multi_gpu_marks(num_gpus=num_gpus)

837

838 def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:

765 - func = fork_new_process_for_each_test(f)

839 + func = create_new_process_for_each_test()(f)

840 for mark in reversed(marks):

841 func = mark(func)

842

tests/v1/engine/test_engine_core.py

+5 -4

@@ -9,7 +9,6 @@ from concurrent.futures import Future

9 import pytest

10 from transformers import AutoTokenizer

12 -from tests.utils import fork_new_process_for_each_test

12 from vllm import SamplingParams

13 from vllm.engine.arg_utils import EngineArgs

14 from vllm.platforms import current_platform

@@ -19,6 +18,8 @@ from vllm.v1.executor.abstract import Executor, UniProcExecutor

18 from vllm.v1.kv_cache_interface import KVCacheConfig

19 from vllm.v1.outputs import ModelRunnerOutput

21 +from ...utils import create_new_process_for_each_test

22 +

23 if not current_platform.is_cuda():

24 pytest.skip(reason="V1 currently only supported on CUDA.",

25 allow_module_level=True)

@@ -44,7 +45,7 @@ def make_request() -> EngineCoreRequest:

45 )

47 -@fork_new_process_for_each_test

48 +@create_new_process_for_each_test()

49 def test_engine_core(monkeypatch: pytest.MonkeyPatch):

51 with monkeypatch.context() as m:

@@ -158,7 +159,7 @@ def test_engine_core(monkeypatch: pytest.MonkeyPatch):

159 assert len(engine_core.scheduler.running) == 0

160

161

161 -@fork_new_process_for_each_test

162 +@create_new_process_for_each_test()

163 def test_engine_core_advanced_sampling(monkeypatch: pytest.MonkeyPatch):

164 """

165 A basic end-to-end test to verify that the engine functions correctly

@@ -208,7 +209,7 @@ def test_engine_core_advanced_sampling(monkeypatch: pytest.MonkeyPatch):

209 _check_engine_state()

210

211

211 -@fork_new_process_for_each_test

212 +@create_new_process_for_each_test()

213 def test_engine_core_concurrent_batches(monkeypatch: pytest.MonkeyPatch):

214 """

215 Test that the engine can handle multiple concurrent batches.

tests/v1/engine/test_engine_core_client.py

+3 -2

@@ -8,7 +8,6 @@ from typing import Optional

8 import pytest

9 from transformers import AutoTokenizer

11 -from tests.utils import fork_new_process_for_each_test

11 from vllm import SamplingParams

12 from vllm.engine.arg_utils import EngineArgs

13 from vllm.platforms import current_platform

@@ -19,6 +18,8 @@ from vllm.v1.engine.core_client import (AsyncMPClient, EngineCoreClient,

18 SyncMPClient)

19 from vllm.v1.executor.abstract import Executor

21 +from ...utils import create_new_process_for_each_test

22 +

23 if not current_platform.is_cuda():

24 pytest.skip(reason="V1 currently only supported on CUDA.",

25 allow_module_level=True)

@@ -88,7 +89,7 @@ def echo(self, msg: str, err_msg: Optional[str] = None) -> str:

89 return msg

91 -@fork_new_process_for_each_test

92 +@create_new_process_for_each_test()

93 @pytest.mark.parametrize("multiprocessing_mode", [True, False])

94 def test_engine_core_client(monkeypatch: pytest.MonkeyPatch,

95 multiprocessing_mode: bool):