@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 79455cf421

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 16:00 UTC

[Misc] Enable V1 LoRA by default (#15320)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>

Varun Sundar Rabindranath committed Apr 1, 2025 at 04:53 UTC 79455cf42113efb026e39342997f7a878af8bc38

12 files changed +125 -87

tests/entrypoints/openai/test_chat.py

+57 -3

@@ -24,7 +24,23 @@ GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]

26 @pytest.fixture(scope="module")

27 -def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811

27 +def monkeypatch_module():

28 + from _pytest.monkeypatch import MonkeyPatch

29 + mpatch = MonkeyPatch()

30 + yield mpatch

31 + mpatch.undo()

32 +

33 +

34 +@pytest.fixture(scope="module", params=[False, True])

35 +def server(

36 + request,

37 + monkeypatch_module,

38 + zephyr_lora_files, #noqa: F811

39 + zephyr_lora_added_tokens_files): # noqa: F811

40 +

41 + use_v1 = request.param

42 + monkeypatch_module.setenv('VLLM_USE_V1', '1' if use_v1 else '0')

43 +

44 args = [

45 # use half precision for speed and memory savings in CI environment

46 "--dtype",

@@ -49,6 +65,13 @@ def server(zephyr_lora_files, zephyr_lora_added_tokens_files): # noqa: F811

65 yield remote_server

68 +@pytest.fixture

69 +def is_v1_server(server):

70 + import os

71 + assert os.environ['VLLM_USE_V1'] in ['0', '1']

72 + return os.environ['VLLM_USE_V1'] == '1'

73 +

74 +

75 @pytest_asyncio.fixture

76 async def client(server):

77 async with server.get_async_client() as async_client:

@@ -471,8 +494,13 @@ async def test_chat_completion_stream_options(client: openai.AsyncOpenAI,

494 @pytest.mark.asyncio

495 @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)

496 async def test_guided_choice_chat(client: openai.AsyncOpenAI,

497 + is_v1_server: bool,

498 guided_decoding_backend: str,

499 sample_guided_choice):

500 +

501 + if is_v1_server and guided_decoding_backend != 'xgrammar':

502 + pytest.skip("Only xgrammar backend is supported with V1")

503 +

504 messages = [{

505 "role": "system",

506 "content": "you are a helpful assistant"

@@ -511,9 +539,13 @@ async def test_guided_choice_chat(client: openai.AsyncOpenAI,

539

540 @pytest.mark.asyncio

541 @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)

514 -async def test_guided_json_chat(client: openai.AsyncOpenAI,

542 +async def test_guided_json_chat(client: openai.AsyncOpenAI, is_v1_server: bool,

543 guided_decoding_backend: str,

544 sample_json_schema):

545 +

546 + if is_v1_server:

547 + pytest.skip("sample_json_schema has features unsupported in V1")

548 +

549 messages = [{

550 "role": "system",

551 "content": "you are a helpful assistant"

@@ -559,7 +591,12 @@ async def test_guided_json_chat(client: openai.AsyncOpenAI,

591 @pytest.mark.asyncio

592 @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)

593 async def test_guided_regex_chat(client: openai.AsyncOpenAI,

594 + is_v1_server: bool,

595 guided_decoding_backend: str, sample_regex):

596 +

597 + if is_v1_server and guided_decoding_backend != 'xgrammar':

598 + pytest.skip("Only xgrammar backend is supported with V1")

599 +

600 messages = [{

601 "role": "system",

602 "content": "you are a helpful assistant"

@@ -617,8 +654,13 @@ async def test_guided_decoding_type_error(client: openai.AsyncOpenAI):

654 @pytest.mark.asyncio

655 @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)

656 async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,

657 + is_v1_server: bool,

658 guided_decoding_backend: str,

659 sample_guided_choice):

660 +

661 + if is_v1_server and guided_decoding_backend != 'xgrammar':

662 + pytest.skip("Only xgrammar backend is supported with V1")

663 +

664 messages = [{

665 "role": "system",

666 "content": "you are a helpful assistant"

@@ -648,9 +690,13 @@ async def test_guided_choice_chat_logprobs(client: openai.AsyncOpenAI,

690

691 @pytest.mark.asyncio

692 @pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)

651 -async def test_named_tool_use(client: openai.AsyncOpenAI,

693 +async def test_named_tool_use(client: openai.AsyncOpenAI, is_v1_server: bool,

694 guided_decoding_backend: str,

695 sample_json_schema):

696 +

697 + if is_v1_server:

698 + pytest.skip("sample_json_schema has features unsupported on V1")

699 +

700 messages = [{

701 "role": "system",

702 "content": "you are a helpful assistant"

@@ -742,6 +788,10 @@ async def test_named_tool_use(client: openai.AsyncOpenAI,

788 @pytest.mark.asyncio

789 async def test_required_tool_use_not_yet_supported(client: openai.AsyncOpenAI,

790 sample_json_schema):

791 +

792 + if is_v1_server:

793 + pytest.skip("sample_json_schema has features unsupported on V1")

794 +

795 messages = [{

796 "role": "system",

797 "content": "you are a helpful assistant"

@@ -787,6 +837,10 @@ async def test_required_tool_use_not_yet_supported(client: openai.AsyncOpenAI,

837 @pytest.mark.asyncio

838 async def test_inconsistent_tool_choice_and_tools(client: openai.AsyncOpenAI,

839 sample_json_schema):

840 +

841 + if is_v1_server:

842 + pytest.skip("sample_json_schema has features unsupported on V1")

843 +

844 messages = [{

845 "role": "system",

846 "content": "you are a helpful assistant"

tests/lora/test_baichuan.py

+8 -8

@@ -11,6 +11,14 @@ MODEL_PATH = "baichuan-inc/Baichuan-7B"

11 PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501

14 +@pytest.fixture(autouse=True)

15 +def v1(run_with_both_engines_lora):

16 + # Simple autouse wrapper to run both engines for each test

17 + # This can be promoted up to conftest.py to run for every

18 + # test in a package

19 + pass

20 +

21 +

22 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

23 prompts = [

24 PROMPT_TEMPLATE.format(query="How many singers do we have?"),

@@ -40,14 +48,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

48 return generated_texts

43 -@pytest.fixture(autouse=True)

44 -def v1(run_with_both_engines_lora):

45 - # Simple autouse wrapper to run both engines for each test

46 - # This can be promoted up to conftest.py to run for every

47 - # test in a package

48 - pass

49 -

50 -

51 def test_baichuan_lora(baichuan_lora_files):

52 llm = vllm.LLM(MODEL_PATH,

53 max_model_len=1024,

tests/lora/test_chatglm3_tp.py

+8 -8

@@ -18,6 +18,14 @@ EXPECTED_LORA_OUTPUT = [

18 ]

21 +@pytest.fixture(autouse=True)

22 +def v1(run_with_both_engines_lora):

23 + # Simple autouse wrapper to run both engines for each test

24 + # This can be promoted up to conftest.py to run for every

25 + # test in a package

26 + pass

27 +

28 +

29 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

30 prompts = [

31 PROMPT_TEMPLATE.format(query="How many singers do we have?"),

@@ -46,14 +54,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

54 return generated_texts

49 -@pytest.fixture(autouse=True)

50 -def v1(run_with_both_engines_lora):

51 - # Simple autouse wrapper to run both engines for each test

52 - # This can be promoted up to conftest.py to run for every

53 - # test in a package

54 - pass

55 -

56 -

57 @create_new_process_for_each_test()

58 def test_chatglm3_lora(chatglm3_lora_files):

59 llm = vllm.LLM(MODEL_PATH,

tests/lora/test_gemma.py

+8 -8

@@ -9,6 +9,14 @@ from vllm.platforms import current_platform

9 MODEL_PATH = "google/gemma-7b"

12 +@pytest.fixture(autouse=True)

13 +def v1(run_with_both_engines_lora):

14 + # Simple autouse wrapper to run both engines for each test

15 + # This can be promoted up to conftest.py to run for every

16 + # test in a package

17 + pass

18 +

19 +

20 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

21 prompts = [

22 "Quote: Imagination is",

@@ -31,14 +39,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

39 return generated_texts

34 -@pytest.fixture(autouse=True)

35 -def v1(run_with_both_engines_lora):

36 - # Simple autouse wrapper to run both engines for each test

37 - # This can be promoted up to conftest.py to run for every

38 - # test in a package

39 - pass

40 -

41 -

42 # The V1 lora test for this model requires more than 24GB.

43 @pytest.mark.skip_v1

44 @pytest.mark.xfail(current_platform.is_rocm(),

tests/lora/test_layers.py

-5

@@ -1,6 +1,5 @@

1 # SPDX-License-Identifier: Apache-2.0

3 -import importlib

3 import random

4 from copy import deepcopy

5 from dataclasses import dataclass

@@ -82,10 +81,6 @@ def v1(run_with_both_engines_lora):

81 # This can be promoted up to conftest.py to run for every

82 # test in a package

85 - # Reload punica_gpu as the kernels used are tied to engine type.

86 - from vllm.lora.punica_wrapper import punica_gpu

87 - importlib.reload(punica_gpu)

88 -

84 # Release any memory we might be holding on to. CI runs OOMs otherwise.

85 from vllm.lora.ops.triton_ops.utils import (_LORA_A_PTR_DICT,

86 _LORA_B_PTR_DICT)

tests/lora/test_llama_tp.py

+8 -12

@@ -28,6 +28,14 @@ EXPECTED_LORA_OUTPUT = [

28 ]

31 +@pytest.fixture(autouse=True)

32 +def v1(run_with_both_engines_lora):

33 + # Simple autouse wrapper to run both engines for each test

34 + # This can be promoted up to conftest.py to run for every

35 + # test in a package

36 + pass

37 +

38 +

39 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

40 prompts = [

41 "[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501

@@ -71,16 +79,6 @@ def generate_and_test(llm, sql_lora_files):

79 print("removing lora")

74 -@pytest.fixture(autouse=True)

75 -def v1(run_with_both_engines_lora):

76 - # Simple autouse wrapper to run both engines for each test

77 - # This can be promoted up to conftest.py to run for every

78 - # test in a package

79 - pass

80 -

81 -

82 -# V1 Test: Failing due to numerics on V1.

83 -@pytest.mark.skip_v1

82 @create_new_process_for_each_test()

83 def test_llama_lora(sql_lora_files):

@@ -126,8 +124,6 @@ def test_llama_lora_warmup(sql_lora_files):

124 "less when using lora than when not using lora")

125

126

129 -# V1 Test: Failing due to numerics on V1.

130 -@pytest.mark.skip_v1

127 @multi_gpu_test(num_gpus=4)

128 @create_new_process_for_each_test()

129 def test_llama_lora_tp4(sql_lora_files):

tests/lora/test_lora_manager.py

+11 -3

@@ -7,7 +7,6 @@ import torch

7 from safetensors.torch import load_file

8 from torch import nn

10 -from vllm import envs

10 from vllm.config import LoRAConfig

11 from vllm.lora.layers import (ColumnParallelLinearWithLoRA,

12 MergedColumnParallelLinearWithLoRA,

@@ -33,6 +32,17 @@ DEVICES = ([

32 ] if current_platform.is_cuda_alike() else ["cpu"])

35 +@pytest.fixture(scope="function", autouse=True)

36 +def use_v0_only(monkeypatch: pytest.MonkeyPatch):

37 + """

38 + Some tests depend on V0 internals. Since both V0 and V1 use the same

39 + LoRAModelManager it is okay to just test V0.

40 + """

41 + with monkeypatch.context() as m:

42 + m.setenv('VLLM_USE_V1', '0')

43 + yield

44 +

45 +

46 @pytest.mark.parametrize("device", DEVICES)

47 def test_from_lora_tensors(sql_lora_files, device):

48 tensors = load_file(

@@ -411,7 +421,6 @@ def test_lru_lora_model_manager(dist_init, dummy_model, device):

421 assert manager.device == device

422

423

414 -@pytest.mark.skipif(envs.VLLM_USE_V1, reason="Test leverages V0 internals.")

424 @pytest.mark.parametrize("device", DEVICES)

425 def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,

426 sql_lora_files, device):

@@ -491,7 +500,6 @@ def test_lru_cache_worker_adapter_manager(llama_2_7b_model_extra_embeddings,

500 device)

501

502

494 -@pytest.mark.skipif(envs.VLLM_USE_V1, reason="Test leverages V0 internals.")

503 @pytest.mark.parametrize("device", DEVICES)

504 def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,

505 sql_lora_files, device):

tests/lora/test_phi.py

+8 -8

@@ -10,6 +10,14 @@ MODEL_PATH = "microsoft/phi-2"

10 PROMPT_TEMPLATE = "### Instruct: {sql_prompt}\n\n### Context: {context}\n\n### Output:" # noqa: E501

13 +@pytest.fixture(autouse=True)

14 +def v1(run_with_both_engines_lora):

15 + # Simple autouse wrapper to run both engines for each test

16 + # This can be promoted up to conftest.py to run for every

17 + # test in a package

18 + pass

19 +

20 +

21 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

22 prompts = [

23 PROMPT_TEMPLATE.format(

@@ -48,14 +56,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

56 return generated_texts

51 -@pytest.fixture(autouse=True)

52 -def v1(run_with_both_engines_lora):

53 - # Simple autouse wrapper to run both engines for each test

54 - # This can be promoted up to conftest.py to run for every

55 - # test in a package

56 - pass

57 -

58 -

59 # Skipping for V1 for now as we are hitting,

60 # "Head size 80 is not supported by FlashAttention." error.

61 @pytest.mark.skip_v1

tests/lora/test_quant_model.py

+8 -8

@@ -37,6 +37,14 @@ else:

37 ]

40 +@pytest.fixture(autouse=True)

41 +def v1(run_with_both_engines_lora):

42 + # Simple autouse wrapper to run both engines for each test

43 + # This can be promoted up to conftest.py to run for every

44 + # test in a package

45 + pass

46 +

47 +

48 def do_sample(llm: vllm.LLM,

49 lora_path: str,

50 lora_id: int,

@@ -69,14 +77,6 @@ def do_sample(llm: vllm.LLM,

77 return generated_texts

72 -@pytest.fixture(autouse=True)

73 -def v1(run_with_both_engines_lora):

74 - # Simple autouse wrapper to run both engines for each test

75 - # This can be promoted up to conftest.py to run for every

76 - # test in a package

77 - pass

78 -

79 -

80 @pytest.mark.parametrize("model", MODELS)

81 @pytest.mark.parametrize("tp_size", [1])

82 def test_quant_model_lora(tinyllama_lora_files, num_gpus_available, model,

tests/lora/test_transfomers_model.py

+8 -11

@@ -18,6 +18,14 @@ EXPECTED_LORA_OUTPUT = [

18 ]

21 +@pytest.fixture(autouse=True)

22 +def v1(run_with_both_engines_lora):

23 + # Simple autouse wrapper to run both engines for each test

24 + # This can be promoted up to conftest.py to run for every

25 + # test in a package

26 + pass

27 +

28 +

29 def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

30 prompts = [

31 PROMPT_TEMPLATE.format(query="How many singers do we have?"),

@@ -46,15 +54,6 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:

54 return generated_texts

49 -@pytest.fixture(autouse=True)

50 -def v1(run_with_both_engines_lora):

51 - # Simple autouse wrapper to run both engines for each test

52 - # This can be promoted up to conftest.py to run for every

53 - # test in a package

54 - pass

55 -

56 -

57 -@pytest.mark.skip_v1

57 @create_new_process_for_each_test()

58 def test_ilama_lora(ilama_lora_files):

59 llm = vllm.LLM(MODEL_PATH,

@@ -74,7 +73,6 @@ def test_ilama_lora(ilama_lora_files):

73 assert output2[i] == EXPECTED_LORA_OUTPUT[i]

77 -@pytest.mark.skip_v1

76 @multi_gpu_test(num_gpus=4)

77 @create_new_process_for_each_test()

78 def test_ilama_lora_tp4(ilama_lora_files):

@@ -96,7 +94,6 @@ def test_ilama_lora_tp4(ilama_lora_files):

94 assert output2[i] == EXPECTED_LORA_OUTPUT[i]

99 -@pytest.mark.skip_v1

97 @multi_gpu_test(num_gpus=4)

98 @create_new_process_for_each_test()

99 def test_ilama_lora_tp4_fully_sharded_loras(ilama_lora_files):

tests/v1/test_oracle.py

+1 -9

@@ -104,14 +104,6 @@ def test_enable_by_default_fallback(monkeypatch):

104 assert envs.VLLM_USE_V1

105 m.delenv("VLLM_USE_V1")

106

107 - # Should fall back to V0 for experimental config.

108 - _ = AsyncEngineArgs(

109 - model=MODEL,

110 - enable_lora=True,

111 - ).create_engine_config()

112 - assert not envs.VLLM_USE_V1

113 - m.delenv("VLLM_USE_V1")

114 -

107 # Should fall back to V0 for supported model.

108 _ = AsyncEngineArgs(

109 model=UNSUPPORTED_MODELS_V1[0]).create_engine_config()

@@ -125,7 +117,7 @@ def test_v1_llm_by_default(monkeypatch):

117 m.delenv("VLLM_USE_V1")

118

119 # Should default to V1 for supported config.

128 - model = LLM(MODEL, enforce_eager=True)

120 + model = LLM(MODEL, enforce_eager=True, enable_lora=True)

121 print(model.generate("Hello my name is"))

122 assert hasattr(model.llm_engine, "engine_core")

123 m.delenv("VLLM_USE_V1")

vllm/engine/arg_utils.py

-4

@@ -1512,10 +1512,6 @@ class EngineArgs:

1512 and _warn_or_fallback("Engine in background thread")):

1513 return False

1514

1515 - # LoRA is supported on V1, but off by default for now.

1516 - if self.enable_lora and _warn_or_fallback("LORA"):

1517 - return False

1518 -

1515 # PP is supported on V1 with Ray distributed executor,

1516 # but off for MP distributed executor for now.

1517 if (self.pipeline_parallel_size > 1