A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[V1] LoRA - Enable more V1 tests (#14315)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Mar 5, 2025 at 22:55 UTC
3dbd2d813a45f0515349efd4c585f0fb90475c80
5 files changed
+57
-8
tests/lora/test_chatglm3_tp.py
-3
@@ -55,7 +55,6 @@ def v1(run_with_both_engines_lora):
55
pass
56
57
58
-@pytest.mark.skip_v1
58
@fork_new_process_for_each_test
59
def test_chatglm3_lora(chatglm3_lora_files):
60
llm = vllm.LLM(MODEL_PATH,
@@ -75,7 +74,6 @@ def test_chatglm3_lora(chatglm3_lora_files):
74
assert output2[i] == EXPECTED_LORA_OUTPUT[i]
75
76
78
-@pytest.mark.skip_v1
77
@multi_gpu_test(num_gpus=4)
78
@fork_new_process_for_each_test
79
def test_chatglm3_lora_tp4(chatglm3_lora_files):
@@ -97,7 +95,6 @@ def test_chatglm3_lora_tp4(chatglm3_lora_files):
95
assert output2[i] == EXPECTED_LORA_OUTPUT[i]
96
97
100
-@pytest.mark.skip_v1
98
@multi_gpu_test(num_gpus=4)
99
@fork_new_process_for_each_test
100
def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files):
tests/lora/test_mixtral.py
+8
@@ -10,6 +10,14 @@ from vllm.platforms import current_platform
10
MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
11
12
13
+@pytest.fixture(autouse=True)
14
+def v1(run_with_both_engines_lora):
15
+ # Simple autouse wrapper to run both engines for each test
16
+ # This can be promoted up to conftest.py to run for every
17
+ # test in a package
18
+ pass
19
+
20
+
21
def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int,
22
prompts: list[str]) -> list[str]:
23
tests/lora/test_qwen2vl.py
+8
@@ -12,6 +12,14 @@ from vllm.lora.request import LoRARequest
12
from vllm.platforms import current_platform
13
14
15
+@pytest.fixture(autouse=True)
16
+def v1(run_with_both_engines_lora):
17
+ # Simple autouse wrapper to run both engines for each test
18
+ # This can be promoted up to conftest.py to run for every
19
+ # test in a package
20
+ pass
21
+
22
+
23
@dataclass
24
class TestConfig:
25
model_path: str
tests/lora/test_ultravox.py
+9
@@ -4,6 +4,7 @@ import shutil
4
from os import path
5
from tempfile import TemporaryDirectory
6
7
+import pytest
8
import torch
9
from huggingface_hub import snapshot_download
10
from safetensors.torch import load_file, save_file
@@ -21,6 +22,14 @@ VLLM_PLACEHOLDER = "<|reserved_special_token_0|>"
22
PROMPT = "Tell me about a Fool's mate move in 20 words. Provide the moves!"
23
24
25
+@pytest.fixture(autouse=True)
26
+def v1(run_with_both_engines_lora):
27
+ # Simple autouse wrapper to run both engines for each test
28
+ # This can be promoted up to conftest.py to run for every
29
+ # test in a package
30
+ pass
31
+
32
+
33
def llama3_1_8b_chess_lora_path():
34
return snapshot_download(
35
repo_id="mkopecki/chess-lora-adapter-llama-3.1-8b")
tests/lora/test_worker.py
+32
-5
@@ -3,18 +3,45 @@
3
import os
4
import random
5
import tempfile
6
+from typing import Union
7
from unittest.mock import patch
8
9
+import pytest
10
+
11
+import vllm.envs as envs
12
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
13
ModelConfig, ParallelConfig, SchedulerConfig,
14
VllmConfig)
15
from vllm.lora.models import LoRAMapping
16
from vllm.lora.request import LoRARequest
17
+from vllm.v1.worker.gpu_worker import Worker as V1Worker
18
from vllm.worker.worker import Worker
19
20
21
+@pytest.fixture(autouse=True)
22
+def v1(run_with_both_engines_lora):
23
+ # Simple autouse wrapper to run both engines for each test
24
+ # This can be promoted up to conftest.py to run for every
25
+ # test in a package
26
+ pass
27
+
28
+
29
@patch.dict(os.environ, {"RANK": "0"})
30
def test_worker_apply_lora(sql_lora_files):
31
+
32
+ def set_active_loras(worker: Union[Worker, V1Worker],
33
+ lora_requests: list[LoRARequest]):
34
+ lora_mapping = LoRAMapping([], [])
35
+ if isinstance(worker, Worker):
36
+ # v0 case
37
+ worker.model_runner.set_active_loras(lora_requests, lora_mapping)
38
+ else:
39
+ # v1 case
40
+ worker.model_runner.lora_manager.set_active_adapters(
41
+ lora_requests, lora_mapping)
42
+
43
+ worker_cls = V1Worker if envs.VLLM_USE_V1 else Worker
44
+
45
vllm_config = VllmConfig(
46
model_config=ModelConfig(
47
"meta-llama/Llama-2-7b-hf",
@@ -40,16 +67,17 @@ def test_worker_apply_lora(sql_lora_files):
67
lora_config=LoRAConfig(max_lora_rank=8, max_cpu_loras=32,
68
max_loras=32),
69
)
43
- worker = Worker(
70
+ worker = worker_cls(
71
vllm_config=vllm_config,
72
local_rank=0,
73
rank=0,
74
distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
75
)
76
+
77
worker.init_device()
78
worker.load_model()
79
52
- worker.model_runner.set_active_loras([], LoRAMapping([], []))
80
+ set_active_loras(worker, [])
81
assert worker.list_loras() == set()
82
83
n_loras = 32
@@ -57,7 +85,7 @@ def test_worker_apply_lora(sql_lora_files):
85
LoRARequest(str(i + 1), i + 1, sql_lora_files) for i in range(n_loras)
86
]
87
60
- worker.model_runner.set_active_loras(lora_requests, LoRAMapping([], []))
88
+ set_active_loras(worker, lora_requests)
89
assert worker.list_loras() == {
90
lora_request.lora_int_id
91
for lora_request in lora_requests
@@ -69,8 +97,7 @@ def test_worker_apply_lora(sql_lora_files):
97
k=random.randint(1, n_loras))
98
random.shuffle(iter_lora_requests)
99
iter_lora_requests = iter_lora_requests[:-random.randint(0, n_loras)]
72
- worker.model_runner.set_active_loras(iter_lora_requests,
73
- LoRAMapping([], []))
100
+ set_active_loras(worker, lora_requests)
101
assert worker.list_loras().issuperset(
102
{lora_request.lora_int_id
103
for lora_request in iter_lora_requests})