A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[CI/Build] LoRA : make add_lora_test safer (#15181)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Mar 20, 2025 at 21:28 UTC
0cfe7d386d66623eca6920c65f12c3179d346c81
1 file changed
+8
-40
tests/lora/test_add_lora.py
+8
-40
@@ -1,10 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
import asyncio
3
import time
4
-from pathlib import Path
4
5
import pytest
7
-from huggingface_hub import snapshot_download
6
7
import vllm.envs as env
8
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -13,35 +11,9 @@ from vllm.lora.request import LoRARequest
11
from vllm.sampling_params import SamplingParams
12
from vllm.utils import merge_async_iterators
13
16
-MODEL_PATH = "meta-llama/Llama-2-7b-hf"
17
-LORA_MODULE_DOWNLOAD_PATH = None # Populated by download_and_prepare_lora_module() #noqa
18
-LORA_RANK = 8
19
-DEFAULT_MAX_LORAS = 16 * 3
20
-
21
-
22
-def download_and_prepare_lora_module():
23
- """
24
- Request submission is expensive when the LoRA adapters have their own
25
- tokenizers. This is because, for each request with a new LoRA adapter ID,
26
- the front-end loads the tokenizer from disk.
27
-
28
- In this test, as we are comparing request processing times, we want to
29
- minimize any extra activity. To this effect, we download the LoRA
30
- adapter and remove all the tokenizer files, so the engine will default
31
- to the base model tokenizer.
32
- """
33
- global LORA_MODULE_DOWNLOAD_PATH
34
-
35
- LORA_MODULE_HF_PATH = "yard1/llama-2-7b-sql-lora-test"
36
- LORA_MODULE_DOWNLOAD_PATH = snapshot_download(repo_id=LORA_MODULE_HF_PATH)
37
-
38
- tokenizer_files = [
39
- 'added_tokens.json', 'tokenizer_config.json', 'tokenizer.json',
40
- 'tokenizer.model'
41
- ]
42
- for tokenizer_file in tokenizer_files:
43
- del_path = Path(LORA_MODULE_DOWNLOAD_PATH) / tokenizer_file
44
- del_path.unlink(missing_ok=True)
14
+MODEL_PATH = "THUDM/chatglm3-6b"
15
+LORA_RANK = 64
16
+DEFAULT_MAX_LORAS = 4 * 3
17
18
19
@pytest.fixture(autouse=True)
@@ -52,11 +24,9 @@ def v1(run_with_both_engines_lora):
24
pass
25
26
55
-def get_lora_requests() -> list[LoRARequest]:
27
+def get_lora_requests(lora_path) -> list[LoRARequest]:
28
lora_requests: list[LoRARequest] = [
57
- LoRARequest(lora_name=f"{i}",
58
- lora_int_id=i,
59
- lora_path=LORA_MODULE_DOWNLOAD_PATH)
29
+ LoRARequest(lora_name=f"{i}", lora_int_id=i, lora_path=lora_path)
30
for i in range(1, DEFAULT_MAX_LORAS + 1)
31
]
32
return lora_requests
@@ -93,7 +63,7 @@ async def requests_processing_time(llm,
63
64
65
@pytest.mark.asyncio
96
-async def test_add_lora():
66
+async def test_add_lora(chatglm3_lora_files):
67
"""
68
The add_lora function is used to pre-load some LoRA adapters into the
69
engine in anticipation of future requests using these adapters. To test
@@ -103,10 +73,7 @@ async def test_add_lora():
73
We measure the request processing time in both cases and expect the time
74
to be lesser in the case with add_lora() calls.
75
"""
106
-
107
- download_and_prepare_lora_module()
108
-
109
- lora_requests: list[LoRARequest] = get_lora_requests()
76
+ lora_requests: list[LoRARequest] = get_lora_requests(chatglm3_lora_files)
77
78
max_loras = len(set([lr.lora_int_id for lr in lora_requests]))
79
# Create engine in eager-mode. Due to high max_loras, the CI can
@@ -118,6 +85,7 @@ async def test_add_lora():
85
max_lora_rank=LORA_RANK,
86
max_model_len=128,
87
gpu_memory_utilization=0.8, #avoid OOM
88
+ trust_remote_code=True,
89
enforce_eager=True)
90
91
# The run_with_both_engines_lora fixture sets up the `VLLM_USE_V1`