A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Enhance lora tests with more layer and rank variations (#3243)
Terry committed
Mar 9, 2024 at 17:14 UTC
0bba88df03754c40bd9135fc2ff9554ffca59c87
3 files changed
+106
csrc/punica/bgmv/bgmv_config.h
+1
@@ -14,6 +14,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
14
f(in_T, out_T, W_T, narrow, 128) \
15
f(in_T, out_T, W_T, narrow, 256) \
16
f(in_T, out_T, W_T, narrow, 512) \
17
+ f(in_T, out_T, W_T, narrow, 768) \
18
f(in_T, out_T, W_T, narrow, 1024) \
19
f(in_T, out_T, W_T, narrow, 1280) \
20
f(in_T, out_T, W_T, narrow, 1728) \
requirements-dev.txt
+1
@@ -21,6 +21,7 @@ einops # required for MPT
21
openai
22
requests
23
ray
24
+peft
25
26
# Benchmarking
27
aiohttp
tests/lora/test_layer_variation.py
new
+104
@@ -0,0 +1,104 @@
1
+from typing import List, Optional
2
+import peft
3
+import pytest
4
+from random import sample
5
+import tempfile
6
+from transformers import AutoModelForCausalLM
7
+
8
+import vllm
9
+from vllm.lora.request import LoRARequest
10
+from .conftest import cleanup
11
+
12
+MODEL_PATH = "Felladrin/Llama-68M-Chat-v1"
13
+PROMPTS = [
14
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
15
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
16
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
17
+]
18
+
19
+
20
+def get_lora_model(model_id: str, target_modules: List[str], rank: int):
21
+ model = AutoModelForCausalLM.from_pretrained(model_id)
22
+ lora_config = peft.tuners.lora.LoraConfig(target_modules, rank)
23
+ lora_model = peft.PeftModel(model, lora_config)
24
+ return lora_model
25
+
26
+
27
+def do_sample(llm,
28
+ lora_path: Optional[str] = None,
29
+ lora_id: Optional[int] = None,
30
+ logprobs: int = 0,
31
+ n_tokens: int = 256):
32
+ prompts = PROMPTS
33
+ sampling_params = vllm.SamplingParams(temperature=0,
34
+ max_tokens=n_tokens,
35
+ logprobs=logprobs,
36
+ stop=["[/assistant]"])
37
+ outputs = llm.generate(
38
+ prompts,
39
+ sampling_params,
40
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
41
+ if lora_id else None)
42
+ # Print the outputs.
43
+ generated_texts = []
44
+ generated_logprobs = []
45
+ for output in outputs:
46
+ prompt = output.prompt
47
+ generated_text = output.outputs[0].text
48
+ generated_texts.append(generated_text)
49
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
50
+ generated_logprobs.append([
51
+ list(logprob.keys()) for out in output.outputs
52
+ for logprob in out.logprobs
53
+ ])
54
+ return generated_logprobs if logprobs else generated_texts
55
+
56
+
57
+SUPPORTED_MODULES = [
58
+ "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
59
+ "lm_head"
60
+]
61
+TARGET_MODULES_LIST = []
62
+for length in range(2, 6):
63
+ TARGET_MODULES_LIST.extend(
64
+ [sample(SUPPORTED_MODULES, length) for _ in range(3)])
65
+
66
+
67
+# Test the correctness when layer and rank are varied
68
+# step 1: init a base model and serve with LoRA to get the reference results
69
+# step 2: merge the same LoRA to the base model, serve the merged model
70
+# step 3: compare the results from step 1 and step 2
71
+@pytest.mark.parametrize("tp_size", [1])
72
+@pytest.mark.parametrize("target_modules", TARGET_MODULES_LIST)
73
+@pytest.mark.parametrize("rank", [8, 16, 32, 64])
74
+def test_layer_variation_correctness(tp_size, target_modules, rank):
75
+ llm = vllm.LLM(MODEL_PATH,
76
+ enable_lora=True,
77
+ max_num_seqs=16,
78
+ max_loras=4,
79
+ tensor_parallel_size=tp_size,
80
+ worker_use_ray=True)
81
+ model = get_lora_model(MODEL_PATH, target_modules, rank)
82
+ with tempfile.TemporaryDirectory() as tmpdir:
83
+ model.save_pretrained(tmpdir)
84
+ merged_probs = do_sample(llm, tmpdir, 1, logprobs=5, n_tokens=32)
85
+ del llm
86
+ cleanup()
87
+ reference_id_sets = [set(prob[0]) for prob in merged_probs]
88
+
89
+ model = get_lora_model(MODEL_PATH, target_modules, rank)
90
+ with tempfile.TemporaryDirectory() as tmpdir:
91
+ merged_model = model.merge_and_unload()
92
+ merged_model.save_pretrained(tmpdir)
93
+ llm = vllm.LLM(tmpdir,
94
+ tokenizer=MODEL_PATH,
95
+ enable_lora=False,
96
+ max_num_seqs=16,
97
+ tensor_parallel_size=tp_size,
98
+ worker_use_ray=True)
99
+ probs = do_sample(llm, logprobs=5, n_tokens=32)
100
+ del llm
101
+ cleanup()
102
+ # verify the top-5 tokens are identical for each token
103
+ id_sets = [set(prob[0]) for prob in probs]
104
+ assert id_sets == reference_id_sets