@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97992802f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[CI/Build]Reduce the time consumption for LoRA tests (#7396)

Jee Jee Li committed Aug 14, 2024 at 08:27 UTC 97992802f3e6f9f317313f8afc15e2a37fc2722b
3 files changed +3 -128
tests/lora/test_layer_variation.py deleted
-106
@@ -1,106 +0,0 @@
1 -import tempfile
2 -from random import sample
3 -from typing import List, Optional
4 -
5 -import peft
6 -import pytest
7 -from transformers import AutoModelForCausalLM
8 -
9 -import vllm
10 -from vllm.lora.request import LoRARequest
11 -
12 -from .conftest import cleanup
13 -
14 -MODEL_PATH = "Felladrin/Llama-68M-Chat-v1"
15 -PROMPTS = [
16 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501
17 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501
18 - "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]", # noqa: E501
19 -]
20 -
21 -
22 -def get_lora_model(model_id: str, target_modules: List[str], rank: int):
23 - model = AutoModelForCausalLM.from_pretrained(model_id)
24 - lora_config = peft.tuners.lora.LoraConfig(target_modules, rank)
25 - lora_model = peft.PeftModel(model, lora_config)
26 - return lora_model
27 -
28 -
29 -def do_sample(llm: vllm.LLM,
30 - lora_path: Optional[str] = None,
31 - lora_id: Optional[int] = None,
32 - logprobs: int = 0,
33 - n_tokens: int = 256):
34 - prompts = PROMPTS
35 - sampling_params = vllm.SamplingParams(temperature=0,
36 - max_tokens=n_tokens,
37 - logprobs=logprobs,
38 - stop=["[/assistant]"])
39 - outputs = llm.generate(
40 - prompts,
41 - sampling_params,
42 - lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
43 - if lora_id else None)
44 - # Print the outputs.
45 - generated_texts: List[str] = []
46 - generated_logprobs: List[List[List[int]]] = []
47 - for output in outputs:
48 - prompt = output.prompt
49 - generated_text = output.outputs[0].text
50 - generated_texts.append(generated_text)
51 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
52 - generated_logprobs.append([
53 - list(logprob.keys()) for out in output.outputs
54 - for logprob in out.logprobs
55 - ])
56 - return generated_logprobs if logprobs else generated_texts
57 -
58 -
59 -SUPPORTED_MODULES = [
60 - "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
61 - "lm_head"
62 -]
63 -TARGET_MODULES_LIST = []
64 -for length in range(2, 6):
65 - TARGET_MODULES_LIST.extend(
66 - [sample(SUPPORTED_MODULES, length) for _ in range(3)])
67 -
68 -
69 -# Test the correctness when layer and rank are varied
70 -# step 1: init a base model and serve with LoRA to get the reference results
71 -# step 2: merge the same LoRA to the base model, serve the merged model
72 -# step 3: compare the results from step 1 and step 2
73 -@pytest.mark.parametrize("tp_size", [1])
74 -@pytest.mark.parametrize("target_modules", TARGET_MODULES_LIST)
75 -@pytest.mark.parametrize("rank", [8, 16, 32, 64])
76 -def test_layer_variation_correctness(tp_size, target_modules, rank):
77 - llm = vllm.LLM(MODEL_PATH,
78 - enable_lora=True,
79 - max_num_seqs=16,
80 - max_loras=4,
81 - tensor_parallel_size=tp_size,
82 - worker_use_ray=True)
83 - model = get_lora_model(MODEL_PATH, target_modules, rank)
84 - with tempfile.TemporaryDirectory() as tmpdir:
85 - model.save_pretrained(tmpdir)
86 - merged_probs = do_sample(llm, tmpdir, 1, logprobs=5, n_tokens=32)
87 - del llm
88 - cleanup()
89 - reference_id_sets = [set(prob[0]) for prob in merged_probs]
90 -
91 - model = get_lora_model(MODEL_PATH, target_modules, rank)
92 - with tempfile.TemporaryDirectory() as tmpdir:
93 - merged_model = model.merge_and_unload()
94 - merged_model.save_pretrained(tmpdir)
95 - llm = vllm.LLM(tmpdir,
96 - tokenizer=MODEL_PATH,
97 - enable_lora=False,
98 - max_num_seqs=16,
99 - tensor_parallel_size=tp_size,
100 - worker_use_ray=True)
101 - probs = do_sample(llm, logprobs=5, n_tokens=32)
102 - del llm
103 - cleanup()
104 - # verify the top-5 tokens are identical for each token
105 - id_sets = [set(prob[0]) for prob in probs]
106 - assert id_sets == reference_id_sets
tests/lora/test_punica_sizes.py
+1 -1
@@ -98,7 +98,7 @@ HIDDEN_SIZES = [
98 128256,
99 ]
100 #The size of TP
101 -divisibility = [1, 2, 4, 8, 16, 32, 64]
101 +divisibility = [1, 2, 8, 16, 64]
102
103 all_hidden_size = []
104 for div in divisibility:
tests/lora/test_punica_variation.py
+2 -21
@@ -20,10 +20,10 @@ from vllm.triton_utils.libentry import LibEntry
20 from .utils import (generate_data, generate_data_for_expand_nslices,
21 ref_torch_groupgemm)
22
23 -HIDDEN_SIZES = [3424, 4096, 4097]
23 +HIDDEN_SIZES = [4097]
24
25 BATCHES = [1, 4, 16, 32]
26 -NUM_LORA = [1, 4, 8, 16, 32, 64, 128]
26 +NUM_LORA = [1, 8, 32, 128]
27 DTYPES = [torch.float16, torch.bfloat16]
28 MAX_RANKS = [1, 4, 8, 16, 32, 64, 128, 256]
29 SCALES = [0.5]
@@ -321,22 +321,3 @@ def test_punica_expand_nslices(
321
322 slice_offset += hidden_size
323 assert_close(our_outputs, ref_outputs)
324 -
325 -
326 -if __name__ == "__main__":
327 - from itertools import product
328 -
329 - lst = list(
330 - product(
331 - BATCHES,
332 - NUM_LORA,
333 - MAX_RANKS,
334 - [1.0],
335 - [torch.float16],
336 - ["expand"],
337 - SEED,
338 - CUDA_DEVICES,
339 - ))
340 - for ele in lst:
341 - test_punica_bgmv(*ele)
342 - print(f"{ele},pass")