| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | import torch |
| 6 | |
| 7 | import vllm |
| 8 | from vllm.lora.request import LoRARequest |
| 9 | from vllm.platforms import current_platform |
| 10 | |
| 11 | MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1" |
| 12 | |
| 13 | |
| 14 | def do_sample( |
| 15 | llm: vllm.LLM, lora_path: str, lora_id: int, prompts: list[str] |
| 16 | ) -> list[str]: |
| 17 | sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256) |
| 18 | outputs = llm.generate( |
| 19 | prompts, |
| 20 | sampling_params, |
| 21 | lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None, |
| 22 | ) |
| 23 | # Print the outputs. |
| 24 | generated_texts: list[str] = [] |
| 25 | for output in outputs: |
| 26 | prompt = output.prompt |
| 27 | generated_text = output.outputs[0].text.strip() |
| 28 | generated_texts.append(generated_text) |
| 29 | print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") |
| 30 | return generated_texts |
| 31 | |
| 32 | |
| 33 | @pytest.mark.parametrize("tp_size", [4]) |
| 34 | def test_mixtral_lora(mixtral_lora_files, tp_size): |
| 35 | """Original test, the LoRA model has the common target modules, not all.""" |
| 36 | if ( |
| 37 | torch.accelerator.device_count() < tp_size |
| 38 | and tp_size > 1 |
| 39 | and current_platform.is_cuda_alike() |
| 40 | ): |
| 41 | pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}") |
| 42 | |
| 43 | prompts = [ |
| 44 | "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]", # noqa: E501 |
| 45 | "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]", # noqa: E501 |
| 46 | "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]", # noqa: E501 |
| 47 | ] |
| 48 | |
| 49 | llm = vllm.LLM( |
| 50 | MODEL_PATH, |
| 51 | enable_lora=True, |
| 52 | max_num_seqs=16, |
| 53 | max_loras=4, |
| 54 | distributed_executor_backend="ray", |
| 55 | tensor_parallel_size=tp_size, |
| 56 | ) |
| 57 | |
| 58 | expected_lora_output = [ |
| 59 | [ |
| 60 | "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])" # noqa: E501 |
| 61 | ], |
| 62 | [ |
| 63 | "give_opinion(name[SpellForce 3], developer[Grimlore Games], release_year[2017], rating[poor])", # noqa: E501 |
| 64 | "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])", # noqa: E501 |
| 65 | ], |
| 66 | [ |
| 67 | "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])" # noqa: E501 |
| 68 | ], |
| 69 | ] |
| 70 | |
| 71 | def check_outputs(generated: list[str]): |
| 72 | assert len(generated) == len(expected_lora_output) |
| 73 | for gen, gt_choices in zip(generated, expected_lora_output): |
| 74 | assert gen in gt_choices |
| 75 | |
| 76 | check_outputs(do_sample(llm, mixtral_lora_files, lora_id=1, prompts=prompts)) |
| 77 | check_outputs(do_sample(llm, mixtral_lora_files, lora_id=2, prompts=prompts)) |