@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0384aa7150
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 00:30 UTC

[CI/Build] Add gpt-oss LoRA test (#27870)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Oct 31, 2025 at 22:17 UTC 0384aa7150c4c9778efca041ffd1beb3ad2bd694
7 files changed +120 -55
.buildkite/test-amd.yaml
+3 -1
@@ -441,7 +441,7 @@ steps:
441 --ignore=lora/test_llm_with_multi_loras.py \
442 --ignore=lora/test_olmoe_tp.py \
443 --ignore=lora/test_deepseekv2_tp.py \
444 - --ignore=lora/test_gptoss.py \
444 + --ignore=lora/test_gptoss_tp.py \
445 --ignore=lora/test_qwen3moe_tp.py
446 parallelism: 4
447
@@ -1217,6 +1217,8 @@ steps:
1217 - pytest -v -s -x lora/test_llama_tp.py
1218 - pytest -v -s -x lora/test_llm_with_multi_loras.py
1219 - pytest -v -s -x lora/test_olmoe_tp.py
1220 + - pytest -v -s -x lora/test_gptoss_tp.py
1221 +
1222
1223 - label: Weight Loading Multiple GPU Test # 33min
1224 timeout_in_minutes: 45
.buildkite/test-pipeline.yaml
+2 -1
@@ -417,7 +417,7 @@ steps:
417 --ignore=lora/test_llm_with_multi_loras.py \
418 --ignore=lora/test_olmoe_tp.py \
419 --ignore=lora/test_deepseekv2_tp.py \
420 - --ignore=lora/test_gptoss.py \
420 + --ignore=lora/test_gptoss_tp.py \
421 --ignore=lora/test_qwen3moe_tp.py
422
423 parallelism: 4
@@ -1119,6 +1119,7 @@ steps:
1119 - pytest -v -s -x lora/test_llama_tp.py
1120 - pytest -v -s -x lora/test_llm_with_multi_loras.py
1121 - pytest -v -s -x lora/test_olmoe_tp.py
1122 + - pytest -v -s -x lora/test_gptoss_tp.py
1123
1124
1125 - label: Weight Loading Multiple GPU Test # 33min
tests/lora/conftest.py
+1 -1
@@ -237,7 +237,7 @@ def deepseekv2_lora_files():
237
238 @pytest.fixture(scope="session")
239 def gptoss20b_lora_files():
240 - return snapshot_download(repo_id="LevinZheng/gpt-oss-20b-lora-adapter")
240 + return snapshot_download(repo_id="jeeejeee/gpt-oss-20b-lora-adapter-text2sql")
241
242
243 @pytest.fixture(scope="session")
tests/lora/test_deepseekv2_tp.py
+4
@@ -1,6 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +# NOTE To avoid overloading the CI pipeline, this test script will
5 +# not be triggered on CI and is primarily intended for local testing
6 +# and verification.
7 +
8 import vllm
9 from vllm.lora.request import LoRARequest
10
tests/lora/test_gptoss.py deleted
-52
@@ -1,52 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import vllm
5 -from vllm.lora.request import LoRARequest
6 -
7 -MODEL_PATH = "openai/gpt-oss-20b"
8 -
9 -PROMPT_TEMPLATE = "<｜begin▁of▁sentence｜>You are a helpful assistant.\n\nUser: {context}\n\nAssistant:" # noqa: E501
10 -
11 -
12 -def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
13 - prompts = [
14 - PROMPT_TEMPLATE.format(context="Who are you?"),
15 - ]
16 - sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
17 - outputs = llm.generate(
18 - prompts,
19 - sampling_params,
20 - lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
21 - )
22 - # Print the outputs.
23 - generated_texts: list[str] = []
24 - for output in outputs:
25 - prompt = output.prompt
26 - generated_text = output.outputs[0].text.strip()
27 - generated_texts.append(generated_text)
28 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
29 - return generated_texts
30 -
31 -
32 -# FIXME: Load gpt-oss adapter
33 -def test_gptoss20b_lora(gptoss20b_lora_files):
34 - # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
35 - # Otherwise, the lora-test will fail due to CUDA OOM.
36 - llm = vllm.LLM(
37 - MODEL_PATH,
38 - enable_lora=True,
39 - max_loras=4,
40 - trust_remote_code=True,
41 - )
42 -
43 - expected_lora_output = [
44 - "I am an AI language model developed by OpenAI. "
45 - "I am here to help you with any questions or "
46 - "tasks you may have."
47 - ]
48 -
49 - output1 = do_sample(llm, gptoss20b_lora_files, lora_id=1)
50 - print(output1)
51 - for i in range(len(expected_lora_output)):
52 - assert output1[i].startswith(expected_lora_output[i])
tests/lora/test_gptoss_tp.py new
+106
@@ -0,0 +1,106 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import vllm
5 +from vllm.lora.request import LoRARequest
6 +
7 +from ..utils import multi_gpu_test
8 +
9 +MODEL_PATH = "openai/gpt-oss-20b"
10 +
11 +PROMPT_TEMPLATE = """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
12 +Knowledge cutoff: 2024-06
13 +Current date: 2025-10-29
14 +
15 +Reasoning: medium
16 +
17 +# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>user<|message|>I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
18 +"
19 +##Instruction:
20 +farm contains tables such as city, farm, farm_competition, competition_record. Table city has columns such as City_ID, Official_Name, Status, Area_km_2, Population, Census_Ranking. City_ID is the primary key.
21 +Table farm has columns such as Farm_ID, Year, Total_Horses, Working_Horses, Total_Cattle, Oxen, Bulls, Cows, Pigs, Sheep_and_Goats. Farm_ID is the primary key.
22 +Table farm_competition has columns such as Competition_ID, Year, Theme, Host_city_ID, Hosts. Competition_ID is the primary key.
23 +Table competition_record has columns such as Competition_ID, Farm_ID, Rank. Competition_ID is the primary key.
24 +The Host_city_ID of farm_competition is the foreign key of City_ID of city.
25 +The Farm_ID of competition_record is the foreign key of Farm_ID of farm.
26 +The Competition_ID of competition_record is the foreign key of Competition_ID of farm_competition.
27 +
28 +
29 +###Input:
30 +{context}
31 +
32 +###Response:<|end|><|start|>assistant<|channel|>final<|message|>""" # noqa: E501
33 +
34 +EXPECTED_LORA_OUTPUT = [
35 + "SELECT AVG(Working_Horses) FROM farm WHERE Total_Horses > 5000;",
36 + "SELECT AVG(Working_Horses) FROM farm WHERE Total_Horses > 5000;",
37 + "SELECT MAX(Cows) AS Max_Cows, MIN(Cows) AS Min_Cows FROM farm;",
38 + "SELECT MAX(Cows) AS Max_Cows, MIN(Cows) AS Min_Cows FROM farm;",
39 +]
40 +
41 +
42 +def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
43 + prompts = [
44 + PROMPT_TEMPLATE.format(
45 + context="What is the average number of working horses of farms with more than 5000 total number of horses?" # noqa: E501
46 + ), # noqa: E501
47 + PROMPT_TEMPLATE.format(
48 + context="Give the average number of working horses on farms with more than 5000 total horses." # noqa: E501
49 + ), # noqa: E501
50 + PROMPT_TEMPLATE.format(
51 + context="What are the maximum and minimum number of cows across all farms."
52 + ),
53 + PROMPT_TEMPLATE.format(
54 + context="Return the maximum and minimum number of cows across all farms."
55 + ),
56 + ]
57 + sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
58 + outputs = llm.generate(
59 + prompts,
60 + sampling_params,
61 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
62 + )
63 + # Print the outputs.
64 + generated_texts: list[str] = []
65 + for output in outputs:
66 + prompt = output.prompt
67 + generated_text = output.outputs[0].text.strip()
68 + generated_texts.append(generated_text)
69 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
70 +
71 + for i in range(len(EXPECTED_LORA_OUTPUT)):
72 + assert generated_texts[i].startswith(EXPECTED_LORA_OUTPUT[i])
73 +
74 +
75 +def test_gpt_oss_lora(gptoss20b_lora_files):
76 + llm = vllm.LLM(
77 + MODEL_PATH,
78 + max_model_len=1024,
79 + enable_lora=True,
80 + max_loras=4,
81 + max_lora_rank=8,
82 + compilation_config=vllm.config.CompilationConfig( # Avoid OOM
83 + cudagraph_specialize_lora=False,
84 + ),
85 + )
86 +
87 + generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
88 + generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
89 +
90 +
91 +@multi_gpu_test(num_gpus=2)
92 +def test_gpt_oss_lora_tp2(gptoss20b_lora_files):
93 + llm = vllm.LLM(
94 + MODEL_PATH,
95 + max_model_len=1024,
96 + enable_lora=True,
97 + max_loras=2,
98 + max_lora_rank=8,
99 + tensor_parallel_size=2,
100 + compilation_config=vllm.config.CompilationConfig( # Avoid OOM
101 + cudagraph_specialize_lora=False,
102 + ),
103 + )
104 +
105 + generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
106 + generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
tests/lora/test_qwen3moe_tp.py
+4
@@ -1,6 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +
5 +# NOTE To avoid overloading the CI pipeline, this test script will not
6 +# be triggered on CI and is primarily intended for local testing and verification.
7 +
8 import vllm
9 from vllm.lora.request import LoRARequest
10