A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[CI/Build] LoRA: Delete long context tests (#15503)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Mar 25, 2025 at 17:18 UTC
ff38f0a32c5f7a2dd1aa3bee3806d957adacb9bd
2 files changed
+1
-304
.buildkite/test-pipeline.yaml
+1
-3
@@ -287,7 +287,7 @@ steps:
287
source_file_dependencies:
288
- vllm/lora
289
- tests/lora
290
- command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_minicpmv_tp.py --ignore=lora/test_transfomers_model.py
290
+ command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_minicpmv_tp.py --ignore=lora/test_transfomers_model.py
291
parallelism: 4
292
293
- label: PyTorch Fullgraph Smoke Test # 9min
@@ -592,8 +592,6 @@ steps:
592
# FIXIT: find out which code initialize cuda before running the test
593
# before the fix, we need to use spawn to test it
594
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
595
- # This test runs llama 13B, so it is required to run on 4 GPUs.
596
- - pytest -v -s -x lora/test_long_context.py
595
# There is some Tensor Parallelism related processing logic in LoRA that
596
# requires multi-GPU testing for validation.
597
- pytest -v -s -x lora/test_chatglm3_tp.py
tests/lora/test_long_context.py
deleted
-301
@@ -1,301 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-import ast
4
-from typing import Optional
5
-
6
-import numpy as np
7
-import pytest
8
-
9
-import vllm
10
-from vllm import SamplingParams
11
-from vllm.lora.layers import LinearScalingRotaryEmbeddingWithLoRA
12
-from vllm.lora.request import LoRARequest
13
-from vllm.model_executor.layers.rotary_embedding import (
14
- LinearScalingRotaryEmbedding)
15
-
16
-from .data.long_context_test_data import prompts_and_responses
17
-
18
-context_len_to_scaling_factor = {
19
- "16k": 4,
20
- "32k": 8,
21
-}
22
-
23
-# We use the same sampling params for all requests
24
-sampling_params = SamplingParams(
25
- temperature=0,
26
- max_tokens=100,
27
-)
28
-
29
-
30
-def _create_lora_request(lora_id, long_context_infos):
31
- context_len = long_context_infos[lora_id]["context_length"]
32
- scaling_factor = context_len_to_scaling_factor[context_len]
33
- return LoRARequest(
34
- # There are 2 LoRAs for 16K, we need to add lora_id to indicate
35
- # they are different LoRAs.
36
- context_len + str(lora_id),
37
- lora_id,
38
- long_context_infos[lora_id]["lora"],
39
- None,
40
- 4096 * scaling_factor,
41
- )
42
-
43
-
44
-def evaluate_json_response(model_response, golden_response):
45
- """Evaluates the model response against the golden response.
46
-
47
- Returns a score between 0 and 1, where 1 is a perfect match and 0 is no
48
- match. The score quantifies how well the model is able to extract the
49
- golden JSON from the long context.
50
- """
51
- try:
52
- model_response = ast.literal_eval(model_response)
53
- except Exception as e:
54
- raise ValueError(
55
- f"Model response is not a valid JSON. Expected {golden_response}, "
56
- f"got {model_response}") from e
57
-
58
- # Normally, we would flatten the dictionary and compare the values, but in
59
- # this case, we know that the dictionary is only 2 levels deep
60
- positive_values = 0
61
- total_values = 0
62
- # We look at all the attributes of the person that we are extracting a
63
- # biography of and copmare them to the golden response
64
- for person_attribute, person_attribute_value in golden_response.items():
65
- if person_attribute in model_response:
66
- if isinstance(person_attribute_value, dict):
67
- for (sub_attribute,
68
- sub_attribute_value) in person_attribute_value.items():
69
- total_values += 1
70
- if sub_attribute in model_response[
71
- person_attribute] and model_response[
72
- person_attribute][
73
- sub_attribute] == sub_attribute_value:
74
- positive_values += 1
75
- else:
76
- total_values += 1
77
- if model_response[person_attribute] == person_attribute_value:
78
- positive_values += 1
79
- else:
80
- # We count a missing sub-dict as a single missed value.
81
- total_values += 1
82
-
83
- # Return a score between 0 and 1
84
- return positive_values / total_values
85
-
86
-
87
-def generate(
88
- llm: vllm.LLM,
89
- inputs: tuple[str, SamplingParams, Optional[LoRARequest]],
90
-):
91
- prompts, sampling_param, lora_request = inputs
92
- outputs = llm.generate(prompts, sampling_param, lora_request=lora_request)
93
- return outputs[0].outputs[0].text.strip()
94
-
95
-
96
-def batched_generate(
97
- llm: vllm.LLM,
98
- inputs: list[tuple[str, SamplingParams, Optional[LoRARequest]]],
99
-):
100
- for input in inputs:
101
- prompt, sampling_param, lora_req = input
102
- # Add requests to the engine and run the engine
103
- llm._validate_and_add_requests(prompt,
104
- sampling_param,
105
- lora_request=lora_req,
106
- prompt_adapter_request=None)
107
-
108
- outputs = llm._run_engine(use_tqdm=True)
109
- return [outputs[i].outputs[0].text.strip() for i in range(len(outputs))]
110
-
111
-
112
-@pytest.fixture(scope="module")
113
-def lora_llm(long_context_infos):
114
- scaling_factors = [
115
- context_len_to_scaling_factor[info["context_length"]]
116
- for info in long_context_infos.values()
117
- ]
118
-
119
- llm = vllm.LLM(
120
- "meta-llama/Llama-2-13b-chat-hf",
121
- enable_lora=True,
122
- max_num_seqs=16,
123
- max_loras=2,
124
- long_lora_scaling_factors=tuple(scaling_factors),
125
- max_num_batched_tokens=4096 * 8,
126
- tensor_parallel_size=4,
127
- # FIXME enable async output processor
128
- disable_async_output_proc=True,
129
- distributed_executor_backend="mp",
130
- enable_chunked_prefill=True)
131
- yield llm
132
- del llm
133
-
134
-
135
-def test_rotary_emb_replaced(dist_init):
136
- """Verify rotary emb in all the layers are replaced"""
137
- from vllm.engine.arg_utils import EngineArgs
138
- from vllm.worker.model_runner import ModelRunner
139
- engine_args = EngineArgs("meta-llama/Llama-2-7b-hf",
140
- long_lora_scaling_factors=(4.0, ),
141
- enable_lora=True)
142
- engine_config = engine_args.create_engine_config()
143
- model_runner = ModelRunner(
144
- vllm_config=engine_config,
145
- is_driver_worker=True,
146
- )
147
- model_runner.load_model()
148
- rotary_emb_count = 0
149
- for module_name, module in model_runner.model.named_modules(
150
- remove_duplicate=False):
151
- if "rotary_emb" in module_name:
152
- if "base_layer" not in module_name:
153
- rotary_emb_count += 1
154
- assert isinstance(module, LinearScalingRotaryEmbeddingWithLoRA)
155
- else:
156
- assert isinstance(module, LinearScalingRotaryEmbedding)
157
- # Llama 2 has 32 layers.
158
- assert rotary_emb_count == 32
159
-
160
-
161
-@pytest.mark.skip_global_cleanup
162
-def test_batched_rope_kernel(lora_llm, long_context_infos):
163
- """We test the batched kernel by comparing the results of batched an
164
- non-batched generation.
165
- """
166
- # Create non batched results first to compare against batched results
167
- non_batched_results: list[str] = []
168
-
169
- for lora_id, info in long_context_infos.items():
170
- context_len = info["context_length"]
171
- lora_prompt = (prompts_and_responses[context_len][0]["prompt"],
172
- sampling_params,
173
- _create_lora_request(lora_id, long_context_infos))
174
- lora_output = generate(lora_llm, lora_prompt)
175
- non_batched_results.append(lora_output)
176
-
177
- # Create batched results
178
- # Each element of the batch must be
179
- # (prompt, prompt_sampling_params, prompt_lora_request)
180
- batched_prompts: list[tuple[str, SamplingParams,
181
- Optional[LoRARequest]]] = []
182
- for lora_id, info in long_context_infos.items():
183
- context_len = info["context_length"]
184
- batched_prompts.extend([
185
- (prompts_and_responses[context_len][0]["prompt"], sampling_params,
186
- _create_lora_request(lora_id, long_context_infos))
187
- ])
188
- batched_results = batched_generate(lora_llm, batched_prompts)
189
-
190
- # Results should be the same
191
- for non_batched, batched in zip(non_batched_results, batched_results):
192
- assert non_batched == batched, (
193
- "Non batched and batched results should be the "
194
- f"same:\n{batched}\n{non_batched}")
195
-
196
-
197
-@pytest.mark.skip_global_cleanup
198
-def test_self_consistency(lora_llm, long_context_infos):
199
- """We test consistency of the batched kernel by permuting batched
200
- inputs and comparing the results to the non-permuted batched results.
201
- """
202
- num_loras = len(long_context_infos)
203
-
204
- # Create results in order of long_context_infos
205
- batched_prompts: list[tuple[str, SamplingParams,
206
- Optional[LoRARequest]]] = []
207
- for lora_id, info in long_context_infos.items():
208
- context_len = info["context_length"]
209
- batched_prompts.extend([
210
- (prompts_and_responses[context_len][0]["prompt"], sampling_params,
211
- _create_lora_request(lora_id, long_context_infos))
212
- ])
213
-
214
- batched_results = batched_generate(lora_llm, batched_prompts)
215
-
216
- permutation = np.random.default_rng(seed=42).permutation(num_loras)
217
-
218
- # Create results in random order of permutation
219
- batched_prompts = []
220
- for i in permutation:
221
- lora_id, info = list(long_context_infos.items())[i]
222
- context_len = info["context_length"]
223
- batched_prompts.extend([
224
- (prompts_and_responses[context_len][0]["prompt"], sampling_params,
225
- _create_lora_request(lora_id, long_context_infos))
226
- ])
227
-
228
- permutated_batched_results = batched_generate(lora_llm, batched_prompts)
229
-
230
- # Results should be the same
231
- for i in range(num_loras):
232
- assert batched_results[i] == permutated_batched_results[
233
- permutation[i]], (
234
- f"Results should be the same:\n{batched_results[i]}"
235
- f"\n{permutated_batched_results[permutation[i]]}")
236
-
237
-
238
-@pytest.mark.skip_global_cleanup
239
-def test_quality(lora_llm, long_context_infos):
240
- """We test the quality of the answers given by the LoRA model by
241
- comparing the generated text to the merged model's outputs.
242
-
243
- This is effectively a mini-benchmark over four prompts.
244
- If this test fails, this indicates that the quality of the LoRA model
245
- is suboptimal compared to the merged model. For example, if the model
246
- does not output valid dictionaries, this test will fail.
247
-
248
- If needed for testing, the merged versions of the models are available
249
- as part of the `conftest`.
250
-
251
- The test is expected to run for about 1 minute on a p4de.24xlarge
252
- instance.
253
- """
254
- scores: list[float] = []
255
- for lora_id, info in long_context_infos.items():
256
- context_len = info["context_length"]
257
- for prompt_and_response in prompts_and_responses[context_len]:
258
- lora_prompt = (prompt_and_response["prompt"], sampling_params,
259
- _create_lora_request(lora_id, long_context_infos))
260
- response = generate(lora_llm, lora_prompt)
261
- golden_answer = prompt_and_response["golden_answer"]
262
- score = evaluate_json_response(response, golden_answer)
263
- scores.append(score)
264
- assert score > 0.3, ("Quality of the answer is not good enough. "
265
- f"Expected {golden_answer}, got {response}")
266
- assert np.mean(scores) > 0.5
267
-
268
-
269
-@pytest.mark.skip_global_cleanup
270
-def test_max_len(lora_llm, long_context_infos):
271
- """Test that we raise an ValueError when the input of a given LoRA
272
- model exceeds the maximum length."""
273
- # Since each LoRA model has a different maximum length, we need to
274
- # test each one separately
275
- for lora_id, info in long_context_infos.items():
276
- context_len = info["context_length"]
277
- lora_request = _create_lora_request(lora_id, long_context_infos)
278
- # Good prompt should be fine
279
- good_prompt = prompts_and_responses[context_len][0]["prompt"]
280
- generate(lora_llm, (good_prompt, sampling_params, lora_request))
281
- # Bad prompt should raise an error
282
- bad_prompt = good_prompt * 2
283
- with pytest.raises(ValueError):
284
- generate(lora_llm, (bad_prompt, sampling_params, lora_request))
285
-
286
- # Also test batched
287
- batched_prompts: list[tuple[str, SamplingParams,
288
- Optional[LoRARequest]]] = []
289
- for lora_id_with_bad_inputs in long_context_infos:
290
- for lora_id, info in long_context_infos.items():
291
- context_len = info["context_length"]
292
- batched_prompts.extend([
293
- (prompts_and_responses[context_len][0]["prompt"] *
294
- (2 if lora_id == lora_id_with_bad_inputs else 1),
295
- sampling_params,
296
- _create_lora_request(lora_id, long_context_infos))
297
- ])
298
- # Turn good prompt into bad prompt inside of batched prompts
299
-
300
- with pytest.raises(ValueError):
301
- batched_generate(lora_llm, batched_prompts)