A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[CI/Build][LoRA] Temporarily fix long context failure issue (#9579)
Jee Jee Li committed
Oct 22, 2024 at 19:32 UTC
a48e3ec0523b4ac7230159bb38ae1dc4a2f0346a
1 file changed
+20
-11
tests/lora/test_long_context.py
+20
-11
@@ -28,9 +28,15 @@ sampling_params = SamplingParams(
28
def _create_lora_request(lora_id, long_context_infos):
29
context_len = long_context_infos[lora_id]["context_length"]
30
scaling_factor = context_len_to_scaling_factor[context_len]
31
- return LoRARequest(context_len, lora_id,
32
- long_context_infos[lora_id]["lora"], None,
33
- 4096 * scaling_factor)
31
+ return LoRARequest(
32
+ # There are 2 LoRAs for 16K, we need to add lora_id to indicate
33
+ # they are different LoRAs.
34
+ context_len + str(lora_id),
35
+ lora_id,
36
+ long_context_infos[lora_id]["lora"],
37
+ None,
38
+ 4096 * scaling_factor,
39
+ )
40
41
42
def evaluate_json_response(model_response, golden_response):
@@ -108,14 +114,17 @@ def lora_llm(long_context_infos):
114
for info in long_context_infos.values()
115
]
116
111
- llm = vllm.LLM("meta-llama/Llama-2-13b-chat-hf",
112
- enable_lora=True,
113
- max_num_seqs=16,
114
- max_loras=2,
115
- long_lora_scaling_factors=tuple(scaling_factors),
116
- max_num_batched_tokens=4096 * 8,
117
- tensor_parallel_size=4,
118
- distributed_executor_backend="mp")
117
+ llm = vllm.LLM(
118
+ "meta-llama/Llama-2-13b-chat-hf",
119
+ enable_lora=True,
120
+ max_num_seqs=16,
121
+ max_loras=2,
122
+ long_lora_scaling_factors=tuple(scaling_factors),
123
+ max_num_batched_tokens=4096 * 8,
124
+ tensor_parallel_size=4,
125
+ # FIXME enable async output processor
126
+ disable_async_output_proc=True,
127
+ distributed_executor_backend="mp")
128
yield llm
129
del llm
130