@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a48e3ec052
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[CI/Build][LoRA] Temporarily fix long context failure issue (#9579)

Jee Jee Li committed Oct 22, 2024 at 19:32 UTC a48e3ec0523b4ac7230159bb38ae1dc4a2f0346a
1 file changed +20 -11
tests/lora/test_long_context.py
+20 -11
@@ -28,9 +28,15 @@ sampling_params = SamplingParams(
28 def _create_lora_request(lora_id, long_context_infos):
29 context_len = long_context_infos[lora_id]["context_length"]
30 scaling_factor = context_len_to_scaling_factor[context_len]
31 - return LoRARequest(context_len, lora_id,
32 - long_context_infos[lora_id]["lora"], None,
33 - 4096 * scaling_factor)
31 + return LoRARequest(
32 + # There are 2 LoRAs for 16K, we need to add lora_id to indicate
33 + # they are different LoRAs.
34 + context_len + str(lora_id),
35 + lora_id,
36 + long_context_infos[lora_id]["lora"],
37 + None,
38 + 4096 * scaling_factor,
39 + )
40
41
42 def evaluate_json_response(model_response, golden_response):
@@ -108,14 +114,17 @@ def lora_llm(long_context_infos):
114 for info in long_context_infos.values()
115 ]
116
111 - llm = vllm.LLM("meta-llama/Llama-2-13b-chat-hf",
112 - enable_lora=True,
113 - max_num_seqs=16,
114 - max_loras=2,
115 - long_lora_scaling_factors=tuple(scaling_factors),
116 - max_num_batched_tokens=4096 * 8,
117 - tensor_parallel_size=4,
118 - distributed_executor_backend="mp")
117 + llm = vllm.LLM(
118 + "meta-llama/Llama-2-13b-chat-hf",
119 + enable_lora=True,
120 + max_num_seqs=16,
121 + max_loras=2,
122 + long_lora_scaling_factors=tuple(scaling_factors),
123 + max_num_batched_tokens=4096 * 8,
124 + tensor_parallel_size=4,
125 + # FIXME enable async output processor
126 + disable_async_output_proc=True,
127 + distributed_executor_backend="mp")
128 yield llm
129 del llm
130