A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 02:15 UTC
fix LoRA-related examples (#29956)
Signed-off-by: Iceber Gu <caiwei95@hotmail.com>
Iceber Gu committed
Dec 4, 2025 at 11:48 UTC
33a3d6c79826aa7938db45a4e24a213664859cc0
2 files changed
+7
-27
examples/offline_inference/lora_with_quantization_inference.py
+4
-12
@@ -23,31 +23,23 @@ def create_test_prompts(
23
# this is an example of using quantization without LoRA
24
(
25
"My name is",
26
- SamplingParams(
27
- temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128
28
- ),
26
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
27
None,
28
),
29
# the next three examples use quantization with LoRA
30
(
31
"my name is",
34
- SamplingParams(
35
- temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128
36
- ),
32
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
33
LoRARequest("lora-test-1", 1, lora_path),
34
),
35
(
36
"The capital of USA is",
41
- SamplingParams(
42
- temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128
43
- ),
37
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
38
LoRARequest("lora-test-2", 1, lora_path),
39
),
40
(
41
"The capital of France is",
48
- SamplingParams(
49
- temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128
50
- ),
42
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
43
LoRARequest("lora-test-3", 1, lora_path),
44
),
45
]
examples/offline_inference/multilora_inference.py
+3
-15
@@ -27,9 +27,7 @@ def create_test_prompts(
27
return [
28
(
29
"A robot may not injure a human being",
30
- SamplingParams(
31
- temperature=0.0, logprobs=1, prompt_logprobs=1, max_tokens=128
32
- ),
30
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
31
None,
32
),
33
(
@@ -41,22 +39,12 @@ def create_test_prompts(
39
),
40
(
41
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
44
- SamplingParams(
45
- temperature=0.0,
46
- logprobs=1,
47
- prompt_logprobs=1,
48
- max_tokens=128,
49
- ),
42
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
43
LoRARequest("sql-lora", 1, lora_path),
44
),
45
(
46
"[user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_74 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]", # noqa: E501
54
- SamplingParams(
55
- temperature=0.0,
56
- logprobs=1,
57
- prompt_logprobs=1,
58
- max_tokens=128,
59
- ),
47
+ SamplingParams(temperature=0.0, logprobs=1, max_tokens=128),
48
LoRARequest("sql-lora2", 2, lora_path),
49
),
50
]