A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 10:00 UTC
[Bugfix] Fix RequestOutput miss lora_request (#30636)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 16, 2025 at 17:36 UTC
0e391e757039c8a49a7956226c59d1fbde72459d
3 files changed
+19
-7
tests/lora/test_gptoss_tp.py
+5
-1
@@ -76,6 +76,8 @@ def test_gpt_oss_lora(gptoss20b_lora_files):
76
enable_lora=True,
77
max_loras=4,
78
max_lora_rank=8,
79
+ max_num_seqs=2,
80
+ max_num_batched_tokens=2048,
81
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
82
cudagraph_specialize_lora=False,
83
),
@@ -94,8 +96,10 @@ def test_gpt_oss_lora_tp2(gptoss20b_lora_files, fully_sharded_loras):
96
enable_lora=True,
97
max_loras=2,
98
max_lora_rank=8,
97
- max_num_seqs=16,
99
+ max_num_seqs=2,
100
+ max_num_batched_tokens=2048,
101
tensor_parallel_size=2,
102
+ gpu_memory_utilization=0.8,
103
fully_sharded_loras=fully_sharded_loras,
104
compilation_config=vllm.config.CompilationConfig( # Avoid OOM
105
cudagraph_specialize_lora=False,
tests/lora/test_llama_tp.py
+8
-1
@@ -76,11 +76,18 @@ def do_sample(
76
if lora_id
77
else None,
78
)
79
- # Print the outputs.
79
+ lora_request = LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None
80
generated_texts: list[str] = []
81
for output in outputs:
82
prompt = output.prompt
83
generated_text = output.outputs[0].text
84
+ # The output should include correct lora_request info
85
+ if lora_request is not None:
86
+ assert output.lora_request.lora_name == lora_request.lora_name
87
+ assert output.lora_request.lora_int_id == lora_request.lora_int_id
88
+ assert output.lora_request.lora_path == lora_request.lora_path
89
+ else:
90
+ assert output.lora_request is None
91
generated_texts.append(generated_text)
92
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
93
return generated_texts
vllm/v1/engine/output_processor.py
+6
-5
@@ -8,6 +8,7 @@ from typing import Any, cast
8
9
import torch
10
11
+from vllm.lora.request import LoRARequest
12
from vllm.outputs import (
13
CompletionOutput,
14
PoolingOutput,
@@ -93,7 +94,7 @@ class RequestState:
94
request_id: str,
95
parent_req: ParentRequest | None,
96
request_index: int,
96
- lora_name: str | None,
97
+ lora_request: LoRARequest | None,
98
output_kind: RequestOutputKind,
99
prompt: str | None,
100
prompt_token_ids: list[int] | None,
@@ -112,7 +113,8 @@ class RequestState:
113
self.request_id = request_id
114
self.parent_req = parent_req
115
self.request_index = request_index
115
- self.lora_name = lora_name
116
+ self.lora_request = lora_request
117
+ self.lora_name = lora_request.lora_name if lora_request is not None else None
118
self.output_kind = output_kind
119
self.prompt = prompt
120
self.prompt_token_ids = prompt_token_ids
@@ -178,9 +180,7 @@ class RequestState:
180
request_id=request.request_id,
181
parent_req=parent_req,
182
request_index=request_index,
181
- lora_name=(
182
- request.lora_request.name if request.lora_request is not None else None
183
- ),
183
+ lora_request=request.lora_request,
184
output_kind=output_kind,
185
prompt=prompt,
186
prompt_token_ids=request.prompt_token_ids,
@@ -289,6 +289,7 @@ class RequestState:
289
290
return RequestOutput(
291
request_id=request_id,
292
+ lora_request=self.lora_request,
293
prompt=self.prompt,
294
prompt_token_ids=prompt_token_ids,
295
prompt_logprobs=prompt_logprobs,