A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 08:15 UTC
[n-gen] DO NOT repeatedly return finished child requests (#28591)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com>
Jialin Ouyang committed
Nov 12, 2025 at 19:36 UTC
a1d3866dda6539a0e9e2cfc49e9cb1e887baaaec
2 files changed
+115
-3
tests/v1/engine/test_parallel_sampling.py
new
+103
@@ -0,0 +1,103 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from vllm import SamplingParams
5
+from vllm.outputs import CompletionOutput
6
+from vllm.sampling_params import RequestOutputKind
7
+from vllm.v1.engine.parallel_sampling import ParentRequest
8
+
9
+
10
+def test_parent_request_to_output_stream() -> None:
11
+ parent_request = ParentRequest("parent_id", SamplingParams(n=2))
12
+ parent_request.child_requests = {"child_id_0", "child_id_1"}
13
+ output_0 = CompletionOutput(
14
+ index=0, text="child 0", token_ids=[], cumulative_logprob=None, logprobs=None
15
+ )
16
+ output_1 = CompletionOutput(
17
+ index=1, text="child 1", token_ids=[], cumulative_logprob=None, logprobs=None
18
+ )
19
+ # Request not finished
20
+ assert ("parent_id", [output_0], False) == parent_request.get_outputs(
21
+ "child_id_0", output_0
22
+ )
23
+ assert ("parent_id", [output_1], False) == parent_request.get_outputs(
24
+ "child_id_1", output_1
25
+ )
26
+ assert ("parent_id", [output_0], False) == parent_request.get_outputs(
27
+ "child_id_0", output_0
28
+ )
29
+ assert ("parent_id", [output_1], False) == parent_request.get_outputs(
30
+ "child_id_1", output_1
31
+ )
32
+
33
+ # output_1 finished
34
+ output_1.finish_reason = "ended"
35
+ assert ("parent_id", [output_0], False) == parent_request.get_outputs(
36
+ "child_id_0", output_0
37
+ )
38
+ assert ("parent_id", [output_1], False) == parent_request.get_outputs(
39
+ "child_id_1", output_1
40
+ )
41
+ # Finished output_1 had already returned, DO NOT returned again
42
+ assert ("parent_id", [output_0], False) == parent_request.get_outputs(
43
+ "child_id_0", output_0
44
+ )
45
+ assert parent_request.get_outputs("child_id_1", output_1) == (
46
+ "parent_id",
47
+ [],
48
+ False,
49
+ )
50
+
51
+ # output_0 finished
52
+ output_0.finish_reason = "ended"
53
+ assert ("parent_id", [output_0], True) == parent_request.get_outputs(
54
+ "child_id_0", output_0
55
+ )
56
+ assert parent_request.get_outputs("child_id_1", output_1) == ("parent_id", [], True)
57
+ # Finished output_0 had already returned, DO NOT returned again
58
+ assert parent_request.get_outputs("child_id_0", output_0) == ("parent_id", [], True)
59
+ assert parent_request.get_outputs("child_id_1", output_1) == ("parent_id", [], True)
60
+
61
+
62
+def test_parent_request_to_output_final_only() -> None:
63
+ parent_request = ParentRequest(
64
+ "parent_id", SamplingParams(n=2, output_kind=RequestOutputKind.FINAL_ONLY)
65
+ )
66
+ parent_request.child_requests = {"child_id_0", "child_id_1"}
67
+ output_0 = CompletionOutput(
68
+ index=0, text="child 0", token_ids=[], cumulative_logprob=None, logprobs=None
69
+ )
70
+ output_1 = CompletionOutput(
71
+ index=1, text="child 1", token_ids=[], cumulative_logprob=None, logprobs=None
72
+ )
73
+ # Request not finished, return nothing
74
+ assert parent_request.get_outputs("child_id_0", output_0) == (
75
+ "parent_id",
76
+ [],
77
+ False,
78
+ )
79
+ assert parent_request.get_outputs("child_id_1", output_1) == (
80
+ "parent_id",
81
+ [],
82
+ False,
83
+ )
84
+ # output_1 finished, but outputs won't be returned until all child requests finished
85
+ output_1.finish_reason = "ended"
86
+ assert parent_request.get_outputs("child_id_0", output_0) == (
87
+ "parent_id",
88
+ [],
89
+ False,
90
+ )
91
+ assert parent_request.get_outputs("child_id_1", output_1) == (
92
+ "parent_id",
93
+ [],
94
+ False,
95
+ )
96
+ # output_0 finished, as all child requests finished, the output would be returned
97
+ output_0.finish_reason = "ended"
98
+ assert ("parent_id", [output_0, output_1], True) == parent_request.get_outputs(
99
+ "child_id_0", output_0
100
+ )
101
+ assert ("parent_id", [output_0, output_1], True) == parent_request.get_outputs(
102
+ "child_id_1", output_1
103
+ )
vllm/v1/engine/parallel_sampling.py
+12
-3
@@ -97,12 +97,21 @@ class ParentRequest:
97
child_request_id: str,
98
completion_output: CompletionOutput,
99
) -> tuple[str, list[CompletionOutput], bool]:
100
+ already_finished_and_returned: bool = False
101
if completion_output.finished():
101
- self.child_requests.remove(child_request_id)
102
+ if child_request_id in self.child_requests:
103
+ self.child_requests.remove(child_request_id)
104
+ else:
105
+ # child request ID is not available in child_requests
106
+ # which means the request had finished in previous
107
+ # batch step and returned to the client earlier
108
+ already_finished_and_returned = True
109
110
if self.sampling_params.output_kind != RequestOutputKind.FINAL_ONLY:
104
- # If streaming, just return the current output.
105
- outputs = [completion_output]
111
+ # If streaming, just return the current output
112
+ #
113
+ # DO NOT output finished and already returned child request to client again
114
+ outputs = [] if already_finished_and_returned else [completion_output]
115
else:
116
# If not streaming, aggregate the n final outputs.
117
self.output_aggregator[completion_output.index] = completion_output