[Bugfix][Frontend] Return all choices from /inference/v1/generate when n > 1 (#52399)
Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co>
Quentin Gallouédec committed
Aug 19, 2026 at 08:48 UTC
92bdee05cb4ea5e94c4cce3eb0544f51d6eece8d
2 files changed
+27
-2
tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
+24
@@ -183,6 +183,30 @@ async def test_generate_defaults_max_tokens_when_omitted(client):
183
)
184
185
186
+@pytest.mark.asyncio
187
+@pytest.mark.skipif(
188
+ envs.VLLM_USE_RUST_FRONTEND,
189
+ reason="parallel sampling (n > 1) is not supported by the Rust frontend",
190
+)
191
+async def test_generate_returns_all_choices_when_n_greater_than_one(client):
192
+ """Regression: ``n > 1`` must return ``n`` choices.
193
+
194
+ Non-streaming requests kept ``SamplingParams``' default output kind,
195
+ ``CUMULATIVE``, so only the sequences updated during the last engine step
196
+ reached the response and the others were silently dropped.
197
+ """
198
+ payload = {
199
+ "model": MODEL_NAME,
200
+ "token_ids": [1, 2, 3],
201
+ "sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
202
+ "stream": False,
203
+ }
204
+ resp = await client.post(GEN_ENDPOINT, json=payload)
205
+ resp.raise_for_status()
206
+ data = resp.json()
207
+ assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
208
+
209
+
210
@pytest.mark.asyncio
211
async def test_generate_stream(client):
212
payload = {
vllm/entrypoints/scale_out/token_in_token_out/serving.py
+3
-2
@@ -218,8 +218,9 @@ class ServingTokens(GenerateBaseServing):
218
219
if self.force_no_detokenize:
220
sampling_params.detokenize = False
221
- if request.stream:
222
- sampling_params.output_kind = RequestOutputKind.DELTA
221
+ sampling_params.output_kind = (
222
+ RequestOutputKind.DELTA if request.stream else RequestOutputKind.FINAL_ONLY
223
+ )
224
225
self._log_inputs(
226
request_id,