@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f790068600
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 00:00 UTC

[Core] Add a random suffix to frontend-provided request IDs (#27987)

Signed-off-by: Mark McLoughlin <markmc@redhat.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>

Mark McLoughlin committed Dec 23, 2025 at 21:05 UTC f790068600a0920e95d33d22686355bf8e48b077
16 files changed +328 -154
tests/v1/engine/test_async_llm.py
+2 -2
@@ -260,7 +260,7 @@ async def test_multi_abort(output_kind: RequestOutputKind):
260
261 # Use multi-abort to abort multiple requests at once
262 abort_request_ids = [request_ids[i] for i in REQUEST_IDS_TO_ABORT]
263 - await engine.abort(abort_request_ids)
263 + await engine.abort(abort_request_ids, internal=False)
264
265 # Wait for all tasks to complete
266 results = await asyncio.gather(*tasks, return_exceptions=True)
@@ -609,7 +609,7 @@ async def test_abort_final_output(output_kind: RequestOutputKind):
609 await asyncio.sleep(0.5)
610
611 # Abort the request
612 - await engine.abort(request_id)
612 + await engine.abort(request_id, internal=False)
613
614 # Wait for generation to complete and return final output
615 final_output = await generated
tests/v1/engine/test_engine_core.py
+7 -1
@@ -40,10 +40,16 @@ TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)
40 PROMPT = "I am Gyoubu Masataka Oniwa"
41 PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids
42
43 +_REQUEST_COUNTER = 0
44 +
45
46 def make_request() -> EngineCoreRequest:
47 + global _REQUEST_COUNTER
48 + _REQUEST_COUNTER += 1
49 + request_id = f"request-{_REQUEST_COUNTER}"
50 return EngineCoreRequest(
46 - request_id=str(uuid.uuid4()),
51 + request_id=request_id,
52 + external_req_id=f"{request_id}-{uuid.uuid4()}",
53 prompt_token_ids=PROMPT_TOKENS,
54 mm_features=None,
55 sampling_params=SamplingParams(),
tests/v1/engine/test_engine_core_client.py
+7 -1
@@ -45,6 +45,8 @@ TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)
45 PROMPT = "Hello my name is Robert and I love quantization kernels"
46 PROMPT_TOKENS = TOKENIZER(PROMPT).input_ids
47
48 +_REQUEST_COUNTER = 0
49 +
50
51 def make_request(
52 params: SamplingParams, prompt_tokens_ids: list[int] | None = None
@@ -52,8 +54,12 @@ def make_request(
54 if not prompt_tokens_ids:
55 prompt_tokens_ids = PROMPT_TOKENS
56
57 + global _REQUEST_COUNTER
58 + _REQUEST_COUNTER += 1
59 + request_id = f"request-{_REQUEST_COUNTER}"
60 return EngineCoreRequest(
56 - request_id=str(uuid.uuid4()),
61 + request_id=request_id,
62 + external_req_id=f"{request_id}-{uuid.uuid4()}",
63 prompt_token_ids=prompt_tokens_ids,
64 mm_features=None,
65 sampling_params=params,
tests/v1/engine/test_fast_incdec_prefix_err.py
+1
@@ -27,6 +27,7 @@ def test_fast_inc_detok_invalid_utf8_err_case():
27 params = SamplingParams(skip_special_tokens=True)
28 request = EngineCoreRequest(
29 request_id="test",
30 + external_req_id="test-ext",
31 prompt_token_ids=prompt_token_ids,
32 mm_features=None,
33 sampling_params=params,
tests/v1/engine/test_output_processor.py
+82 -44
@@ -58,12 +58,12 @@ def test_incremental_detokenization(
58 output_processor = OutputProcessor(
59 dummy_test_vectors.tokenizer, log_stats=False, stream_interval=stream_interval
60 )
61 - engine_core = MockEngineCore(tokens_list=dummy_test_vectors.generation_tokens)
61
62 # Make N requests.
63 requests = [
64 EngineCoreRequest(
66 - request_id=f"request-{idx}",
65 + request_id=f"request-{idx}-int",
66 + external_req_id=f"request-{idx}",
67 prompt_token_ids=prompt_tokens,
68 mm_features=None,
69 eos_token_id=None,
@@ -83,6 +83,11 @@ def test_incremental_detokenization(
83 for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
84 ]
85
86 + engine_core = MockEngineCore(
87 + tokens_list=dummy_test_vectors.generation_tokens,
88 + request_ids=[req.request_id for req in requests],
89 + )
90 +
91 # Add requests to the detokenizer.
92 for request, prompt in zip(requests, dummy_test_vectors.prompt_strings):
93 output_processor.add_request(request, prompt)
@@ -438,15 +443,6 @@ def test_logprobs_processor(
443 dummy_test_vectors,
444 ):
445 output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=False)
441 - engine_core = MockEngineCore(
442 - tokens_list=dummy_test_vectors.generation_tokens,
443 - generated_logprobs_raw=None
444 - if num_sample_logprobs is None
445 - else dummy_test_vectors.generation_logprobs,
446 - prompt_logprobs_raw=None
447 - if num_prompt_logprobs is None
448 - else dummy_test_vectors.prompt_logprobs,
449 - )
446
447 # Make N requests.
448 request_id_list = [
@@ -454,7 +450,8 @@ def test_logprobs_processor(
450 ]
451 requests = [
452 EngineCoreRequest(
457 - request_id=request_id_list[idx],
453 + request_id=request_id_list[idx] + "-int",
454 + external_req_id=request_id_list[idx],
455 prompt_token_ids=prompt_tokens,
456 mm_features=None,
457 eos_token_id=None,
@@ -476,6 +473,17 @@ def test_logprobs_processor(
473 for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
474 ]
475
476 + engine_core = MockEngineCore(
477 + tokens_list=dummy_test_vectors.generation_tokens,
478 + generated_logprobs_raw=None
479 + if num_sample_logprobs is None
480 + else dummy_test_vectors.generation_logprobs,
481 + prompt_logprobs_raw=None
482 + if num_prompt_logprobs is None
483 + else dummy_test_vectors.prompt_logprobs,
484 + request_ids=[req.request_id for req in requests],
485 + )
486 +
487 # Add requests to the detokenizer.
488 for request, prompt in zip(requests, dummy_test_vectors.prompt_strings):
489 output_processor.add_request(request, prompt)
@@ -621,19 +629,12 @@ def test_stop_token(
629 ]
630 prompt_string = dummy_test_vectors.prompt_strings[0]
631 prompt_tokens = dummy_test_vectors.prompt_tokens[0]
624 - engine_core = MockEngineCore(
625 - tokens_list=[generation_tokens],
626 - generated_logprobs_raw=[generation_logprobs] if do_logprobs else None,
627 - prompt_logprobs_raw=None,
628 - eos_token_id=eos_token_id,
629 - stop_token_ids=stop_token_ids,
630 - ignore_eos=ignore_eos,
631 - )
632
633 # Make request.
634 request_id = "request-0"
635 request = EngineCoreRequest(
636 request_id=request_id,
637 + external_req_id=request_id + "-ext",
638 prompt_token_ids=prompt_tokens,
639 mm_features=None,
640 eos_token_id=eos_token_id,
@@ -655,6 +656,16 @@ def test_stop_token(
656 pooling_params=None,
657 )
658
659 + engine_core = MockEngineCore(
660 + tokens_list=[generation_tokens],
661 + generated_logprobs_raw=[generation_logprobs] if do_logprobs else None,
662 + prompt_logprobs_raw=None,
663 + eos_token_id=eos_token_id,
664 + stop_token_ids=stop_token_ids,
665 + ignore_eos=ignore_eos,
666 + request_ids=[request.request_id],
667 + )
668 +
669 # Add request to the detokenizer.
670 output_processor.add_request(request, prompt_string)
671
@@ -720,13 +731,6 @@ def test_stop_string(
731 dummy_test_vectors,
732 ):
733 output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=False)
723 - engine_core = MockEngineCore(
724 - tokens_list=dummy_test_vectors.generation_tokens,
725 - generated_logprobs_raw=dummy_test_vectors.generation_logprobs
726 - if num_sample_logprobs
727 - else None,
728 - prompt_logprobs_raw=None,
729 - )
734
735 # Make N requests.
736 request_id_list = [
@@ -734,7 +738,8 @@ def test_stop_string(
738 ]
739 requests = [
740 EngineCoreRequest(
737 - request_id=request_id_list[idx],
741 + request_id=request_id_list[idx] + "-int",
742 + external_req_id=request_id_list[idx],
743 prompt_token_ids=prompt_tokens,
744 mm_features=None,
745 eos_token_id=None,
@@ -756,6 +761,15 @@ def test_stop_string(
761 for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
762 ]
763
764 + engine_core = MockEngineCore(
765 + tokens_list=dummy_test_vectors.generation_tokens,
766 + generated_logprobs_raw=dummy_test_vectors.generation_logprobs
767 + if num_sample_logprobs
768 + else None,
769 + prompt_logprobs_raw=None,
770 + request_ids=[req.request_id for req in requests],
771 + )
772 +
773 # Add requests to the detokenizer.
774 for request, prompt in zip(requests, dummy_test_vectors.prompt_strings):
775 output_processor.add_request(request, prompt)
@@ -813,9 +827,12 @@ def test_stop_string(
827 for idx, (ref_gen_str, stop_str) in enumerate(
828 zip(dummy_test_vectors.generation_strings, STOP_STRINGS)
829 ):
816 - # Request should be aborted.
830 + # Request should be aborted (check internal ID in abort list).
831 + internal_request_id = f"request-{idx}-int"
832 + assert internal_request_id in aborted
833 +
834 + # Use external ID for collecting outputs
835 request_id = f"request-{idx}"
818 - assert request_id in aborted
836
837 # Collected values that were generated.
838 gen_str = gen_strings[request_id]
@@ -848,13 +865,13 @@ def test_stop_string(
865
866 def test_iteration_stats(dummy_test_vectors):
867 output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=True)
851 - engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
868 engine_core_timestamp = time.monotonic()
869
870 # Make N requests.
871 requests = [
872 EngineCoreRequest(
873 request_id=f"request-{idx}",
874 + external_req_id=f"request-{idx}-ext",
875 prompt_token_ids=prompt_tokens,
876 mm_features=None,
877 eos_token_id=None,
@@ -868,6 +885,11 @@ def test_iteration_stats(dummy_test_vectors):
885 for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
886 ]
887
888 + engine_core = MockEngineCore(
889 + dummy_test_vectors.generation_tokens,
890 + request_ids=[req.request_id for req in requests],
891 + )
892 +
893 # Add all requests except one to the OutputProcessor.
894 num_active = len(dummy_test_vectors.generation_tokens) - 1
895 for request in requests[:num_active]:
@@ -922,7 +944,6 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
944 output_processor = OutputProcessor(
945 dummy_test_vectors.tokenizer, log_stats=log_stats
946 )
925 - engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
947 engine_core_timestamp = time.monotonic()
948
949 # Create LoRA requests
@@ -936,7 +957,8 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
957 lora_assignments = [lora1, lora2, None]
958 requests = [
959 EngineCoreRequest(
939 - request_id=f"request-{idx}",
960 + request_id=f"request-{idx}-int",
961 + external_req_id=f"request-{idx}",
962 prompt_token_ids=prompt_tokens,
963 mm_features=None,
964 eos_token_id=None,
@@ -950,6 +972,11 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
972 for idx, prompt_tokens in enumerate(dummy_test_vectors.prompt_tokens)
973 ]
974
975 + engine_core = MockEngineCore(
976 + dummy_test_vectors.generation_tokens,
977 + request_ids=[req.request_id for req in requests],
978 + )
979 +
980 # Add all requests to the OutputProcessor
981 for request in requests:
982 output_processor.add_request(request, None)
@@ -1015,9 +1042,9 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
1042 outputs = EngineCoreOutputs(
1043 outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()
1044 )
1018 - # Find and mark request-0 as finished (it uses lora-1)
1045 + # Find and mark request-0-int as finished (it uses lora-1)
1046 for output in outputs.outputs:
1020 - if output.request_id == "request-0":
1047 + if output.request_id == "request-0-int":
1048 output.finish_reason = FinishReason.LENGTH
1049 break
1050
@@ -1040,9 +1067,9 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
1067 outputs = EngineCoreOutputs(
1068 outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()
1069 )
1043 - # Find and mark request-1 as finished (it uses lora-2)
1070 + # Find and mark request-1-int as finished (it uses lora-2)
1071 for output in outputs.outputs:
1045 - if output.request_id == "request-1":
1072 + if output.request_id == "request-1-int":
1073 output.finish_reason = FinishReason.LENGTH
1074 break
1075
@@ -1064,9 +1091,9 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
1091 outputs = EngineCoreOutputs(
1092 outputs=engine_core.get_outputs(), scheduler_stats=SchedulerStats()
1093 )
1067 - # Find and mark request-2 as finished (it has no LoRA)
1094 + # Find and mark request-2-int as finished (it has no LoRA)
1095 for output in outputs.outputs:
1069 - if output.request_id == "request-2":
1096 + if output.request_id == "request-2-int":
1097 output.finish_reason = FinishReason.LENGTH
1098 break
1099
@@ -1107,7 +1134,9 @@ async def test_request_output_collector():
1134 for idx in range(NUM_REQS)
1135 ]
1136
1110 - collector = RequestOutputCollector(RequestOutputKind.DELTA)
1137 + collector = RequestOutputCollector(
1138 + RequestOutputKind.DELTA, request_id="my-request-id-int"
1139 + )
1140
1141 # CASE 1: Put then get.
1142 outputs = make_outputs()
@@ -1163,7 +1192,9 @@ async def test_request_output_collector():
1192 @pytest.mark.asyncio
1193 async def test_cumulative_output_collector_n():
1194 """Test collector correctly handles multiple outputs by index."""
1166 - collector = RequestOutputCollector(RequestOutputKind.CUMULATIVE)
1195 + collector = RequestOutputCollector(
1196 + RequestOutputKind.CUMULATIVE, request_id="my-request-id-int"
1197 + )
1198 outputs = [
1199 RequestOutput(
1200 request_id="my-request-id",
@@ -1242,11 +1273,13 @@ async def test_cumulative_output_collector_n():
1273
1274
1275 @pytest.mark.parametrize("runner", ["generate", "pooling"])
1245 -def test_abort_requests(runner: str, dummy_test_vectors):
1276 +@pytest.mark.parametrize("abort_by", ["internal", "external"])
1277 +def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
1278 output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=True)
1279 requests = [
1280 EngineCoreRequest(
1281 request_id=f"request-{idx}",
1282 + external_req_id=f"external-{idx}",
1283 prompt_token_ids=prompt_tokens,
1284 mm_features=None,
1285 eos_token_id=None,
@@ -1265,8 +1298,13 @@ def test_abort_requests(runner: str, dummy_test_vectors):
1298 output_kind = request.sampling_params.output_kind
1299 else:
1300 output_kind = request.pooling_params.output_kind
1268 - queue = RequestOutputCollector(output_kind=output_kind)
1301 + queue = RequestOutputCollector(
1302 + output_kind=output_kind, request_id=request.request_id
1303 + )
1304 output_processor.add_request(request, None, queue=queue)
1305
1306 for request in requests:
1272 - output_processor.abort_requests([request.request_id])
1307 + if abort_by == "internal":
1308 + output_processor.abort_requests([request.request_id], internal=True)
1309 + else:
1310 + output_processor.abort_requests([request.external_req_id], internal=False)
tests/v1/engine/test_parallel_sampling.py
+37 -56
@@ -4,11 +4,12 @@
4 from vllm import SamplingParams
5 from vllm.outputs import CompletionOutput
6 from vllm.sampling_params import RequestOutputKind
7 +from vllm.v1.engine import EngineCoreRequest
8 from vllm.v1.engine.parallel_sampling import ParentRequest
9
10
11 def test_parent_request_to_output_stream() -> None:
11 - parent_request = ParentRequest("parent_id", SamplingParams(n=2))
12 + parent_request = ParentRequest(make_request(SamplingParams(n=2)))
13 parent_request.child_requests = {"child_id_0", "child_id_1"}
14 output_0 = CompletionOutput(
15 index=0, text="child 0", token_ids=[], cumulative_logprob=None, logprobs=None
@@ -17,51 +18,31 @@ def test_parent_request_to_output_stream() -> None:
18 index=1, text="child 1", token_ids=[], cumulative_logprob=None, logprobs=None
19 )
20 # Request not finished
20 - assert ("parent_id", [output_0], False) == parent_request.get_outputs(
21 - "child_id_0", output_0
22 - )
23 - assert ("parent_id", [output_1], False) == parent_request.get_outputs(
24 - "child_id_1", output_1
25 - )
26 - assert ("parent_id", [output_0], False) == parent_request.get_outputs(
27 - "child_id_0", output_0
28 - )
29 - assert ("parent_id", [output_1], False) == parent_request.get_outputs(
30 - "child_id_1", output_1
31 - )
21 + assert ([output_0], False) == parent_request.get_outputs("child_id_0", output_0)
22 + assert ([output_1], False) == parent_request.get_outputs("child_id_1", output_1)
23 + assert ([output_0], False) == parent_request.get_outputs("child_id_0", output_0)
24 + assert ([output_1], False) == parent_request.get_outputs("child_id_1", output_1)
25
26 # output_1 finished
27 output_1.finish_reason = "ended"
35 - assert ("parent_id", [output_0], False) == parent_request.get_outputs(
36 - "child_id_0", output_0
37 - )
38 - assert ("parent_id", [output_1], False) == parent_request.get_outputs(
39 - "child_id_1", output_1
40 - )
28 + assert ([output_0], False) == parent_request.get_outputs("child_id_0", output_0)
29 + assert ([output_1], False) == parent_request.get_outputs("child_id_1", output_1)
30 # Finished output_1 had already returned, DO NOT returned again
42 - assert ("parent_id", [output_0], False) == parent_request.get_outputs(
43 - "child_id_0", output_0
44 - )
45 - assert parent_request.get_outputs("child_id_1", output_1) == (
46 - "parent_id",
47 - [],
48 - False,
49 - )
31 + assert ([output_0], False) == parent_request.get_outputs("child_id_0", output_0)
32 + assert parent_request.get_outputs("child_id_1", output_1) == ([], False)
33
34 # output_0 finished
35 output_0.finish_reason = "ended"
53 - assert ("parent_id", [output_0], True) == parent_request.get_outputs(
54 - "child_id_0", output_0
55 - )
56 - assert parent_request.get_outputs("child_id_1", output_1) == ("parent_id", [], True)
36 + assert ([output_0], True) == parent_request.get_outputs("child_id_0", output_0)
37 + assert parent_request.get_outputs("child_id_1", output_1) == ([], True)
38 # Finished output_0 had already returned, DO NOT returned again
58 - assert parent_request.get_outputs("child_id_0", output_0) == ("parent_id", [], True)
59 - assert parent_request.get_outputs("child_id_1", output_1) == ("parent_id", [], True)
39 + assert parent_request.get_outputs("child_id_0", output_0) == ([], True)
40 + assert parent_request.get_outputs("child_id_1", output_1) == ([], True)
41
42
43 def test_parent_request_to_output_final_only() -> None:
44 parent_request = ParentRequest(
64 - "parent_id", SamplingParams(n=2, output_kind=RequestOutputKind.FINAL_ONLY)
45 + make_request(SamplingParams(n=2, output_kind=RequestOutputKind.FINAL_ONLY))
46 )
47 parent_request.child_requests = {"child_id_0", "child_id_1"}
48 output_0 = CompletionOutput(
@@ -71,33 +52,33 @@ def test_parent_request_to_output_final_only() -> None:
52 index=1, text="child 1", token_ids=[], cumulative_logprob=None, logprobs=None
53 )
54 # Request not finished, return nothing
74 - assert parent_request.get_outputs("child_id_0", output_0) == (
75 - "parent_id",
76 - [],
77 - False,
78 - )
79 - assert parent_request.get_outputs("child_id_1", output_1) == (
80 - "parent_id",
81 - [],
82 - False,
83 - )
55 + assert parent_request.get_outputs("child_id_0", output_0) == ([], False)
56 + assert parent_request.get_outputs("child_id_1", output_1) == ([], False)
57 # output_1 finished, but outputs won't be returned until all child requests finished
58 output_1.finish_reason = "ended"
86 - assert parent_request.get_outputs("child_id_0", output_0) == (
87 - "parent_id",
88 - [],
89 - False,
90 - )
91 - assert parent_request.get_outputs("child_id_1", output_1) == (
92 - "parent_id",
93 - [],
94 - False,
95 - )
59 + assert parent_request.get_outputs("child_id_0", output_0) == ([], False)
60 + assert parent_request.get_outputs("child_id_1", output_1) == ([], False)
61 # output_0 finished, as all child requests finished, the output would be returned
62 output_0.finish_reason = "ended"
98 - assert ("parent_id", [output_0, output_1], True) == parent_request.get_outputs(
63 + assert ([output_0, output_1], True) == parent_request.get_outputs(
64 "child_id_0", output_0
65 )
101 - assert ("parent_id", [output_0, output_1], True) == parent_request.get_outputs(
66 + assert ([output_0, output_1], True) == parent_request.get_outputs(
67 "child_id_1", output_1
68 )
69 +
70 +
71 +def make_request(sampling_params: SamplingParams) -> EngineCoreRequest:
72 + return EngineCoreRequest(
73 + request_id="parent_id",
74 + external_req_id="ext_parent_id",
75 + prompt_token_ids=None,
76 + mm_features=None,
77 + sampling_params=sampling_params,
78 + pooling_params=None,
79 + eos_token_id=None,
80 + arrival_time=0.0,
81 + lora_request=None,
82 + cache_salt=None,
83 + data_parallel_rank=None,
84 + )
tests/v1/engine/test_process_multi_modal_uuids.py
+15 -5
@@ -6,6 +6,7 @@ import pytest
6 from vllm.assets.image import ImageAsset
7 from vllm.assets.video import VideoAsset
8 from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig
9 +from vllm.multimodal import MultiModalUUIDDict
10 from vllm.sampling_params import SamplingParams
11 from vllm.v1.engine import input_processor as input_processor_mod
12 from vllm.v1.engine.input_processor import InputProcessor
@@ -166,7 +167,7 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
167 monkeypatch, mm_cache_gb=0.0, enable_prefix_caching=False
168 )
169
169 - captured: dict[str, object] = {}
170 + captured: dict[str, MultiModalUUIDDict] = {}
171
172 def fake_preprocess(
173 prompt, *, tokenization_kwargs=None, lora_request=None, mm_uuids=None
@@ -196,7 +197,16 @@ def test_multi_modal_uuids_ignored_when_caching_disabled(monkeypatch):
197 )
198
199 # Expect request-id-based overrides are passed through
199 - assert captured["mm_uuids"] == {
200 - "image": [f"{request_id}-image-0", f"{request_id}-image-1"],
201 - "video": [f"{request_id}-video-0"],
202 - }
200 + mm_uuids = captured["mm_uuids"]
201 + assert set(mm_uuids.keys()) == {"image", "video"}
202 + assert len(mm_uuids["image"]) == 2
203 + assert len(mm_uuids["video"]) == 1
204 + assert mm_uuids["image"][0].startswith(f"{request_id}-image-") and mm_uuids[
205 + "image"
206 + ][0].endswith("-0")
207 + assert mm_uuids["image"][1].startswith(f"{request_id}-image-") and mm_uuids[
208 + "image"
209 + ][1].endswith("-1")
210 + assert mm_uuids["video"][0].startswith(f"{request_id}-video-") and mm_uuids[
211 + "video"
212 + ][0].endswith("-0")
tests/v1/engine/utils.py
+7 -1
@@ -343,6 +343,7 @@ class MockEngineCore:
343 eos_token_id: int | None = None,
344 stop_token_ids: list[int] | None = None,
345 ignore_eos: bool = False,
346 + request_ids: list[str] | None = None,
347 ) -> None:
348 self.num_requests = len(tokens_list)
349 self.tokens_list = tokens_list
@@ -355,6 +356,11 @@ class MockEngineCore:
356 self.eos_token_id = eos_token_id
357 self.stop_token_ids = stop_token_ids
358 self.ignore_eos = ignore_eos
359 + self.request_ids = (
360 + request_ids
361 + if request_ids is not None
362 + else [f"request-{i}" for i in range(self.num_requests)]
363 + )
364
365 def get_outputs(self) -> list[EngineCoreOutput]:
366 do_logprobs = self.do_logprobs
@@ -386,7 +392,7 @@ class MockEngineCore:
392 prompt_logprobs = None
393 new_token_id = token_ids[token_idx]
394 output = EngineCoreOutput(
389 - request_id=f"request-{req_idx}",
395 + request_id=self.request_ids[req_idx],
396 new_token_ids=[new_token_id],
397 new_logprobs=logprobs,
398 new_prompt_logprobs_tensors=prompt_logprobs,
tests/v1/kv_connector/unit/test_nixl_connector.py
+35 -6
@@ -41,10 +41,13 @@ from vllm.distributed.kv_transfer.kv_transfer_state import (
41 has_kv_transfer_group,
42 )
43 from vllm.forward_context import ForwardContext
44 +from vllm.outputs import RequestOutput
45 from vllm.platforms import current_platform
46 from vllm.platforms.interface import Platform
47 from vllm.sampling_params import SamplingParams
48 from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
49 +from vllm.v1.engine import EngineCoreRequest
50 +from vllm.v1.engine.output_processor import OutputProcessor
51 from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
52 from vllm.v1.request import RequestStatus
53
@@ -1265,6 +1268,22 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
1268 run_test_and_cleanup()
1269
1270
1271 +class RequestIdMapper:
1272 + """Helper class to map external request IDs to internal request IDs."""
1273 +
1274 + def __init__(self, output_processor: OutputProcessor):
1275 + self.req_id_mapping: dict[str, str] = {}
1276 + self.original_add_request = output_processor.add_request
1277 + output_processor.add_request = self._add_request
1278 +
1279 + def _add_request(self, request: EngineCoreRequest, *args, **kwargs):
1280 + self.req_id_mapping[request.external_req_id] = request.request_id
1281 + return self.original_add_request(request, *args, **kwargs)
1282 +
1283 + def __call__(self, external_req_id: str) -> str:
1284 + return self.req_id_mapping[external_req_id]
1285 +
1286 +
1287 def _run_abort_timeout_test(llm: LLM, timeout: int):
1288 """Helper function to run the abort timeout test logic."""
1289 remote_prefill_opts = {
@@ -1286,24 +1305,34 @@ def _run_abort_timeout_test(llm: LLM, timeout: int):
1305 0
1306 ].req_to_blocks
1307
1308 + id_mapper = RequestIdMapper(llm.llm_engine.output_processor)
1309 +
1310 + def req_id(outputs: list[RequestOutput]) -> str:
1311 + assert len(outputs) == 1
1312 + return id_mapper(outputs[0].request_id)
1313 +
1314 padding = "Just making this request a little longer so that we're sure "
1315 "we're not hitting the small-request lower bound beneath which we don't "
1316 "actually trigger the whole kv transfer, but rather just recompute the "
1317 "blocks on D."
1293 - _ = llm.generate([f"What is the capital of Japan? {padding}"], sampling_params)
1318 + req0_id = req_id(
1319 + llm.generate([f"What is the capital of Japan? {padding}"], sampling_params)
1320 + )
1321
1322 # Request finished but not freed
1296 - assert "0" in scheduler.finished_req_ids and "0" in req_to_blocks
1323 + assert req0_id in scheduler.finished_req_ids and req0_id in req_to_blocks
1324 # Some other request, 0 still not freed
1298 - _ = llm.generate([f"What is the capital of Italy? {padding}"], sampling_params)
1299 - assert "0" in req_to_blocks
1300 - assert "1" in scheduler.finished_req_ids and "1" in req_to_blocks
1325 + req1_id = req_id(
1326 + llm.generate([f"What is the capital of Italy? {padding}"], sampling_params)
1327 + )
1328 + assert req0_id in req_to_blocks
1329 + assert req1_id in scheduler.finished_req_ids and req1_id in req_to_blocks
1330
1331 # Wait for timeout and trigger another scheduler loop
1332 time.sleep(timeout)
1333 _ = llm.generate([f"What is the capital of France? {padding}"], sampling_params)
1334 # Request-0 times out and is cleared!
1306 - assert "0" not in req_to_blocks
1335 + assert req0_id not in req_to_blocks
1336 # Need to shutdown the background thread to release NIXL side channel port
1337 llm.llm_engine.engine_core.shutdown()
1338
vllm/entrypoints/llm.py
+2 -2
@@ -1621,7 +1621,7 @@ class LLM:
1621 added_request_ids.append(request_id)
1622 except Exception as e:
1623 if added_request_ids:
1624 - self.llm_engine.abort_request(added_request_ids)
1624 + self.llm_engine.abort_request(added_request_ids, internal=True)
1625 raise e
1626
1627 def _validate_mm_data_and_uuids(
@@ -1731,7 +1731,7 @@ class LLM:
1731 priority=priority,
1732 prompt_text=prompt_text,
1733 )
1734 - return request_id
1734 + return engine_request.request_id
1735
1736 def _run_engine(
1737 self, *, use_tqdm: bool | Callable[..., tqdm] = True
vllm/v1/engine/__init__.py
+6
@@ -75,6 +75,12 @@ class EngineCoreRequest(
75
76 trace_headers: Mapping[str, str] | None = None
77
78 + # The user-provided request ID. This field is set internally,
79 + # copied from the provided request_id that's originally assigned
80 + # to the request_id field, see InputProcessor.assign_request_id().
81 + # Used in outputs and to support abort(req_id, internal=False).
82 + external_req_id: str | None = None
83 +
84 @property
85 def params(self) -> SamplingParams | PoolingParams:
86 """Return the processed params (sampling or pooling)."""
vllm/v1/engine/async_llm.py
+27 -11
@@ -290,12 +290,15 @@ class AsyncLLM(EngineClient):
290
291 is_pooling = isinstance(params, PoolingParams)
292
293 - # Create a new output collector for the request.
294 - queue = RequestOutputCollector(output_kind=params.output_kind)
295 -
293 # Convert Input --> Request.
294 if isinstance(prompt, EngineCoreRequest):
295 request = prompt
296 + if request_id != request.request_id:
297 + logger.warning_once(
298 + "AsyncLLM.add_request() was passed a request_id parameter that "
299 + "does not match the EngineCoreRequest.request_id attribute. The "
300 + "latter will be used, and the former will be ignored."
301 + )
302 else:
303 assert prompt_text is None
304 request = self.input_processor.process_inputs(
@@ -314,6 +317,11 @@ class AsyncLLM(EngineClient):
317 elif isinstance(prompt, Mapping):
318 prompt_text = cast(str | None, prompt.get("prompt"))
319
320 + self.input_processor.assign_request_id(request)
321 +
322 + # Create a new output collector for the request.
323 + queue = RequestOutputCollector(params.output_kind, request.request_id)
324 +
325 # Use cloned params that may have been updated in process_inputs()
326 params = request.params
327
@@ -325,7 +333,7 @@ class AsyncLLM(EngineClient):
333 assert isinstance(parent_params, SamplingParams)
334
335 # Fan out child requests (for n>1).
328 - parent_request = ParentRequest(request_id, parent_params)
336 + parent_request = ParentRequest(request)
337 for idx in range(parent_params.n):
338 request_id, child_params = parent_request.get_child_info(idx)
339 child_request = request if idx == parent_params.n - 1 else copy(request)
@@ -396,6 +404,7 @@ class AsyncLLM(EngineClient):
404 "prompt logprobs"
405 )
406
407 + q: RequestOutputCollector | None = None
408 try:
409 # We start the output_handler on the first call to generate() so
410 # we can call __init__ before the event loop, which enables us
@@ -446,7 +455,8 @@ class AsyncLLM(EngineClient):
455 # is cancelled or the generator is garbage collected. So,
456 # we abort the request if we end up here.
457 except (asyncio.CancelledError, GeneratorExit):
449 - await self.abort(request_id)
458 + if q is not None:
459 + await self.abort(q.request_id, internal=True)
460 if self.log_requests:
461 logger.info("Request %s aborted.", request_id)
462 raise
@@ -465,7 +475,8 @@ class AsyncLLM(EngineClient):
475
476 # Unexpected error in the generate() task (possibly recoverable).
477 except Exception as e:
468 - await self.abort(request_id)
478 + if q is not None:
479 + await self.abort(q.request_id, internal=True)
480 if self.log_requests:
481 logger.info("Request %s failed.", request_id)
482 raise EngineGenerateError() from e
@@ -541,13 +552,15 @@ class AsyncLLM(EngineClient):
552
553 self.output_handler = asyncio.create_task(output_handler())
554
544 - async def abort(self, request_id: str | Iterable[str]) -> None:
555 + async def abort(
556 + self, request_id: str | Iterable[str], internal: bool = False
557 + ) -> None:
558 """Abort RequestId in OutputProcessor and EngineCore."""
559
560 request_ids = (
561 (request_id,) if isinstance(request_id, str) else as_list(request_id)
562 )
550 - all_request_ids = self.output_processor.abort_requests(request_ids)
563 + all_request_ids = self.output_processor.abort_requests(request_ids, internal)
564 await self.engine_core.abort_requests_async(all_request_ids)
565
566 if self.log_requests:
@@ -581,7 +594,7 @@ class AsyncLLM(EngineClient):
594 if not wait_for_inflight_requests:
595 request_ids = list(self.output_processor.request_states.keys())
596 if request_ids:
584 - await self.abort(request_ids)
597 + await self.abort(request_ids, internal=True)
598
599 # Wait for running requests to drain before clearing cache.
600 if self.output_processor.has_unfinished_requests():
@@ -633,6 +646,7 @@ class AsyncLLM(EngineClient):
646 TODO: Remove truncate_prompt_tokens in v0.15.
647 """
648
649 + q: RequestOutputCollector | None = None
650 try:
651 # We start the output_handler on the first call to generate() so
652 # we can call __init__ before the event loop, which enables us
@@ -687,7 +701,8 @@ class AsyncLLM(EngineClient):
701 # If the request is disconnected by the client, generate()
702 # is cancelled. So, we abort the request if we end up here.
703 except asyncio.CancelledError:
690 - await self.abort(request_id)
704 + if q is not None:
705 + await self.abort(q.request_id, internal=True)
706 if self.log_requests:
707 logger.info("Request %s aborted.", request_id)
708 raise
@@ -706,7 +721,8 @@ class AsyncLLM(EngineClient):
721
722 # Unexpected error in the generate() task (possibly recoverable).
723 except Exception as e:
709 - await self.abort(request_id)
724 + if q is not None:
725 + await self.abort(q.request_id, internal=True)
726 if self.log_requests:
727 logger.info("Request %s failed.", request_id)
728 raise EngineGenerateError() from e
vllm/v1/engine/input_processor.py
+14 -1
@@ -21,7 +21,7 @@ from vllm.pooling_params import PoolingParams
21 from vllm.sampling_params import SamplingParams
22 from vllm.tokenizers import TokenizerLike
23 from vllm.tokenizers.mistral import MistralTokenizer
24 -from vllm.utils import length_from_prompt_token_ids_or_embeds
24 +from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid
25 from vllm.v1.engine import EngineCoreRequest
26 from vllm.v1.metrics.stats import MultiModalCacheStats
27 from vllm.v1.structured_output.backend_guidance import (
@@ -406,6 +406,19 @@ class InputProcessor:
406 mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]
407 return mm_uuids
408
409 + @staticmethod
410 + def assign_request_id(request: EngineCoreRequest):
411 + """Replace the externally supplied request ID with an internal request ID
412 + that adds 8 random characters in order to ensure uniquness.
413 + """
414 + if request.external_req_id is not None:
415 + raise ValueError(
416 + "The external_req_id field should not be set on EngineCoreRequests"
417 + " passed to vLLM; use the request_id field."
418 + )
419 + request.external_req_id = request.request_id
420 + request.request_id = f"{request.external_req_id}-{random_uuid():.8}"
421 +
422 def process_inputs(
423 self,
424 request_id: str,
vllm/v1/engine/llm_engine.py
+11 -3
@@ -213,10 +213,10 @@ class LLMEngine:
213 def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
214 return self.engine_core.get_supported_tasks()
215
216 - def abort_request(self, request_ids: list[str]) -> None:
216 + def abort_request(self, request_ids: list[str], internal: bool = False) -> None:
217 """Remove request_ids from EngineCore and Detokenizer."""
218
219 - request_ids = self.output_processor.abort_requests(request_ids)
219 + request_ids = self.output_processor.abort_requests(request_ids, internal)
220 self.engine_core.abort_requests(request_ids)
221
222 def add_request(
@@ -238,6 +238,12 @@ class LLMEngine:
238 # Process raw inputs into the request.
239 if isinstance(prompt, EngineCoreRequest):
240 request = prompt
241 + if request_id != request.request_id:
242 + logger.warning_once(
243 + "AsyncLLM.add_request() was passed a request_id parameter that "
244 + "does not match the EngineCoreRequest.request_id attribute. The "
245 + "latter will be used, and the former will be ignored."
246 + )
247 else:
248 assert prompt_text is None
249 request = self.input_processor.process_inputs(
@@ -255,6 +261,8 @@ class LLMEngine:
261 elif isinstance(prompt, Mapping):
262 prompt_text = cast(str | None, prompt.get("prompt"))
263
264 + self.input_processor.assign_request_id(request)
265 +
266 # Use cloned params that may have been updated in process_inputs()
267 params = request.params
268
@@ -268,7 +276,7 @@ class LLMEngine:
276 return
277
278 # Fan out child requests (for n>1).
271 - parent_req = ParentRequest(request_id, params)
279 + parent_req = ParentRequest(request)
280 for idx in range(n):
281 request_id, child_params = parent_req.get_child_info(idx)
282 child_request = request if idx == n - 1 else copy(request)
vllm/v1/engine/output_processor.py
+66 -17
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import asyncio
5 +from collections import defaultdict
6 from collections.abc import Iterable
7 from dataclasses import dataclass
8 from typing import Any, cast
@@ -40,8 +41,9 @@ class RequestOutputCollector:
41 producer gets ahead of the consumer.
42 """
43
43 - def __init__(self, output_kind: RequestOutputKind):
44 + def __init__(self, output_kind: RequestOutputKind, request_id: str):
45 self.aggregate = output_kind == RequestOutputKind.DELTA
46 + self.request_id = request_id
47 self.output: RequestOutput | PoolingRequestOutput | Exception | None = None
48 self.ready = asyncio.Event()
49
@@ -92,6 +94,7 @@ class RequestState:
94 def __init__(
95 self,
96 request_id: str,
97 + external_req_id: str,
98 parent_req: ParentRequest | None,
99 request_index: int,
100 lora_request: LoRARequest | None,
@@ -111,6 +114,7 @@ class RequestState:
114 temperature: float | None = None,
115 ):
116 self.request_id = request_id
117 + self.external_req_id = external_req_id
118 self.parent_req = parent_req
119 self.request_index = request_index
120 self.lora_request = lora_request
@@ -176,8 +180,10 @@ class RequestState:
180 assert request.pooling_params is not None
181 output_kind = request.pooling_params.output_kind
182
183 + assert request.external_req_id is not None
184 return cls(
185 request_id=request.request_id,
186 + external_req_id=request.external_req_id,
187 parent_req=parent_req,
188 request_index=request_index,
189 lora_request=request.lora_request,
@@ -235,10 +241,13 @@ class RequestState:
241 ]
242 self.sent_tokens_offset = len(self.detokenizer.output_token_ids)
243
238 - request_id = self.request_id
244 + external_req_id = self.external_req_id
245 +
246 if pooling_output is not None:
247 return self._new_request_output(
241 - request_id, [self._new_pooling_output(pooling_output)], finished
248 + external_req_id,
249 + [self._new_pooling_output(pooling_output)],
250 + finished,
251 )
252
253 output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
@@ -246,19 +255,18 @@ class RequestState:
255 if self.parent_req is None:
256 outputs = [output]
257 else:
249 - request_id, outputs, finished = self.parent_req.get_outputs(
250 - request_id, output
251 - )
258 + outputs, finished = self.parent_req.get_outputs(self.request_id, output)
259 if not outputs:
260 return None
261 + external_req_id = self.parent_req.external_req_id
262
263 return self._new_request_output(
256 - request_id, outputs, finished, kv_transfer_params
264 + external_req_id, outputs, finished, kv_transfer_params
265 )
266
267 def _new_request_output(
268 self,
261 - request_id: str,
269 + external_req_id: str,
270 outputs: list[CompletionOutput] | list[PoolingOutput],
271 finished: bool,
272 kv_transfer_params: dict[str, Any] | None = None,
@@ -269,7 +277,7 @@ class RequestState:
277 # Prompt embeddings are currently not supported by pooling requests.
278 assert self.prompt_token_ids is not None
279 return PoolingRequestOutput(
272 - request_id=request_id,
280 + request_id=external_req_id,
281 outputs=first_output,
282 num_cached_tokens=self.num_cached_tokens,
283 prompt_token_ids=self.prompt_token_ids,
@@ -288,7 +296,7 @@ class RequestState:
296 prompt_token_ids = [0] * len(self.prompt_embeds)
297
298 return RequestOutput(
291 - request_id=request_id,
299 + request_id=external_req_id, # request_id is what was provided externally
300 lora_request=self.lora_request,
301 prompt=self.prompt,
302 prompt_token_ids=prompt_token_ids,
@@ -352,6 +360,7 @@ class OutputProcessor:
360 self.stream_interval = stream_interval
361 self.request_states: dict[str, RequestState] = {}
362 self.parent_requests: dict[str, ParentRequest] = {}
363 + self.external_req_ids: defaultdict[str, list[str]] = defaultdict(list)
364 self.lora_states = LoRARequestStates(log_stats)
365 self.tracer: Tracer | None = None
366 self._requests_drained = asyncio.Event()
@@ -375,12 +384,41 @@ class OutputProcessor:
384 assert state.queue is not None
385 state.queue.put(e)
386
378 - def abort_requests(
379 - self,
380 - request_ids: Iterable[str],
381 - ) -> list[str]:
382 - request_ids_to_abort = []
387 + def abort_requests(self, request_ids: Iterable[str], internal: bool) -> list[str]:
388 + """Abort a list of requests.
389 +
390 + The request_ids may be either external request IDs (those passed to
391 + InputProcessor.process_inputs()) or internal request IDs (those randomly
392 + generated when creating the EngineCoreRequest).
393 +
394 + If an external request ID is provided, and that external request ID
395 + was used for multiple requests, all requests associated with that external
396 + request ID are aborted.
397 +
398 + In the case of parallel sampling, a request ID may be used to identify
399 + a parent request, in which case the associated child requests are aborted
400 + also.
401 + """
402 +
403 + internal_req_ids = []
404 for request_id in request_ids:
405 + if internal:
406 + # Internal ID - this may be a parent request
407 + internal_req_ids.append(request_id)
408 +
409 + # Remove internal ID from the external->internal mapping
410 + if req_state := self.request_states.get(request_id):
411 + external_req_id = req_state.external_req_id
412 + internal_ids = self.external_req_ids[external_req_id]
413 + internal_ids.remove(request_id)
414 + if not internal_ids:
415 + del self.external_req_ids[external_req_id]
416 + elif internal_ids := self.external_req_ids.pop(request_id, []):
417 + # External ID - abort all requests in the external->internal mapping
418 + internal_req_ids.extend(internal_ids)
419 +
420 + request_ids_to_abort = []
421 + for request_id in internal_req_ids:
422 req_state = self.request_states.pop(request_id, None)
423 if req_state is not None:
424 self.lora_states.request_finished(request_id, req_state.lora_name)
@@ -404,7 +442,7 @@ class OutputProcessor:
442 # Abort children prior to removing the parent.
443 if parent.child_requests:
444 child_reqs = list(parent.child_requests)
407 - child_reqs = self.abort_requests(child_reqs)
445 + child_reqs = self.abort_requests(child_reqs, internal=True)
446 request_ids_to_abort.extend(child_reqs)
447 self.parent_requests.pop(request_id, None)
448 if not self.request_states:
@@ -439,6 +477,9 @@ class OutputProcessor:
477 if parent_req:
478 self.parent_requests[parent_req.request_id] = parent_req
479
480 + # Track the external_req_id -> [internal_req_id, ...] mapping
481 + self.external_req_ids[req_state.external_req_id].append(request_id)
482 +
483 def process_outputs(
484 self,
485 engine_core_outputs: list[EngineCoreOutput],
@@ -522,6 +563,12 @@ class OutputProcessor:
563 # Free completed requests.
564 if finish_reason is not None:
565 self.request_states.pop(req_id)
566 +
567 + internal_ids = self.external_req_ids[req_state.external_req_id]
568 + internal_ids.remove(req_id)
569 + if not internal_ids:
570 + del self.external_req_ids[req_state.external_req_id]
571 +
572 # Remove parent request if applicable.
573 parent_req = req_state.parent_req
574 if parent_req and not parent_req.child_requests:
@@ -597,7 +644,9 @@ class OutputProcessor:
644 )
645
646 # meta
600 - span.set_attribute(SpanAttributes.GEN_AI_REQUEST_ID, req_state.request_id)
647 + span.set_attribute(
648 + SpanAttributes.GEN_AI_REQUEST_ID, req_state.external_req_id
649 + )
650 if req_state.top_p:
651 span.set_attribute(SpanAttributes.GEN_AI_REQUEST_TOP_P, req_state.top_p)
652 if req_state.max_tokens_param:
vllm/v1/engine/parallel_sampling.py
+9 -4
@@ -6,6 +6,7 @@ from typing import Optional, cast
6
7 from vllm.outputs import CompletionOutput
8 from vllm.sampling_params import RequestOutputKind, SamplingParams
9 +from vllm.v1.engine import EngineCoreRequest
10 from vllm.v1.metrics.stats import IterationStats
11
12
@@ -17,6 +18,7 @@ class ParentRequest:
18 """
19
20 request_id: str
21 + external_req_id: str
22 sampling_params: SamplingParams
23
24 # To track the completion of child requests
@@ -31,8 +33,11 @@ class ParentRequest:
33 # To efficiently obtain child sampling params
34 cached_child_sampling_params: SamplingParams | None
35
34 - def __init__(self, request_id: str, sampling_params: SamplingParams) -> None:
35 - self.request_id = request_id
36 + def __init__(self, request: EngineCoreRequest) -> None:
37 + assert request.external_req_id is not None
38 + sampling_params = request.params
39 + self.request_id = request.request_id
40 + self.external_req_id = request.external_req_id
41 self.sampling_params = sampling_params
42
43 self.child_requests = set()
@@ -96,7 +101,7 @@ class ParentRequest:
101 self,
102 child_request_id: str,
103 completion_output: CompletionOutput,
99 - ) -> tuple[str, list[CompletionOutput], bool]:
104 + ) -> tuple[list[CompletionOutput], bool]:
105 already_finished_and_returned: bool = False
106 if completion_output.finished():
107 if child_request_id in self.child_requests:
@@ -118,7 +123,7 @@ class ParentRequest:
123 outputs = [] if self.child_requests else self.output_aggregator
124
125 finished = not self.child_requests
121 - return self.request_id, outputs, finished
126 + return outputs, finished
127
128 def observe_num_generation_tokens(self, num_generation_tokens: int):
129 self.max_num_generation_tokens = max(