@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da6ea29f7a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 18:30 UTC

[V1] Avoid redundant input processing in n>1 case (#14985)

Signed-off-by: Nick Hill <nhill@redhat.com>

Nick Hill committed Mar 20, 2025 at 22:24 UTC da6ea29f7a60b75ac4fe7b0ae5204ba96298011c
13 files changed +82 -142
tests/lora/test_tokenizer_group.py
+2 -4
@@ -24,12 +24,10 @@ async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
24 )
25 lora_request = LoRARequest("1", 1, sql_lora_files)
26 assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
27 - request_id="request_id", prompt="prompt", lora_request=lora_request)
27 + prompt="prompt", lora_request=lora_request)
28 assert reference_tokenizer.encode(
29 "prompt") == await tokenizer_group.encode_async(
30 - request_id="request_id",
31 - prompt="prompt",
32 - lora_request=lora_request)
30 + prompt="prompt", lora_request=lora_request)
31 assert isinstance(tokenizer_group.get_lora_tokenizer(None),
32 PreTrainedTokenizerBase)
33 assert tokenizer_group.get_lora_tokenizer(
tests/tokenization/test_tokenizer_group.py
+9 -18
@@ -41,10 +41,10 @@ async def test_tokenizer_group(tokenizer_group_type):
41 max_input_length=None,
42 )
43 assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(
44 - request_id="request_id", prompt="prompt", lora_request=None)
44 + prompt="prompt", lora_request=None)
45 assert reference_tokenizer.encode(
46 - "prompt") == await tokenizer_group.encode_async(
47 - request_id="request_id", prompt="prompt", lora_request=None)
46 + "prompt") == await tokenizer_group.encode_async(prompt="prompt",
47 + lora_request=None)
48 assert isinstance(tokenizer_group.get_lora_tokenizer(None),
49 PreTrainedTokenizerBase)
50 assert tokenizer_group.get_lora_tokenizer(
@@ -69,8 +69,7 @@ async def test_tokenizer_group_pool(tokenizer_group_type):
69 # and check that all requests are processed correctly.
70 num_requests = tokenizer_group_pool.pool_size * 5
71 requests = [
72 - tokenizer_group_pool.encode_async(request_id=str(i),
73 - prompt=f"prompt {i}",
72 + tokenizer_group_pool.encode_async(prompt=f"prompt {i}",
73 lora_request=None)
74 for i in range(num_requests)
75 ]
@@ -161,12 +160,8 @@ async def test_tokenizer_group_ray_pool_fault_tolerance(tokenizer_group_type):
160 fail_at[0] = 1000
161
162 # We should recover successfully.
164 - await tokenizer_group_pool.encode_async(request_id="1",
165 - prompt="prompt",
166 - lora_request=None)
167 - await tokenizer_group_pool.encode_async(request_id="1",
168 - prompt="prompt",
169 - lora_request=None)
163 + await tokenizer_group_pool.encode_async(prompt="prompt", lora_request=None)
164 + await tokenizer_group_pool.encode_async(prompt="prompt", lora_request=None)
165
166 # Check that we have a new actor
167 assert len(tokenizer_group_pool.tokenizer_actors) == len(tokenizer_actors)
@@ -184,8 +179,7 @@ async def test_tokenizer_group_ray_pool_fault_tolerance(tokenizer_group_type):
179
180 # We should fail after re-initialization.
181 with pytest.raises(RuntimeError):
187 - await tokenizer_group_pool.encode_async(request_id="1",
188 - prompt="prompt",
182 + await tokenizer_group_pool.encode_async(prompt="prompt",
183 lora_request=None)
184
185 # check_health should raise the same thing
@@ -206,11 +200,8 @@ async def test_tokenizer_group_ray_pool_fault_tolerance(tokenizer_group_type):
200
201 # Prompt too long error
202 with pytest.raises(ValueError):
209 - await tokenizer_group_pool.encode_async(request_id="1",
210 - prompt="prompt" * 100,
203 + await tokenizer_group_pool.encode_async(prompt="prompt" * 100,
204 lora_request=None)
212 - await tokenizer_group_pool.encode_async(request_id="1",
213 - prompt="prompt",
214 - lora_request=None)
205 + await tokenizer_group_pool.encode_async(prompt="prompt", lora_request=None)
206 # Actors should stay the same.
207 assert tokenizer_group_pool.tokenizer_actors == tokenizer_actors
vllm/engine/async_llm_engine.py
-1
@@ -492,7 +492,6 @@ class _AsyncLLMEngine(LLMEngine):
492
493 preprocessed_inputs = await self.input_preprocessor.preprocess_async(
494 prompt,
495 - request_id=request_id,
495 lora_request=lora_request,
496 prompt_adapter_request=prompt_adapter_request,
497 )
vllm/engine/llm_engine.py
-1
@@ -783,7 +783,6 @@ class LLMEngine:
783
784 preprocessed_inputs = self.input_preprocessor.preprocess(
785 prompt,
786 - request_id=request_id,
786 lora_request=lora_request,
787 prompt_adapter_request=prompt_adapter_request,
788 )
vllm/engine/protocol.py
+1 -4
@@ -81,10 +81,7 @@ class EngineClient(ABC):
81 if is_explicit_encoder_decoder_prompt(prompt):
82 raise NotImplementedError
83 else:
84 - processed_inputs = preprocessor._prompt_to_llm_inputs(
85 - prompt,
86 - request_id=request_id,
87 - )
84 + processed_inputs = preprocessor._prompt_to_llm_inputs(prompt)
85
86 prompt_token_ids = processed_inputs["prompt_token_ids"]
87 prompt_text = processed_inputs.get("prompt")
vllm/inputs/preprocess.py
+9 -54
@@ -182,7 +182,6 @@ class InputPreprocessor:
182 def _tokenize_prompt(
183 self,
184 prompt: str,
185 - request_id: str,
185 lora_request: Optional[LoRARequest],
186 ) -> list[int]:
187 """
@@ -202,15 +201,13 @@ class InputPreprocessor:
201 "do_lower_case", False)):
202 prompt = prompt.lower()
203
205 - return tokenizer.encode(request_id=request_id,
206 - prompt=prompt,
204 + return tokenizer.encode(prompt=prompt,
205 lora_request=lora_request,
206 add_special_tokens=add_special_tokens)
207
208 async def _tokenize_prompt_async(
209 self,
210 prompt: str,
213 - request_id: str,
211 lora_request: Optional[LoRARequest],
212 ) -> list[int]:
213 """Async version of :meth:`_tokenize_prompt`."""
@@ -222,7 +219,6 @@ class InputPreprocessor:
219 # appending an EOS token to the prompt which disrupts generation.
220 add_special_tokens = False
221 return await tokenizer.encode_async(
225 - request_id=request_id,
222 prompt=prompt,
223 lora_request=lora_request,
224 add_special_tokens=add_special_tokens)
@@ -309,7 +305,6 @@ class InputPreprocessor:
305 def _prompt_to_llm_inputs(
306 self,
307 prompt: SingletonPrompt,
312 - request_id: str,
308 lora_request: Optional[LoRARequest] = None,
309 return_mm_hashes: bool = False,
310 ) -> SingletonInputs:
@@ -318,7 +313,6 @@ class InputPreprocessor:
313
314 Arguments:
315
321 - * request_id
316 * prompt: single encoder or decoder input prompt
317 * lora_request: this is only valid for decoder prompts
318 * return_mm_hashes: whether to return multimodal hashes
@@ -333,7 +327,6 @@ class InputPreprocessor:
327 prompt_text = parsed["content"]
328 prompt_token_ids = self._tokenize_prompt(
329 prompt_text,
336 - request_id=request_id,
330 lora_request=lora_request,
331 )
332
@@ -384,7 +377,6 @@ class InputPreprocessor:
377
378 prompt_token_ids = self._tokenize_prompt(
379 prompt_text,
387 - request_id=request_id,
380 lora_request=lora_request,
381 )
382
@@ -400,7 +392,6 @@ class InputPreprocessor:
392 async def _prompt_to_llm_inputs_async(
393 self,
394 prompt: SingletonPrompt,
403 - request_id: str,
395 lora_request: Optional[LoRARequest] = None,
396 return_mm_hashes: bool = False,
397 ) -> SingletonInputs:
@@ -411,7 +402,6 @@ class InputPreprocessor:
402 prompt_text = parsed["content"]
403 prompt_token_ids = await self._tokenize_prompt_async(
404 prompt_text,
414 - request_id=request_id,
405 lora_request=lora_request,
406 )
407
@@ -460,7 +450,6 @@ class InputPreprocessor:
450
451 prompt_token_ids = await self._tokenize_prompt_async(
452 prompt_text,
463 - request_id=request_id,
453 lora_request=lora_request,
454 )
455
@@ -560,7 +549,6 @@ class InputPreprocessor:
549 def _process_encoder_decoder_prompt(
550 self,
551 prompt: PromptType,
563 - request_id: str,
552 ) -> EncoderDecoderInputs:
553 """
554 For encoder/decoder models only:
@@ -587,7 +575,6 @@ class InputPreprocessor:
575 Arguments:
576
577 * prompt: an input prompt
590 - * request_id
578
579 Returns:
580
@@ -598,16 +585,11 @@ class InputPreprocessor:
585
586 if is_explicit_encoder_decoder_prompt(prompt):
587 encoder_inputs = self._prompt_to_llm_inputs(
601 - prompt["encoder_prompt"],
602 - request_id=request_id,
603 - )
588 + prompt["encoder_prompt"])
589 if (decoder_input := prompt["decoder_prompt"]) is None:
590 decoder_inputs = None
591 else:
607 - decoder_inputs = self._prompt_to_llm_inputs(
608 - decoder_input,
609 - request_id=request_id,
610 - )
592 + decoder_inputs = self._prompt_to_llm_inputs(decoder_input)
593 # For multimodal model, override decoder prompt from processor
594 # with explicit decoder prompt.
595 if self.model_config.is_multimodal_model and (
@@ -616,10 +598,7 @@ class InputPreprocessor:
598 self._separate_enc_dec_inputs_from_mm_processor_outputs(
599 encoder_inputs, decoder_inputs))
600 else:
619 - inputs = self._prompt_to_llm_inputs(
620 - prompt,
621 - request_id=request_id,
622 - )
601 + inputs = self._prompt_to_llm_inputs(prompt)
602 if self.model_config.is_multimodal_model and (
603 self._can_process_multimodal()):
604 # Encoder-Decoder Multimodal model
@@ -636,7 +615,6 @@ class InputPreprocessor:
615 async def _process_encoder_decoder_prompt_async(
616 self,
617 prompt: PromptType,
639 - request_id: str,
618 ) -> EncoderDecoderInputs:
619 """Async version of :meth:`_process_encoder_decoder_prompt`."""
620 encoder_inputs: SingletonInputs
@@ -644,18 +622,13 @@ class InputPreprocessor:
622
623 if is_explicit_encoder_decoder_prompt(prompt):
624 encoder_task = self._prompt_to_llm_inputs_async(
647 - prompt["encoder_prompt"],
648 - request_id=request_id,
649 - )
625 + prompt["encoder_prompt"])
626
627 if (decoder_input := prompt["decoder_prompt"]) is None:
628 encoder_inputs = await encoder_task
629 decoder_inputs = None
630 else:
655 - decoder_task = self._prompt_to_llm_inputs_async(
656 - decoder_input,
657 - request_id=request_id,
658 - )
631 + decoder_task = self._prompt_to_llm_inputs_async(decoder_input)
632
633 encoder_inputs, decoder_inputs = await asyncio.gather(
634 encoder_task, decoder_task)
@@ -668,10 +641,7 @@ class InputPreprocessor:
641 self._separate_enc_dec_inputs_from_mm_processor_outputs(
642 encoder_inputs, decoder_inputs))
643 else:
671 - inputs = await self._prompt_to_llm_inputs_async(
672 - prompt,
673 - request_id=request_id,
674 - )
644 + inputs = await self._prompt_to_llm_inputs_async(prompt)
645 if self.model_config.is_multimodal_model and (
646 self._can_process_multimodal()):
647 # Encoder-Decoder Multimodal model
@@ -704,7 +674,6 @@ class InputPreprocessor:
674 def _process_decoder_only_prompt(
675 self,
676 prompt: SingletonPrompt,
707 - request_id: str,
677 lora_request: Optional[LoRARequest] = None,
678 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
679 return_mm_hashes: bool = False,
@@ -716,7 +685,6 @@ class InputPreprocessor:
685 Arguments:
686
687 * prompt: input prompt
719 - * request_id
688 * lora_request
689 * prompt_adapter_request
690 * return_mm_hashes
@@ -728,7 +696,6 @@ class InputPreprocessor:
696
697 prompt_comps = self._prompt_to_llm_inputs(
698 prompt,
731 - request_id=request_id,
699 lora_request=lora_request,
700 return_mm_hashes=return_mm_hashes,
701 )
@@ -741,7 +708,6 @@ class InputPreprocessor:
708 async def _process_decoder_only_prompt_async(
709 self,
710 prompt: SingletonPrompt,
744 - request_id: str,
711 lora_request: Optional[LoRARequest] = None,
712 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
713 return_mm_hashes: bool = False,
@@ -749,7 +715,6 @@ class InputPreprocessor:
715 """Async version of :meth:`_process_decoder_only_prompt`."""
716 prompt_comps = await self._prompt_to_llm_inputs_async(
717 prompt,
752 - request_id=request_id,
718 lora_request=lora_request,
719 return_mm_hashes=return_mm_hashes,
720 )
@@ -762,7 +727,6 @@ class InputPreprocessor:
727 def preprocess(
728 self,
729 prompt: PromptType,
765 - request_id: str,
730 lora_request: Optional[LoRARequest] = None,
731 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
732 return_mm_hashes: bool = False,
@@ -774,10 +738,7 @@ class InputPreprocessor:
738 "returned until they are supported on vLLM V1.")
739 # Encoder-decoder model requires special mapping of
740 # input prompts to encoder & decoder
777 - return self._process_encoder_decoder_prompt(
778 - prompt,
779 - request_id=request_id,
780 - )
741 + return self._process_encoder_decoder_prompt(prompt)
742
743 if is_explicit_encoder_decoder_prompt(prompt):
744 raise ValueError("Cannot pass encoder-decoder prompt "
@@ -786,7 +747,6 @@ class InputPreprocessor:
747 # Decoder-only operation
748 return self._process_decoder_only_prompt(
749 prompt,
789 - request_id=request_id,
750 lora_request=lora_request,
751 prompt_adapter_request=prompt_adapter_request,
752 return_mm_hashes=return_mm_hashes,
@@ -795,7 +755,6 @@ class InputPreprocessor:
755 async def preprocess_async(
756 self,
757 prompt: PromptType,
798 - request_id: str,
758 lora_request: Optional[LoRARequest] = None,
759 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
760 return_mm_hashes: bool = False,
@@ -807,10 +766,7 @@ class InputPreprocessor:
766 "returned until they are supported on vLLM V1.")
767 # Encoder-decoder model requires special mapping of
768 # input prompts to encoder & decoder
810 - return await self._process_encoder_decoder_prompt_async(
811 - prompt,
812 - request_id=request_id,
813 - )
769 + return await self._process_encoder_decoder_prompt_async(prompt)
770
771 if is_explicit_encoder_decoder_prompt(prompt):
772 raise ValueError("Cannot pass encoder-decoder prompt "
@@ -819,7 +775,6 @@ class InputPreprocessor:
775 # Decoder-only operation
776 return await self._process_decoder_only_prompt_async(
777 prompt,
822 - request_id=request_id,
778 lora_request=lora_request,
779 prompt_adapter_request=prompt_adapter_request,
780 return_mm_hashes=return_mm_hashes,
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
-2
@@ -33,7 +33,6 @@ class BaseTokenizerGroup(ABC):
33 @abstractmethod
34 def encode(self,
35 prompt: str,
36 - request_id: Optional[str] = None,
36 lora_request: Optional[LoRARequest] = None,
37 add_special_tokens: Optional[bool] = None) -> List[int]:
38 """Encode a prompt using the tokenizer group."""
@@ -43,7 +42,6 @@ class BaseTokenizerGroup(ABC):
42 async def encode_async(
43 self,
44 prompt: str,
46 - request_id: Optional[str] = None,
45 lora_request: Optional[LoRARequest] = None,
46 add_special_tokens: Optional[bool] = None) -> List[int]:
47 """Encode a prompt using the tokenizer group."""
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2 -8
@@ -113,7 +113,6 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
113
114 def encode(self,
115 prompt: str,
116 - request_id: Optional[str] = None,
116 lora_request: Optional[LoRARequest] = None,
117 add_special_tokens: Optional[bool] = None) -> List[int]:
118 """Encode a prompt using the tokenizer group.
@@ -133,8 +132,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
132 original_actor = actor
133 try:
134 ret = ray.get(
136 - actor.encode.remote(request_id=request_id,
137 - prompt=prompt,
135 + actor.encode.remote(prompt=prompt,
136 lora_request=lora_request,
137 add_special_tokens=add_special_tokens))
138 except ActorDiedError as e:
@@ -145,8 +143,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
143 actor = self._init_actor()
144 try:
145 ret = ray.get(
148 - actor.encode.remote(request_id=request_id,
149 - prompt=prompt,
146 + actor.encode.remote(prompt=prompt,
147 lora_request=lora_request,
148 add_special_tokens=add_special_tokens))
149 except ActorDiedError as e:
@@ -164,7 +161,6 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
161 async def encode_async(
162 self,
163 prompt: str,
167 - request_id: Optional[str] = None,
164 lora_request: Optional[LoRARequest] = None,
165 add_special_tokens: Optional[bool] = None) -> List[int]:
166 """Encode a prompt using the tokenizer group.
@@ -184,7 +180,6 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
180 original_actor = actor
181 try:
182 ret = await actor.encode.remote(
187 - request_id=request_id,
183 prompt=prompt,
184 lora_request=lora_request,
185 add_special_tokens=add_special_tokens)
@@ -196,7 +191,6 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
191 actor = self._init_actor()
192 try:
193 ret = await actor.encode.remote(
199 - request_id=request_id,
194 prompt=prompt,
195 lora_request=lora_request,
196 add_special_tokens=add_special_tokens)
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
-2
@@ -56,7 +56,6 @@ class TokenizerGroup(BaseTokenizerGroup):
56
57 def encode(self,
58 prompt: str,
59 - request_id: Optional[str] = None,
59 lora_request: Optional[LoRARequest] = None,
60 add_special_tokens: Optional[bool] = None) -> List[int]:
61 tokenizer = self.get_lora_tokenizer(lora_request)
@@ -69,7 +68,6 @@ class TokenizerGroup(BaseTokenizerGroup):
68 async def encode_async(
69 self,
70 prompt: str,
72 - request_id: Optional[str] = None,
71 lora_request: Optional[LoRARequest] = None,
72 add_special_tokens: Optional[bool] = None) -> List[int]:
73 tokenizer = await self.get_lora_tokenizer_async(lora_request)
vllm/v1/engine/async_llm.py
+32 -19
@@ -4,6 +4,7 @@ import asyncio
4 import logging
5 import os
6 from collections.abc import AsyncGenerator, Mapping
7 +from copy import copy
8 from typing import Optional, Union
9
10 import numpy as np
@@ -25,6 +26,7 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer
26 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
27 from vllm.usage.usage_lib import UsageContext
28 from vllm.utils import Device, cdiv, kill_process_tree
29 +from vllm.v1.engine import EngineCoreRequest
30 from vllm.v1.engine.core_client import EngineCoreClient
31 from vllm.v1.engine.output_processor import OutputProcessor
32 from vllm.v1.engine.parallel_sampling import ParentRequest
@@ -177,33 +179,44 @@ class AsyncLLM(EngineClient):
179 ) -> asyncio.Queue[RequestOutput]:
180 """Add new request to the AsyncLLM."""
181
180 - # 1) Create a new output queue for the request.
182 + # Create a new output queue for the request.
183 queue: asyncio.Queue[RequestOutput] = asyncio.Queue()
184
183 - # 2) Fan out child requests (for n>1)
184 - parent_req = ParentRequest.from_params(request_id, params)
185 + # Convert Input --> Request.
186 + request = self.processor.process_inputs(request_id, prompt, params,
187 + arrival_time, lora_request,
188 + trace_headers,
189 + prompt_adapter_request,
190 + priority)
191 +
192 n = params.n if isinstance(params, SamplingParams) else 1
186 - for idx in range(n):
187 - if parent_req is not None:
188 - request_id, params = parent_req.get_child_info(idx)
193
190 - # 3) Convert Input --> Request.
191 - request = self.processor.process_inputs(request_id, prompt, params,
192 - arrival_time, lora_request,
193 - trace_headers,
194 - prompt_adapter_request,
195 - priority)
194 + if n == 1:
195 + await self._add_request(request, None, 0, queue)
196 + return queue
197
197 - # 4) Add the request to OutputProcessor (this process).
198 - self.output_processor.add_request(request, parent_req, idx, queue)
198 + # Fan out child requests (for n>1).
199 + parent_request = ParentRequest(request_id, params)
200 + for idx in range(n):
201 + request_id, params = parent_request.get_child_info(idx)
202 + child_request = request if idx == n - 1 else copy(request)
203 + child_request.request_id = request_id
204 + child_request.sampling_params = params
205 + await self._add_request(child_request, parent_request, idx, queue)
206 + return queue
207
200 - # 5) Add the EngineCoreRequest to EngineCore (separate process).
201 - await self.engine_core.add_request_async(request)
208 + async def _add_request(self, request: EngineCoreRequest,
209 + parent_req: Optional[ParentRequest], index: int,
210 + queue: asyncio.Queue[RequestOutput]):
211
203 - if self.log_requests:
204 - logger.info("Added request %s.", request_id)
212 + # Add the request to OutputProcessor (this process).
213 + self.output_processor.add_request(request, parent_req, index, queue)
214
206 - return queue
215 + # Add the EngineCoreRequest to EngineCore (separate process).
216 + await self.engine_core.add_request_async(request)
217 +
218 + if self.log_requests:
219 + logger.info("Added request %s.", request.request_id)
220
221 # TODO: we should support multiple prompts in one call, as you
222 # can do with LLM.generate. So that for multi-prompt completion
vllm/v1/engine/llm_engine.py
+26 -16
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from collections.abc import Mapping
4 +from copy import copy
5 from typing import Optional, Union
6
7 from typing_extensions import TypeVar
@@ -179,25 +180,34 @@ class LLMEngine:
180 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
181 priority: int = 0,
182 ) -> None:
182 - # 1) Fan out child requests (for n>1)
183 - parent_req = ParentRequest.from_params(request_id, params)
184 - n = params.n if isinstance(params, SamplingParams) else 1
185 - for idx in range(n):
186 - if parent_req is not None:
187 - request_id, params = parent_req.get_child_info(idx)
188 -
189 - # 2) Process raw inputs into the request.
190 - request = self.processor.process_inputs(request_id, prompt, params,
191 - arrival_time, lora_request,
192 - trace_headers,
193 - prompt_adapter_request,
194 - priority)
183 + # Process raw inputs into the request.
184 + request = self.processor.process_inputs(request_id, prompt, params,
185 + arrival_time, lora_request,
186 + trace_headers,
187 + prompt_adapter_request,
188 + priority)
189
196 - # 3) Make a new RequestState and queue.
197 - self.output_processor.add_request(request, parent_req, idx)
190 + n = params.n if isinstance(params, SamplingParams) else 1
191
199 - # 3) Add the request to EngineCore.
192 + if n == 1:
193 + # Make a new RequestState and queue.
194 + self.output_processor.add_request(request, None, 0)
195 + # Add the request to EngineCore.
196 self.engine_core.add_request(request)
197 + return
198 +
199 + # Fan out child requests (for n>1).
200 + parent_req = ParentRequest(request_id, params)
201 + for idx in range(n):
202 + request_id, params = parent_req.get_child_info(idx)
203 + child_request = request if idx == n - 1 else copy(request)
204 + child_request.request_id = request_id
205 + child_request.sampling_params = params
206 +
207 + # Make a new RequestState and queue.
208 + self.output_processor.add_request(child_request, parent_req, idx)
209 + # Add the request to EngineCore.
210 + self.engine_core.add_request(child_request)
211
212 def step(self) -> list[RequestOutput]:
213
vllm/v1/engine/parallel_sampling.py
+1 -12
@@ -1,10 +1,9 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from copy import copy
4 -from typing import Optional, Union
4 +from typing import Optional
5
6 from vllm.outputs import CompletionOutput
7 -from vllm.pooling_params import PoolingParams
7 from vllm.sampling_params import RequestOutputKind, SamplingParams
8 from vllm.v1.metrics.stats import IterationStats
9
@@ -43,16 +42,6 @@ class ParentRequest:
42 self.max_num_generation_tokens = 0
43 self.cached_child_sampling_params = None
44
46 - @classmethod
47 - def from_params(
48 - cls,
49 - request_id: str,
50 - params: Union[SamplingParams, PoolingParams],
51 - ) -> Optional['ParentRequest']:
52 - if not isinstance(params, SamplingParams) or params.n == 1:
53 - return None
54 - return cls(request_id, params)
55 -
45 def _get_child_sampling_params(
46 self,
47 index: int,
vllm/v1/engine/processor.py
-1
@@ -173,7 +173,6 @@ class Processor:
173 # 3. Apply prompt adapter to prompt token ids if one exists.
174 processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
175 prompt,
176 - request_id=request_id,
176 lora_request=lora_request,
177 prompt_adapter_request=prompt_adapter_request,
178 return_mm_hashes=self.use_hash,