@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 168cab6bbf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend] API support for beam search (#9087)

Co-authored-by: youkaichao <youkaichao@126.com>

Brendan Wong committed Oct 5, 2024 at 23:39 UTC 168cab6bbfb733f97defc8c1aa13df90c5319f19
12 files changed +275 -68
benchmarks/benchmark_throughput.py
+8 -4
@@ -15,6 +15,7 @@ from vllm.engine.arg_utils import DEVICE_OPTIONS, AsyncEngineArgs, EngineArgs
15 from vllm.entrypoints.openai.api_server import (
16 build_async_engine_client_from_engine_args)
17 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
18 +from vllm.sampling_params import BeamSearchParams
19 from vllm.utils import FlexibleArgumentParser, merge_async_iterators
20
21
@@ -145,10 +146,13 @@ def run_vllm(
146 for prompt, input_len, _output_len in requests:
147 assert _output_len == output_len
148 start = time.perf_counter()
148 - llm.beam_search(prompts,
149 - beam_width=n,
150 - max_tokens=output_len,
151 - ignore_eos=True)
149 + llm.beam_search(
150 + prompts,
151 + BeamSearchParams(
152 + beam_width=n,
153 + max_tokens=output_len,
154 + ignore_eos=True,
155 + ))
156 end = time.perf_counter()
157 return end - start
158
tests/conftest.py
+4 -1
@@ -35,6 +35,7 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
35 to_enc_dec_tuple_list, zip_enc_dec_prompts)
36 from vllm.logger import init_logger
37 from vllm.outputs import RequestOutput
38 +from vllm.sampling_params import BeamSearchParams
39 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
40 identity, is_cpu)
41
@@ -812,7 +813,9 @@ class VllmRunner:
813 beam_width: int,
814 max_tokens: int,
815 ) -> List[Tuple[List[List[int]], List[str]]]:
815 - outputs = self.model.beam_search(prompts, beam_width, max_tokens)
816 + outputs = self.model.beam_search(
817 + prompts,
818 + BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens))
819 returned_outputs = []
820 for output in outputs:
821 token_ids = [x.tokens for x in output.sequences]
tests/entrypoints/openai/test_completion.py
+24 -19
@@ -495,25 +495,30 @@ async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
495 assert len(batch.choices) == 2
496 assert batch.choices[0].text == batch.choices[1].text
497
498 - # test n = 2
499 - batch = await client.completions.create(
500 - model=model_name,
501 - prompt=prompts,
502 - n=2,
503 - max_tokens=5,
504 - temperature=0.0,
505 - extra_body=dict(
506 - # NOTE: this has to be true for n > 1 in vLLM, but not necessary
507 - # for official client.
508 - use_beam_search=True),
509 - )
510 - assert len(batch.choices) == 4
511 - assert batch.choices[0].text != batch.choices[
512 - 1].text, "beam search should be different"
513 - assert batch.choices[0].text == batch.choices[
514 - 2].text, "two copies of the same prompt should be the same"
515 - assert batch.choices[1].text == batch.choices[
516 - 3].text, "two copies of the same prompt should be the same"
498 + try:
499 + # test n = 2
500 + batch = await client.completions.create(
501 + model=model_name,
502 + prompt=prompts,
503 + n=2,
504 + max_tokens=5,
505 + temperature=0.0,
506 + extra_body=dict(
507 + # NOTE: this has to be true for n > 1 in vLLM, but
508 + # not necessary for official client.
509 + use_beam_search=True),
510 + )
511 + assert len(batch.choices) == 4
512 + assert batch.choices[0].text != batch.choices[
513 + 1].text, "beam search should be different"
514 + assert batch.choices[0].text == batch.choices[
515 + 2].text, "two copies of the same prompt should be the same"
516 + assert batch.choices[1].text == batch.choices[
517 + 3].text, "two copies of the same prompt should be the same"
518 + except BadRequestError as e:
519 + # the only allowed exception is when beam search is not supported
520 + # in the default mqllmengine
521 + assert "--disable-frontend-multiprocessing" in str(e)
522
523 # test streaming
524 batch = await client.completions.create(
vllm/engine/async_llm_engine.py
+103 -4
@@ -14,23 +14,26 @@ from vllm.engine.arg_utils import AsyncEngineArgs
14 from vllm.engine.async_timeout import asyncio_timeout
15 from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
16 from vllm.engine.metrics_types import StatLoggerBase
17 +from vllm.entrypoints.llm import BeamSearchSequence
18 from vllm.executor.executor_base import ExecutorAsyncBase
19 from vllm.executor.gpu_executor import GPUExecutorAsync
20 from vllm.executor.ray_utils import initialize_ray_cluster
20 -from vllm.inputs import PromptType
21 +from vllm.inputs import PromptType, TokensPrompt
22 from vllm.logger import init_logger
23 from vllm.lora.request import LoRARequest
24 from vllm.model_executor.guided_decoding import (
25 get_guided_decoding_logits_processor)
26 from vllm.model_executor.layers.sampler import SamplerOutput
26 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27 +from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
28 + RequestOutput)
29 from vllm.pooling_params import PoolingParams
30 from vllm.prompt_adapter.request import PromptAdapterRequest
29 -from vllm.sampling_params import SamplingParams
31 +from vllm.sampling_params import BeamSearchParams, SamplingParams
32 from vllm.sequence import ExecuteModelRequest
33 from vllm.transformers_utils.tokenizer import AnyTokenizer
34 from vllm.usage.usage_lib import UsageContext
33 -from vllm.utils import deprecate_kwargs, weak_bind
35 +from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36 + random_uuid, weak_bind)
37
38 logger = init_logger(__name__)
39 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -1036,6 +1039,102 @@ class AsyncLLMEngine:
1039 ):
1040 yield LLMEngine.validate_output(output, RequestOutput)
1041
1042 + async def beam_search(
1043 + self,
1044 + prompt: Union[PromptType, List[int]],
1045 + request_id: str,
1046 + params: BeamSearchParams,
1047 + ) -> AsyncGenerator[RequestOutput, None]:
1048 +
1049 + beam_width = params.beam_width
1050 + max_tokens = params.max_tokens
1051 + ignore_eos = params.ignore_eos
1052 + temperature = params.temperature
1053 +
1054 + tokenizer = await self.get_tokenizer()
1055 + tokenizedPrompt = prompt if isinstance(
1056 + prompt, list) else tokenizer.encode(prompt)
1057 + tokenizedLength = len(tokenizedPrompt)
1058 +
1059 + beam_search_params = SamplingParams(logprobs=2 * beam_width,
1060 + max_tokens=1,
1061 + temperature=temperature)
1062 + all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
1063 + completed = []
1064 +
1065 + for _ in range(max_tokens):
1066 + prompts_batch = [
1067 + TokensPrompt(prompt_token_ids=beam.tokens)
1068 + for beam in all_beams
1069 + ]
1070 +
1071 + tasks = []
1072 +
1073 + request_id = f"beam_search-{random_uuid()}"
1074 + for i, individual_prompt in enumerate(prompts_batch):
1075 + request_id_item = f"{request_id}-{i}"
1076 + task = asyncio.create_task(
1077 + collect_from_async_generator(
1078 + self.generate(individual_prompt, beam_search_params,
1079 + request_id_item)))
1080 + tasks.append(task)
1081 +
1082 + output = await asyncio.gather(*tasks)
1083 +
1084 + output = [x[0] for x in output]
1085 +
1086 + logger.info(output)
1087 +
1088 + new_beams = []
1089 + for i, current_beam in enumerate(all_beams):
1090 + result = output[i]
1091 +
1092 + if result.outputs[0].logprobs is not None:
1093 + logprobs = result.outputs[0].logprobs[0]
1094 + for token_id, logprob_obj in logprobs.items():
1095 + new_beam = BeamSearchSequence(
1096 + tokens=current_beam.tokens + [token_id],
1097 + cum_logprob=current_beam.cum_logprob +
1098 + logprob_obj.logprob)
1099 +
1100 + if token_id == tokenizer.eos_token_id and \
1101 + not ignore_eos:
1102 + completed.append(new_beam)
1103 + else:
1104 + new_beams.append(new_beam)
1105 +
1106 + sorted_beams = sorted(new_beams,
1107 + key=lambda x: x.cum_logprob,
1108 + reverse=True)
1109 + all_beams = sorted_beams[:beam_width]
1110 +
1111 + completed.extend(all_beams)
1112 + sorted_completed = sorted(completed,
1113 + key=lambda x: x.cum_logprob,
1114 + reverse=True)
1115 + best_beams = sorted_completed[:beam_width]
1116 +
1117 + for beam in best_beams:
1118 + beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
1119 +
1120 + beam_search_output = RequestOutput(
1121 + request_id=request_id,
1122 + prompt=prompt,
1123 + outputs=[
1124 + CompletionOutput(
1125 + text=beam.text,
1126 + cumulative_logprob=beam.cum_logprob,
1127 + token_ids=beam.tokens,
1128 + index=i,
1129 + logprobs=beam.cum_logprob,
1130 + ) for (i, beam) in enumerate(best_beams)
1131 + ],
1132 + finished=True,
1133 + prompt_token_ids=tokenizedPrompt,
1134 + prompt_logprobs=None)
1135 +
1136 + yield LLMEngine.validate_output(beam_search_output, RequestOutput)
1137 +
1138 async def encode(
1139 self,
1140 prompt: PromptType,
vllm/entrypoints/llm.py
+10 -10
@@ -22,8 +22,8 @@ from vllm.model_executor.guided_decoding.guided_fields import (
22 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
23 from vllm.pooling_params import PoolingParams
24 from vllm.prompt_adapter.request import PromptAdapterRequest
25 -from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
26 - SamplingParams)
25 +from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
26 + RequestOutputKind, SamplingParams)
27 from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
28 get_cached_tokenizer)
29 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -394,10 +394,7 @@ class LLM:
394 def beam_search(
395 self,
396 prompts: List[Union[str, List[int]]],
397 - beam_width: int,
398 - max_tokens: int,
399 - ignore_eos: bool = False,
400 - temperature: float = 0.0,
397 + params: BeamSearchParams,
398 ) -> List[BeamSearchOutput]:
399 """
400 Generate sequences using beam search.
@@ -405,14 +402,17 @@ class LLM:
402 Args:
403 prompts: A list of prompts. Each prompt can be a string or a list
404 of token IDs.
408 - beam_width: The number of beams to keep at each step.
409 - max_tokens: The max number of tokens to generate for each prompt.
410 - temperature: The temperature to use for generation.
411 -
405 + params: The beam search parameters.
406 +
407 TODO: how does beam search work together with length penalty, frequency
408 penalty, and stopping criteria, etc.?
409 """
410
411 + beam_width = params.beam_width
412 + max_tokens = params.max_tokens
413 + temperature = params.temperature
414 + ignore_eos = params.ignore_eos
415 +
416 tokenizer = self.get_tokenizer()
417 # generate 2 * beam_width candidates at each step
418 # following the huggingface transformers implementation
vllm/entrypoints/logger.py
+3 -2
@@ -4,7 +4,7 @@ from vllm.logger import init_logger
4 from vllm.lora.request import LoRARequest
5 from vllm.pooling_params import PoolingParams
6 from vllm.prompt_adapter.request import PromptAdapterRequest
7 -from vllm.sampling_params import SamplingParams
7 +from vllm.sampling_params import BeamSearchParams, SamplingParams
8
9 logger = init_logger(__name__)
10
@@ -21,7 +21,8 @@ class RequestLogger:
21 request_id: str,
22 prompt: Optional[str],
23 prompt_token_ids: Optional[List[int]],
24 - params: Optional[Union[SamplingParams, PoolingParams]],
24 + params: Optional[Union[SamplingParams, PoolingParams,
25 + BeamSearchParams]],
26 lora_request: Optional[LoRARequest],
27 prompt_adapter_request: Optional[PromptAdapterRequest],
28 ) -> None:
vllm/entrypoints/openai/protocol.py
+34 -2
@@ -11,8 +11,8 @@ from typing_extensions import Annotated, Required, TypedDict
11
12 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13 from vllm.pooling_params import PoolingParams
14 -from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
15 - SamplingParams)
14 +from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
15 + RequestOutputKind, SamplingParams)
16 from vllm.sequence import Logprob
17 from vllm.utils import random_uuid
18
@@ -288,6 +288,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
288
289 # doc: end-chat-completion-extra-params
290
291 + def to_beam_search_params(self,
292 + default_max_tokens: int) -> BeamSearchParams:
293 + max_tokens = self.max_tokens
294 + if max_tokens is None:
295 + max_tokens = default_max_tokens
296 +
297 + n = self.n if self.n is not None else 1
298 + temperature = self.temperature if self.temperature is not None else 0.0
299 +
300 + return BeamSearchParams(
301 + beam_width=n,
302 + max_tokens=max_tokens,
303 + ignore_eos=self.ignore_eos,
304 + temperature=temperature,
305 + )
306 +
307 def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
308 max_tokens = self.max_tokens
309 if max_tokens is None:
@@ -567,6 +583,22 @@ class CompletionRequest(OpenAIBaseModel):
583
584 # doc: end-completion-extra-params
585
586 + def to_beam_search_params(self,
587 + default_max_tokens: int) -> BeamSearchParams:
588 + max_tokens = self.max_tokens
589 + if max_tokens is None:
590 + max_tokens = default_max_tokens
591 +
592 + n = self.n if self.n is not None else 1
593 + temperature = self.temperature if self.temperature is not None else 0.0
594 +
595 + return BeamSearchParams(
596 + beam_width=n,
597 + max_tokens=max_tokens,
598 + ignore_eos=self.ignore_eos,
599 + temperature=temperature,
600 + )
601 +
602 def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
603 max_tokens = self.max_tokens
604 if max_tokens is None:
vllm/entrypoints/openai/serving_chat.py
+31 -12
@@ -9,6 +9,7 @@ from typing import Union
9 from fastapi import Request
10
11 from vllm.config import ModelConfig
12 +from vllm.engine.async_llm_engine import AsyncLLMEngine
13 from vllm.engine.protocol import EngineClient
14 from vllm.entrypoints.chat_utils import (ConversationMessage,
15 apply_hf_chat_template,
@@ -33,6 +34,7 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
34 from vllm.inputs import TokensPrompt
35 from vllm.logger import init_logger
36 from vllm.outputs import CompletionOutput, RequestOutput
37 +from vllm.sampling_params import BeamSearchParams, SamplingParams
38 from vllm.sequence import Logprob
39 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
40 log_tracing_disabled_warning)
@@ -203,9 +205,15 @@ class OpenAIServingChat(OpenAIServing):
205
206 assert prompt_inputs is not None
207
206 - sampling_params = request.to_sampling_params(
207 - default_max_tokens=self.max_model_len -
208 - len(prompt_inputs["prompt_token_ids"]))
208 + sampling_params: Union[SamplingParams, BeamSearchParams]
209 + default_max_tokens = self.max_model_len - len(
210 + prompt_inputs["prompt_token_ids"])
211 + if request.use_beam_search:
212 + sampling_params = request.to_beam_search_params(
213 + default_max_tokens)
214 + else:
215 + sampling_params = request.to_sampling_params(
216 + default_max_tokens)
217
218 self._log_inputs(request_id,
219 prompt_inputs,
@@ -227,15 +235,26 @@ class OpenAIServingChat(OpenAIServing):
235 and contains_trace_headers(raw_request.headers)):
236 log_tracing_disabled_warning()
237
230 - result_generator = self.engine_client.generate(
231 - engine_inputs,
232 - sampling_params,
233 - request_id,
234 - lora_request=lora_request,
235 - trace_headers=trace_headers,
236 - prompt_adapter_request=prompt_adapter_request,
237 - priority=request.priority,
238 - )
238 + if isinstance(sampling_params, BeamSearchParams):
239 + if not isinstance(self.engine_client, AsyncLLMEngine):
240 + raise ValueError(
241 + "Beam search in the API server is only supported with"
242 + " AsyncLLMEngine. please add "
243 + "`--disable-frontend-multiprocessing` to "
244 + "use beam search.")
245 + result_generator = self.engine_client.beam_search(
246 + engine_inputs['prompt_token_ids'], request_id,
247 + sampling_params)
248 + else:
249 + result_generator = self.engine_client.generate(
250 + engine_inputs,
251 + sampling_params,
252 + request_id,
253 + lora_request=lora_request,
254 + trace_headers=trace_headers,
255 + prompt_adapter_request=prompt_adapter_request,
256 + priority=request.priority,
257 + )
258 except ValueError as e:
259 # TODO: Use a vllm-specific Validation Error
260 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_completion.py
+34 -12
@@ -8,6 +8,7 @@ from typing import Tuple, Union, cast
8 from fastapi import Request
9
10 from vllm.config import ModelConfig
11 +from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.engine.protocol import EngineClient
13 from vllm.entrypoints.logger import RequestLogger
14 # yapf conflicts with isort for this block
@@ -28,6 +29,7 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
29 PromptAdapterPath)
30 from vllm.logger import init_logger
31 from vllm.outputs import RequestOutput
32 +from vllm.sampling_params import BeamSearchParams, SamplingParams
33 from vllm.sequence import Logprob
34 from vllm.tracing import (contains_trace_headers, extract_trace_headers,
35 log_tracing_disabled_warning)
@@ -120,9 +122,15 @@ class OpenAIServingCompletion(OpenAIServing):
122 ))
123
124 for i, prompt_inputs in enumerate(prompts):
123 - sampling_params = request.to_sampling_params(
124 - default_max_tokens=self.max_model_len -
125 - len(prompt_inputs["prompt_token_ids"]))
125 + sampling_params: Union[SamplingParams, BeamSearchParams]
126 + default_max_tokens = self.max_model_len - len(
127 + prompt_inputs["prompt_token_ids"])
128 + if request.use_beam_search:
129 + sampling_params = request.to_beam_search_params(
130 + default_max_tokens)
131 + else:
132 + sampling_params = request.to_sampling_params(
133 + default_max_tokens)
134
135 request_id_item = f"{request_id}-{i}"
136
@@ -141,15 +149,29 @@ class OpenAIServingCompletion(OpenAIServing):
149 raw_request.headers):
150 log_tracing_disabled_warning()
151
144 - generator = self.engine_client.generate(
145 - {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
146 - sampling_params,
147 - request_id_item,
148 - lora_request=lora_request,
149 - prompt_adapter_request=prompt_adapter_request,
150 - trace_headers=trace_headers,
151 - priority=request.priority,
152 - )
152 + if isinstance(sampling_params, BeamSearchParams):
153 + if not isinstance(self.engine_client, AsyncLLMEngine):
154 + raise ValueError(
155 + "Beam search in the API server is only supported"
156 + " with AsyncLLMEngine. please add "
157 + "`--disable-frontend-multiprocessing` to "
158 + "use beam search.")
159 + generator = self.engine_client.beam_search(
160 + prompt_inputs["prompt_token_ids"], request_id_item,
161 + sampling_params)
162 + else:
163 + generator = self.engine_client.generate(
164 + {
165 + "prompt_token_ids":
166 + prompt_inputs["prompt_token_ids"]
167 + },
168 + sampling_params,
169 + request_id_item,
170 + lora_request=lora_request,
171 + prompt_adapter_request=prompt_adapter_request,
172 + trace_headers=trace_headers,
173 + priority=request.priority,
174 + )
175
176 generators.append(generator)
177 except ValueError as e:
vllm/entrypoints/openai/serving_engine.py
+3 -2
@@ -29,7 +29,7 @@ from vllm.logger import init_logger
29 from vllm.lora.request import LoRARequest
30 from vllm.pooling_params import PoolingParams
31 from vllm.prompt_adapter.request import PromptAdapterRequest
32 -from vllm.sampling_params import SamplingParams
32 +from vllm.sampling_params import BeamSearchParams, SamplingParams
33 from vllm.sequence import Logprob
34 from vllm.transformers_utils.tokenizer import AnyTokenizer
35 from vllm.utils import AtomicCounter
@@ -371,7 +371,8 @@ class OpenAIServing:
371 self,
372 request_id: str,
373 inputs: Union[str, List[int], TextTokensPrompt],
374 - params: Optional[Union[SamplingParams, PoolingParams]],
374 + params: Optional[Union[SamplingParams, PoolingParams,
375 + BeamSearchParams]],
376 lora_request: Optional[LoRARequest],
377 prompt_adapter_request: Optional[PromptAdapterRequest],
378 ) -> None:
vllm/sampling_params.py
+12
@@ -530,3 +530,15 @@ class SamplingParams(
530 f"{self.spaces_between_special_tokens}, "
531 f"truncate_prompt_tokens={self.truncate_prompt_tokens}), "
532 f"guided_decoding={self.guided_decoding}")
533 +
534 +
535 +class BeamSearchParams(
536 + msgspec.Struct,
537 + omit_defaults=True, # type: ignore[call-arg]
538 + # required for @cached_property.
539 + dict=True): # type: ignore[call-arg]
540 + """Beam search parameters for text generation."""
541 + beam_width: int
542 + max_tokens: int
543 + ignore_eos: bool = False
544 + temperature: float = 0.0
vllm/utils.py
+9
@@ -504,6 +504,15 @@ async def merge_async_iterators(
504 await it.aclose()
505
506
507 +async def collect_from_async_generator(
508 + iterator: AsyncGenerator[T, None]) -> List[T]:
509 + """Collect all items from an async generator into a list."""
510 + items = []
511 + async for item in iterator:
512 + items.append(item)
513 + return items
514 +
515 +
516 def get_ip() -> str:
517 host_ip = envs.VLLM_HOST_IP
518 if host_ip: