A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend] API support for beam search (#9087)
Co-authored-by: youkaichao <youkaichao@126.com>
Brendan Wong committed
Oct 5, 2024 at 23:39 UTC
168cab6bbfb733f97defc8c1aa13df90c5319f19
12 files changed
+275
-68
benchmarks/benchmark_throughput.py
+8
-4
@@ -15,6 +15,7 @@ from vllm.engine.arg_utils import DEVICE_OPTIONS, AsyncEngineArgs, EngineArgs
15
from vllm.entrypoints.openai.api_server import (
16
build_async_engine_client_from_engine_args)
17
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
18
+from vllm.sampling_params import BeamSearchParams
19
from vllm.utils import FlexibleArgumentParser, merge_async_iterators
20
21
@@ -145,10 +146,13 @@ def run_vllm(
146
for prompt, input_len, _output_len in requests:
147
assert _output_len == output_len
148
start = time.perf_counter()
148
- llm.beam_search(prompts,
149
- beam_width=n,
150
- max_tokens=output_len,
151
- ignore_eos=True)
149
+ llm.beam_search(
150
+ prompts,
151
+ BeamSearchParams(
152
+ beam_width=n,
153
+ max_tokens=output_len,
154
+ ignore_eos=True,
155
+ ))
156
end = time.perf_counter()
157
return end - start
158
tests/conftest.py
+4
-1
@@ -35,6 +35,7 @@ from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
35
to_enc_dec_tuple_list, zip_enc_dec_prompts)
36
from vllm.logger import init_logger
37
from vllm.outputs import RequestOutput
38
+from vllm.sampling_params import BeamSearchParams
39
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, cuda_device_count_stateless,
40
identity, is_cpu)
41
@@ -812,7 +813,9 @@ class VllmRunner:
813
beam_width: int,
814
max_tokens: int,
815
) -> List[Tuple[List[List[int]], List[str]]]:
815
- outputs = self.model.beam_search(prompts, beam_width, max_tokens)
816
+ outputs = self.model.beam_search(
817
+ prompts,
818
+ BeamSearchParams(beam_width=beam_width, max_tokens=max_tokens))
819
returned_outputs = []
820
for output in outputs:
821
token_ids = [x.tokens for x in output.sequences]
tests/entrypoints/openai/test_completion.py
+24
-19
@@ -495,25 +495,30 @@ async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
495
assert len(batch.choices) == 2
496
assert batch.choices[0].text == batch.choices[1].text
497
498
- # test n = 2
499
- batch = await client.completions.create(
500
- model=model_name,
501
- prompt=prompts,
502
- n=2,
503
- max_tokens=5,
504
- temperature=0.0,
505
- extra_body=dict(
506
- # NOTE: this has to be true for n > 1 in vLLM, but not necessary
507
- # for official client.
508
- use_beam_search=True),
509
- )
510
- assert len(batch.choices) == 4
511
- assert batch.choices[0].text != batch.choices[
512
- 1].text, "beam search should be different"
513
- assert batch.choices[0].text == batch.choices[
514
- 2].text, "two copies of the same prompt should be the same"
515
- assert batch.choices[1].text == batch.choices[
516
- 3].text, "two copies of the same prompt should be the same"
498
+ try:
499
+ # test n = 2
500
+ batch = await client.completions.create(
501
+ model=model_name,
502
+ prompt=prompts,
503
+ n=2,
504
+ max_tokens=5,
505
+ temperature=0.0,
506
+ extra_body=dict(
507
+ # NOTE: this has to be true for n > 1 in vLLM, but
508
+ # not necessary for official client.
509
+ use_beam_search=True),
510
+ )
511
+ assert len(batch.choices) == 4
512
+ assert batch.choices[0].text != batch.choices[
513
+ 1].text, "beam search should be different"
514
+ assert batch.choices[0].text == batch.choices[
515
+ 2].text, "two copies of the same prompt should be the same"
516
+ assert batch.choices[1].text == batch.choices[
517
+ 3].text, "two copies of the same prompt should be the same"
518
+ except BadRequestError as e:
519
+ # the only allowed exception is when beam search is not supported
520
+ # in the default mqllmengine
521
+ assert "--disable-frontend-multiprocessing" in str(e)
522
523
# test streaming
524
batch = await client.completions.create(
vllm/engine/async_llm_engine.py
+103
-4
@@ -14,23 +14,26 @@ from vllm.engine.arg_utils import AsyncEngineArgs
14
from vllm.engine.async_timeout import asyncio_timeout
15
from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
16
from vllm.engine.metrics_types import StatLoggerBase
17
+from vllm.entrypoints.llm import BeamSearchSequence
18
from vllm.executor.executor_base import ExecutorAsyncBase
19
from vllm.executor.gpu_executor import GPUExecutorAsync
20
from vllm.executor.ray_utils import initialize_ray_cluster
20
-from vllm.inputs import PromptType
21
+from vllm.inputs import PromptType, TokensPrompt
22
from vllm.logger import init_logger
23
from vllm.lora.request import LoRARequest
24
from vllm.model_executor.guided_decoding import (
25
get_guided_decoding_logits_processor)
26
from vllm.model_executor.layers.sampler import SamplerOutput
26
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27
+from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
28
+ RequestOutput)
29
from vllm.pooling_params import PoolingParams
30
from vllm.prompt_adapter.request import PromptAdapterRequest
29
-from vllm.sampling_params import SamplingParams
31
+from vllm.sampling_params import BeamSearchParams, SamplingParams
32
from vllm.sequence import ExecuteModelRequest
33
from vllm.transformers_utils.tokenizer import AnyTokenizer
34
from vllm.usage.usage_lib import UsageContext
33
-from vllm.utils import deprecate_kwargs, weak_bind
35
+from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36
+ random_uuid, weak_bind)
37
38
logger = init_logger(__name__)
39
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -1036,6 +1039,102 @@ class AsyncLLMEngine:
1039
):
1040
yield LLMEngine.validate_output(output, RequestOutput)
1041
1042
+ async def beam_search(
1043
+ self,
1044
+ prompt: Union[PromptType, List[int]],
1045
+ request_id: str,
1046
+ params: BeamSearchParams,
1047
+ ) -> AsyncGenerator[RequestOutput, None]:
1048
+
1049
+ beam_width = params.beam_width
1050
+ max_tokens = params.max_tokens
1051
+ ignore_eos = params.ignore_eos
1052
+ temperature = params.temperature
1053
+
1054
+ tokenizer = await self.get_tokenizer()
1055
+ tokenizedPrompt = prompt if isinstance(
1056
+ prompt, list) else tokenizer.encode(prompt)
1057
+ tokenizedLength = len(tokenizedPrompt)
1058
+
1059
+ beam_search_params = SamplingParams(logprobs=2 * beam_width,
1060
+ max_tokens=1,
1061
+ temperature=temperature)
1062
+ all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
1063
+ completed = []
1064
+
1065
+ for _ in range(max_tokens):
1066
+ prompts_batch = [
1067
+ TokensPrompt(prompt_token_ids=beam.tokens)
1068
+ for beam in all_beams
1069
+ ]
1070
+
1071
+ tasks = []
1072
+
1073
+ request_id = f"beam_search-{random_uuid()}"
1074
+ for i, individual_prompt in enumerate(prompts_batch):
1075
+ request_id_item = f"{request_id}-{i}"
1076
+ task = asyncio.create_task(
1077
+ collect_from_async_generator(
1078
+ self.generate(individual_prompt, beam_search_params,
1079
+ request_id_item)))
1080
+ tasks.append(task)
1081
+
1082
+ output = await asyncio.gather(*tasks)
1083
+
1084
+ output = [x[0] for x in output]
1085
+
1086
+ logger.info(output)
1087
+
1088
+ new_beams = []
1089
+ for i, current_beam in enumerate(all_beams):
1090
+ result = output[i]
1091
+
1092
+ if result.outputs[0].logprobs is not None:
1093
+ logprobs = result.outputs[0].logprobs[0]
1094
+ for token_id, logprob_obj in logprobs.items():
1095
+ new_beam = BeamSearchSequence(
1096
+ tokens=current_beam.tokens + [token_id],
1097
+ cum_logprob=current_beam.cum_logprob +
1098
+ logprob_obj.logprob)
1099
+
1100
+ if token_id == tokenizer.eos_token_id and \
1101
+ not ignore_eos:
1102
+ completed.append(new_beam)
1103
+ else:
1104
+ new_beams.append(new_beam)
1105
+
1106
+ sorted_beams = sorted(new_beams,
1107
+ key=lambda x: x.cum_logprob,
1108
+ reverse=True)
1109
+ all_beams = sorted_beams[:beam_width]
1110
+
1111
+ completed.extend(all_beams)
1112
+ sorted_completed = sorted(completed,
1113
+ key=lambda x: x.cum_logprob,
1114
+ reverse=True)
1115
+ best_beams = sorted_completed[:beam_width]
1116
+
1117
+ for beam in best_beams:
1118
+ beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
1119
+
1120
+ beam_search_output = RequestOutput(
1121
+ request_id=request_id,
1122
+ prompt=prompt,
1123
+ outputs=[
1124
+ CompletionOutput(
1125
+ text=beam.text,
1126
+ cumulative_logprob=beam.cum_logprob,
1127
+ token_ids=beam.tokens,
1128
+ index=i,
1129
+ logprobs=beam.cum_logprob,
1130
+ ) for (i, beam) in enumerate(best_beams)
1131
+ ],
1132
+ finished=True,
1133
+ prompt_token_ids=tokenizedPrompt,
1134
+ prompt_logprobs=None)
1135
+
1136
+ yield LLMEngine.validate_output(beam_search_output, RequestOutput)
1137
+
1138
async def encode(
1139
self,
1140
prompt: PromptType,
vllm/entrypoints/llm.py
+10
-10
@@ -22,8 +22,8 @@ from vllm.model_executor.guided_decoding.guided_fields import (
22
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
23
from vllm.pooling_params import PoolingParams
24
from vllm.prompt_adapter.request import PromptAdapterRequest
25
-from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
26
- SamplingParams)
25
+from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
26
+ RequestOutputKind, SamplingParams)
27
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
28
get_cached_tokenizer)
29
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -394,10 +394,7 @@ class LLM:
394
def beam_search(
395
self,
396
prompts: List[Union[str, List[int]]],
397
- beam_width: int,
398
- max_tokens: int,
399
- ignore_eos: bool = False,
400
- temperature: float = 0.0,
397
+ params: BeamSearchParams,
398
) -> List[BeamSearchOutput]:
399
"""
400
Generate sequences using beam search.
@@ -405,14 +402,17 @@ class LLM:
402
Args:
403
prompts: A list of prompts. Each prompt can be a string or a list
404
of token IDs.
408
- beam_width: The number of beams to keep at each step.
409
- max_tokens: The max number of tokens to generate for each prompt.
410
- temperature: The temperature to use for generation.
411
-
405
+ params: The beam search parameters.
406
+
407
TODO: how does beam search work together with length penalty, frequency
408
penalty, and stopping criteria, etc.?
409
"""
410
411
+ beam_width = params.beam_width
412
+ max_tokens = params.max_tokens
413
+ temperature = params.temperature
414
+ ignore_eos = params.ignore_eos
415
+
416
tokenizer = self.get_tokenizer()
417
# generate 2 * beam_width candidates at each step
418
# following the huggingface transformers implementation
vllm/entrypoints/logger.py
+3
-2
@@ -4,7 +4,7 @@ from vllm.logger import init_logger
4
from vllm.lora.request import LoRARequest
5
from vllm.pooling_params import PoolingParams
6
from vllm.prompt_adapter.request import PromptAdapterRequest
7
-from vllm.sampling_params import SamplingParams
7
+from vllm.sampling_params import BeamSearchParams, SamplingParams
8
9
logger = init_logger(__name__)
10
@@ -21,7 +21,8 @@ class RequestLogger:
21
request_id: str,
22
prompt: Optional[str],
23
prompt_token_ids: Optional[List[int]],
24
- params: Optional[Union[SamplingParams, PoolingParams]],
24
+ params: Optional[Union[SamplingParams, PoolingParams,
25
+ BeamSearchParams]],
26
lora_request: Optional[LoRARequest],
27
prompt_adapter_request: Optional[PromptAdapterRequest],
28
) -> None:
vllm/entrypoints/openai/protocol.py
+34
-2
@@ -11,8 +11,8 @@ from typing_extensions import Annotated, Required, TypedDict
11
12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.pooling_params import PoolingParams
14
-from vllm.sampling_params import (GuidedDecodingParams, RequestOutputKind,
15
- SamplingParams)
14
+from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
15
+ RequestOutputKind, SamplingParams)
16
from vllm.sequence import Logprob
17
from vllm.utils import random_uuid
18
@@ -288,6 +288,22 @@ class ChatCompletionRequest(OpenAIBaseModel):
288
289
# doc: end-chat-completion-extra-params
290
291
+ def to_beam_search_params(self,
292
+ default_max_tokens: int) -> BeamSearchParams:
293
+ max_tokens = self.max_tokens
294
+ if max_tokens is None:
295
+ max_tokens = default_max_tokens
296
+
297
+ n = self.n if self.n is not None else 1
298
+ temperature = self.temperature if self.temperature is not None else 0.0
299
+
300
+ return BeamSearchParams(
301
+ beam_width=n,
302
+ max_tokens=max_tokens,
303
+ ignore_eos=self.ignore_eos,
304
+ temperature=temperature,
305
+ )
306
+
307
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
308
max_tokens = self.max_tokens
309
if max_tokens is None:
@@ -567,6 +583,22 @@ class CompletionRequest(OpenAIBaseModel):
583
584
# doc: end-completion-extra-params
585
586
+ def to_beam_search_params(self,
587
+ default_max_tokens: int) -> BeamSearchParams:
588
+ max_tokens = self.max_tokens
589
+ if max_tokens is None:
590
+ max_tokens = default_max_tokens
591
+
592
+ n = self.n if self.n is not None else 1
593
+ temperature = self.temperature if self.temperature is not None else 0.0
594
+
595
+ return BeamSearchParams(
596
+ beam_width=n,
597
+ max_tokens=max_tokens,
598
+ ignore_eos=self.ignore_eos,
599
+ temperature=temperature,
600
+ )
601
+
602
def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
603
max_tokens = self.max_tokens
604
if max_tokens is None:
vllm/entrypoints/openai/serving_chat.py
+31
-12
@@ -9,6 +9,7 @@ from typing import Union
9
from fastapi import Request
10
11
from vllm.config import ModelConfig
12
+from vllm.engine.async_llm_engine import AsyncLLMEngine
13
from vllm.engine.protocol import EngineClient
14
from vllm.entrypoints.chat_utils import (ConversationMessage,
15
apply_hf_chat_template,
@@ -33,6 +34,7 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
34
from vllm.inputs import TokensPrompt
35
from vllm.logger import init_logger
36
from vllm.outputs import CompletionOutput, RequestOutput
37
+from vllm.sampling_params import BeamSearchParams, SamplingParams
38
from vllm.sequence import Logprob
39
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
40
log_tracing_disabled_warning)
@@ -203,9 +205,15 @@ class OpenAIServingChat(OpenAIServing):
205
206
assert prompt_inputs is not None
207
206
- sampling_params = request.to_sampling_params(
207
- default_max_tokens=self.max_model_len -
208
- len(prompt_inputs["prompt_token_ids"]))
208
+ sampling_params: Union[SamplingParams, BeamSearchParams]
209
+ default_max_tokens = self.max_model_len - len(
210
+ prompt_inputs["prompt_token_ids"])
211
+ if request.use_beam_search:
212
+ sampling_params = request.to_beam_search_params(
213
+ default_max_tokens)
214
+ else:
215
+ sampling_params = request.to_sampling_params(
216
+ default_max_tokens)
217
218
self._log_inputs(request_id,
219
prompt_inputs,
@@ -227,15 +235,26 @@ class OpenAIServingChat(OpenAIServing):
235
and contains_trace_headers(raw_request.headers)):
236
log_tracing_disabled_warning()
237
230
- result_generator = self.engine_client.generate(
231
- engine_inputs,
232
- sampling_params,
233
- request_id,
234
- lora_request=lora_request,
235
- trace_headers=trace_headers,
236
- prompt_adapter_request=prompt_adapter_request,
237
- priority=request.priority,
238
- )
238
+ if isinstance(sampling_params, BeamSearchParams):
239
+ if not isinstance(self.engine_client, AsyncLLMEngine):
240
+ raise ValueError(
241
+ "Beam search in the API server is only supported with"
242
+ " AsyncLLMEngine. please add "
243
+ "`--disable-frontend-multiprocessing` to "
244
+ "use beam search.")
245
+ result_generator = self.engine_client.beam_search(
246
+ engine_inputs['prompt_token_ids'], request_id,
247
+ sampling_params)
248
+ else:
249
+ result_generator = self.engine_client.generate(
250
+ engine_inputs,
251
+ sampling_params,
252
+ request_id,
253
+ lora_request=lora_request,
254
+ trace_headers=trace_headers,
255
+ prompt_adapter_request=prompt_adapter_request,
256
+ priority=request.priority,
257
+ )
258
except ValueError as e:
259
# TODO: Use a vllm-specific Validation Error
260
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_completion.py
+34
-12
@@ -8,6 +8,7 @@ from typing import Tuple, Union, cast
8
from fastapi import Request
9
10
from vllm.config import ModelConfig
11
+from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.logger import RequestLogger
14
# yapf conflicts with isort for this block
@@ -28,6 +29,7 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
29
PromptAdapterPath)
30
from vllm.logger import init_logger
31
from vllm.outputs import RequestOutput
32
+from vllm.sampling_params import BeamSearchParams, SamplingParams
33
from vllm.sequence import Logprob
34
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
35
log_tracing_disabled_warning)
@@ -120,9 +122,15 @@ class OpenAIServingCompletion(OpenAIServing):
122
))
123
124
for i, prompt_inputs in enumerate(prompts):
123
- sampling_params = request.to_sampling_params(
124
- default_max_tokens=self.max_model_len -
125
- len(prompt_inputs["prompt_token_ids"]))
125
+ sampling_params: Union[SamplingParams, BeamSearchParams]
126
+ default_max_tokens = self.max_model_len - len(
127
+ prompt_inputs["prompt_token_ids"])
128
+ if request.use_beam_search:
129
+ sampling_params = request.to_beam_search_params(
130
+ default_max_tokens)
131
+ else:
132
+ sampling_params = request.to_sampling_params(
133
+ default_max_tokens)
134
135
request_id_item = f"{request_id}-{i}"
136
@@ -141,15 +149,29 @@ class OpenAIServingCompletion(OpenAIServing):
149
raw_request.headers):
150
log_tracing_disabled_warning()
151
144
- generator = self.engine_client.generate(
145
- {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
146
- sampling_params,
147
- request_id_item,
148
- lora_request=lora_request,
149
- prompt_adapter_request=prompt_adapter_request,
150
- trace_headers=trace_headers,
151
- priority=request.priority,
152
- )
152
+ if isinstance(sampling_params, BeamSearchParams):
153
+ if not isinstance(self.engine_client, AsyncLLMEngine):
154
+ raise ValueError(
155
+ "Beam search in the API server is only supported"
156
+ " with AsyncLLMEngine. please add "
157
+ "`--disable-frontend-multiprocessing` to "
158
+ "use beam search.")
159
+ generator = self.engine_client.beam_search(
160
+ prompt_inputs["prompt_token_ids"], request_id_item,
161
+ sampling_params)
162
+ else:
163
+ generator = self.engine_client.generate(
164
+ {
165
+ "prompt_token_ids":
166
+ prompt_inputs["prompt_token_ids"]
167
+ },
168
+ sampling_params,
169
+ request_id_item,
170
+ lora_request=lora_request,
171
+ prompt_adapter_request=prompt_adapter_request,
172
+ trace_headers=trace_headers,
173
+ priority=request.priority,
174
+ )
175
176
generators.append(generator)
177
except ValueError as e:
vllm/entrypoints/openai/serving_engine.py
+3
-2
@@ -29,7 +29,7 @@ from vllm.logger import init_logger
29
from vllm.lora.request import LoRARequest
30
from vllm.pooling_params import PoolingParams
31
from vllm.prompt_adapter.request import PromptAdapterRequest
32
-from vllm.sampling_params import SamplingParams
32
+from vllm.sampling_params import BeamSearchParams, SamplingParams
33
from vllm.sequence import Logprob
34
from vllm.transformers_utils.tokenizer import AnyTokenizer
35
from vllm.utils import AtomicCounter
@@ -371,7 +371,8 @@ class OpenAIServing:
371
self,
372
request_id: str,
373
inputs: Union[str, List[int], TextTokensPrompt],
374
- params: Optional[Union[SamplingParams, PoolingParams]],
374
+ params: Optional[Union[SamplingParams, PoolingParams,
375
+ BeamSearchParams]],
376
lora_request: Optional[LoRARequest],
377
prompt_adapter_request: Optional[PromptAdapterRequest],
378
) -> None:
vllm/sampling_params.py
+12
@@ -530,3 +530,15 @@ class SamplingParams(
530
f"{self.spaces_between_special_tokens}, "
531
f"truncate_prompt_tokens={self.truncate_prompt_tokens}), "
532
f"guided_decoding={self.guided_decoding}")
533
+
534
+
535
+class BeamSearchParams(
536
+ msgspec.Struct,
537
+ omit_defaults=True, # type: ignore[call-arg]
538
+ # required for @cached_property.
539
+ dict=True): # type: ignore[call-arg]
540
+ """Beam search parameters for text generation."""
541
+ beam_width: int
542
+ max_tokens: int
543
+ ignore_eos: bool = False
544
+ temperature: float = 0.0
vllm/utils.py
+9
@@ -504,6 +504,15 @@ async def merge_async_iterators(
504
await it.aclose()
505
506
507
+async def collect_from_async_generator(
508
+ iterator: AsyncGenerator[T, None]) -> List[T]:
509
+ """Collect all items from an async generator into a list."""
510
+ items = []
511
+ async for item in iterator:
512
+ items.append(item)
513
+ return items
514
+
515
+
516
def get_ip() -> str:
517
host_ip = envs.VLLM_HOST_IP
518
if host_ip: