A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend] API support for beam search for MQLLMEngine (#9117)
Brendan Wong committed
Oct 7, 2024 at 22:51 UTC
8c746226c956f7c8a4672689fee91c7d22befed6
8 files changed
+215
-106
tests/entrypoints/openai/test_completion.py
+19
-24
@@ -495,30 +495,25 @@ async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
495
assert len(batch.choices) == 2
496
assert batch.choices[0].text == batch.choices[1].text
497
498
- try:
499
- # test n = 2
500
- batch = await client.completions.create(
501
- model=model_name,
502
- prompt=prompts,
503
- n=2,
504
- max_tokens=5,
505
- temperature=0.0,
506
- extra_body=dict(
507
- # NOTE: this has to be true for n > 1 in vLLM, but
508
- # not necessary for official client.
509
- use_beam_search=True),
510
- )
511
- assert len(batch.choices) == 4
512
- assert batch.choices[0].text != batch.choices[
513
- 1].text, "beam search should be different"
514
- assert batch.choices[0].text == batch.choices[
515
- 2].text, "two copies of the same prompt should be the same"
516
- assert batch.choices[1].text == batch.choices[
517
- 3].text, "two copies of the same prompt should be the same"
518
- except BadRequestError as e:
519
- # the only allowed exception is when beam search is not supported
520
- # in the default mqllmengine
521
- assert "--disable-frontend-multiprocessing" in str(e)
498
+ # test n = 2
499
+ batch = await client.completions.create(
500
+ model=model_name,
501
+ prompt=prompts,
502
+ n=2,
503
+ max_tokens=5,
504
+ temperature=0.0,
505
+ extra_body=dict(
506
+ # NOTE: this has to be true for n > 1 in vLLM, but
507
+ # not necessary for official client.
508
+ use_beam_search=True),
509
+ )
510
+ assert len(batch.choices) == 4
511
+ assert batch.choices[0].text != batch.choices[
512
+ 1].text, "beam search should be different"
513
+ assert batch.choices[0].text == batch.choices[
514
+ 2].text, "two copies of the same prompt should be the same"
515
+ assert batch.choices[1].text == batch.choices[
516
+ 3].text, "two copies of the same prompt should be the same"
517
518
# test streaming
519
batch = await client.completions.create(
vllm/beam_search.py
new
+61
@@ -0,0 +1,61 @@
1
+from dataclasses import dataclass
2
+from typing import List, Optional
3
+
4
+
5
+@dataclass
6
+class BeamSearchSequence:
7
+ """A sequence for beam search.
8
+ It keeps track of the tokens and the log probability of the sequence.
9
+ The text field is optional and will only be filled when the sequence is
10
+ about to be returned to the user.
11
+ """
12
+ # The tokens includes the prompt.
13
+ tokens: List[int]
14
+ cum_logprob: float = 0.0
15
+ text: Optional[str] = None
16
+
17
+
18
+@dataclass
19
+class BeamSearchOutput:
20
+ """The output of beam search.
21
+ It contains the list of the best beam search sequences.
22
+ The length of the list is equal to the beam width.
23
+ """
24
+ sequences: List[BeamSearchSequence]
25
+
26
+
27
+class BeamSearchInstance:
28
+
29
+ def __init__(self, prompt_tokens: List[int]):
30
+ self.beams: List[BeamSearchSequence] = [
31
+ BeamSearchSequence(tokens=prompt_tokens)
32
+ ]
33
+ self.completed: List[BeamSearchSequence] = []
34
+
35
+
36
+def get_beam_search_score(
37
+ tokens: List[int],
38
+ cumulative_logprob: float,
39
+ eos_token_id: int,
40
+ length_penalty: float = 1.0,
41
+) -> float:
42
+ """Calculate the beam search score with length penalty.
43
+
44
+ Adapted from
45
+
46
+ https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
47
+ """
48
+ seq_len = len(tokens)
49
+ if tokens[-1] == eos_token_id:
50
+ seq_len -= 1
51
+
52
+ return cumulative_logprob / (seq_len**length_penalty)
53
+
54
+
55
+def create_sort_beams_key_function(eos_token_id: int, length_penalty: float):
56
+
57
+ def sort_beams_key(x: BeamSearchSequence) -> float:
58
+ return get_beam_search_score(x.tokens, x.cum_logprob, eos_token_id,
59
+ length_penalty)
60
+
61
+ return sort_beams_key
vllm/engine/async_llm_engine.py
+5
-7
@@ -7,6 +7,7 @@ from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
7
from weakref import ReferenceType
8
9
import vllm.envs as envs
10
+from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
11
from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
12
ParallelConfig, SchedulerConfig)
13
from vllm.core.scheduler import SchedulerOutputs
@@ -14,7 +15,6 @@ from vllm.engine.arg_utils import AsyncEngineArgs
15
from vllm.engine.async_timeout import asyncio_timeout
16
from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
17
from vllm.engine.metrics_types import StatLoggerBase
17
-from vllm.entrypoints.llm import BeamSearchSequence
18
from vllm.executor.executor_base import ExecutorAsyncBase
19
from vllm.executor.gpu_executor import GPUExecutorAsync
20
from vllm.executor.ray_utils import initialize_ray_cluster
@@ -33,7 +33,7 @@ from vllm.sequence import ExecuteModelRequest
33
from vllm.transformers_utils.tokenizer import AnyTokenizer
34
from vllm.usage.usage_lib import UsageContext
35
from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36
- get_beam_search_score, random_uuid, weak_bind)
36
+ random_uuid, weak_bind)
37
38
logger = init_logger(__name__)
39
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -1052,16 +1052,14 @@ class AsyncLLMEngine:
1052
temperature = params.temperature
1053
length_penalty = params.length_penalty
1054
1055
- def sort_beams_key(x: BeamSearchSequence) -> float:
1056
- return get_beam_search_score(x.tokens, x.cum_logprob,
1057
- tokenizer.eos_token_id,
1058
- length_penalty)
1059
-
1055
tokenizer = await self.get_tokenizer()
1056
tokenizedPrompt = prompt if isinstance(
1057
prompt, list) else tokenizer.encode(prompt)
1058
tokenizedLength = len(tokenizedPrompt)
1059
1060
+ sort_beams_key = create_sort_beams_key_function(
1061
+ tokenizer.eos_token_id, length_penalty)
1062
+
1063
beam_search_params = SamplingParams(logprobs=2 * beam_width,
1064
max_tokens=1,
1065
temperature=temperature)
vllm/engine/multiprocessing/client.py
+107
-6
@@ -2,8 +2,8 @@ import asyncio
2
import copy
3
import pickle
4
from contextlib import contextmanager, suppress
5
-from typing import (Any, AsyncGenerator, Dict, Iterator, Mapping, Optional,
6
- Union, overload)
5
+from typing import (Any, AsyncGenerator, Dict, Iterator, List, Mapping,
6
+ Optional, Union, overload)
7
8
import cloudpickle
9
import zmq
@@ -12,6 +12,7 @@ from zmq import Frame # type: ignore[attr-defined]
12
from zmq.asyncio import Socket
13
14
from vllm import PoolingParams
15
+from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
16
from vllm.config import DecodingConfig, EngineConfig, ModelConfig
17
from vllm.engine.arg_utils import AsyncEngineArgs
18
# yapf conflicts with isort for this block
@@ -27,14 +28,16 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
28
RPCUProfileRequest)
29
# yapf: enable
30
from vllm.envs import VLLM_RPC_TIMEOUT
30
-from vllm.inputs import PromptType
31
+from vllm.inputs import PromptType, TokensPrompt
32
from vllm.logger import init_logger
33
from vllm.lora.request import LoRARequest
33
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
34
+from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
35
+ RequestOutput)
36
from vllm.prompt_adapter.request import PromptAdapterRequest
35
-from vllm.sampling_params import SamplingParams
37
+from vllm.sampling_params import BeamSearchParams, SamplingParams
38
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
37
-from vllm.utils import deprecate_kwargs
39
+from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
40
+ random_uuid)
41
42
logger = init_logger(__name__)
43
@@ -441,6 +444,104 @@ class MQLLMEngineClient:
444
lora_request, trace_headers,
445
prompt_adapter_request, priority)
446
447
+ async def beam_search(
448
+ self,
449
+ prompt: Union[PromptType, List[int]],
450
+ request_id: str,
451
+ params: BeamSearchParams,
452
+ ) -> AsyncGenerator[RequestOutput, None]:
453
+
454
+ beam_width = params.beam_width
455
+ max_tokens = params.max_tokens
456
+ ignore_eos = params.ignore_eos
457
+ temperature = params.temperature
458
+ length_penalty = params.length_penalty
459
+
460
+ tokenizer = await self.get_tokenizer(lora_request=None)
461
+ tokenizedPrompt = prompt if isinstance(
462
+ prompt, list) else tokenizer.encode(prompt)
463
+ tokenizedLength = len(tokenizedPrompt)
464
+
465
+ sort_beams_key = create_sort_beams_key_function(
466
+ tokenizer.eos_token_id, length_penalty)
467
+
468
+ beam_search_params = SamplingParams(logprobs=2 * beam_width,
469
+ max_tokens=1,
470
+ temperature=temperature)
471
+ all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
472
+ completed = []
473
+
474
+ for _ in range(max_tokens):
475
+ prompts_batch = [
476
+ TokensPrompt(prompt_token_ids=beam.tokens)
477
+ for beam in all_beams
478
+ ]
479
+
480
+ tasks = []
481
+
482
+ request_id = f"beam_search-{random_uuid()}"
483
+ for i, individual_prompt in enumerate(prompts_batch):
484
+ request_id_item = f"{request_id}-{i}"
485
+ task = asyncio.create_task(
486
+ collect_from_async_generator(
487
+ self.generate(individual_prompt, beam_search_params,
488
+ request_id_item)))
489
+ tasks.append(task)
490
+
491
+ output = await asyncio.gather(*tasks)
492
+
493
+ output = [x[0] for x in output]
494
+
495
+ logger.info(output)
496
+
497
+ new_beams = []
498
+ for i, current_beam in enumerate(all_beams):
499
+ result = output[i]
500
+
501
+ if result.outputs[0].logprobs is not None:
502
+ logprobs = result.outputs[0].logprobs[0]
503
+ for token_id, logprob_obj in logprobs.items():
504
+ new_beam = BeamSearchSequence(
505
+ tokens=current_beam.tokens + [token_id],
506
+ cum_logprob=current_beam.cum_logprob +
507
+ logprob_obj.logprob)
508
+
509
+ if token_id == tokenizer.eos_token_id and \
510
+ not ignore_eos:
511
+ completed.append(new_beam)
512
+ else:
513
+ new_beams.append(new_beam)
514
+
515
+ sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
516
+ all_beams = sorted_beams[:beam_width]
517
+
518
+ completed.extend(all_beams)
519
+ sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
520
+ best_beams = sorted_completed[:beam_width]
521
+
522
+ for beam in best_beams:
523
+ beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
524
+
525
+ beam_search_output = RequestOutput(
526
+ request_id=request_id,
527
+ prompt=prompt,
528
+ outputs=[
529
+ CompletionOutput(
530
+ text=beam.text,
531
+ cumulative_logprob=beam.cum_logprob,
532
+ token_ids=beam.tokens,
533
+ index=i,
534
+ logprobs=beam.cum_logprob,
535
+ ) for (i, beam) in enumerate(best_beams)
536
+ ],
537
+ finished=True,
538
+ prompt_token_ids=tokenizedPrompt,
539
+ prompt_logprobs=None)
540
+
541
+ logger.info(beam_search_output)
542
+
543
+ yield beam_search_output
544
+
545
@overload # DEPRECATED
546
def encode(
547
self,
vllm/entrypoints/llm.py
+3
-34
@@ -1,12 +1,13 @@
1
import itertools
2
import warnings
3
from contextlib import contextmanager
4
-from dataclasses import dataclass
4
from typing import (Any, ClassVar, Dict, List, Optional, Sequence, Tuple,
5
Union, cast, overload)
6
7
from tqdm import tqdm
8
9
+from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
10
+ BeamSearchSequence, get_beam_search_score)
11
from vllm.engine.arg_utils import EngineArgs
12
from vllm.engine.llm_engine import LLMEngine
13
from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
@@ -28,43 +29,11 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
29
get_cached_tokenizer)
30
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
31
from vllm.usage.usage_lib import UsageContext
31
-from vllm.utils import (Counter, deprecate_kwargs, get_beam_search_score,
32
- is_list_of)
32
+from vllm.utils import Counter, deprecate_kwargs, is_list_of
33
34
logger = init_logger(__name__)
35
36
37
-@dataclass
38
-class BeamSearchSequence:
39
- """A sequence for beam search.
40
- It keeps track of the tokens and the log probability of the sequence.
41
- The text field is optional and will only be filled when the sequence is
42
- about to be returned to the user.
43
- """
44
- # The tokens includes the prompt.
45
- tokens: List[int]
46
- cum_logprob: float = 0.0
47
- text: Optional[str] = None
48
-
49
-
50
-@dataclass
51
-class BeamSearchOutput:
52
- """The output of beam search.
53
- It contains the list of the best beam search sequences.
54
- The length of the list is equal to the beam width.
55
- """
56
- sequences: List[BeamSearchSequence]
57
-
58
-
59
-class BeamSearchInstance:
60
-
61
- def __init__(self, prompt_tokens: List[int]):
62
- self.beams: List[BeamSearchSequence] = [
63
- BeamSearchSequence(tokens=prompt_tokens)
64
- ]
65
- self.completed: List[BeamSearchSequence] = []
66
-
67
-
37
class LLM:
38
"""An LLM for generating texts from given prompts and sampling parameters.
39
vllm/entrypoints/openai/serving_chat.py
+10
-8
@@ -10,6 +10,7 @@ from fastapi import Request
10
11
from vllm.config import ModelConfig
12
from vllm.engine.async_llm_engine import AsyncLLMEngine
13
+from vllm.engine.multiprocessing.client import MQLLMEngineClient
14
from vllm.engine.protocol import EngineClient
15
from vllm.entrypoints.chat_utils import (ConversationMessage,
16
apply_hf_chat_template,
@@ -236,15 +237,16 @@ class OpenAIServingChat(OpenAIServing):
237
log_tracing_disabled_warning()
238
239
if isinstance(sampling_params, BeamSearchParams):
239
- if not isinstance(self.engine_client, AsyncLLMEngine):
240
- raise ValueError(
241
- "Beam search in the API server is only supported with"
242
- " AsyncLLMEngine. please add "
243
- "`--disable-frontend-multiprocessing` to "
244
- "use beam search.")
240
+ assert isinstance(self.engine_client,
241
+ (AsyncLLMEngine,
242
+ MQLLMEngineClient)), \
243
+ "Beam search is only supported with" \
244
+ "AsyncLLMEngine and MQLLMEngineClient."
245
result_generator = self.engine_client.beam_search(
246
- engine_inputs['prompt_token_ids'], request_id,
247
- sampling_params)
246
+ engine_inputs['prompt_token_ids'],
247
+ request_id,
248
+ sampling_params,
249
+ )
250
else:
251
result_generator = self.engine_client.generate(
252
engine_inputs,
vllm/entrypoints/openai/serving_completion.py
+10
-8
@@ -9,6 +9,7 @@ from fastapi import Request
9
10
from vllm.config import ModelConfig
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
+from vllm.engine.multiprocessing.client import MQLLMEngineClient
13
from vllm.engine.protocol import EngineClient
14
from vllm.entrypoints.logger import RequestLogger
15
# yapf conflicts with isort for this block
@@ -150,15 +151,16 @@ class OpenAIServingCompletion(OpenAIServing):
151
log_tracing_disabled_warning()
152
153
if isinstance(sampling_params, BeamSearchParams):
153
- if not isinstance(self.engine_client, AsyncLLMEngine):
154
- raise ValueError(
155
- "Beam search in the API server is only supported"
156
- " with AsyncLLMEngine. please add "
157
- "`--disable-frontend-multiprocessing` to "
158
- "use beam search.")
154
+ assert isinstance(self.engine_client,
155
+ (AsyncLLMEngine,
156
+ MQLLMEngineClient)), \
157
+ "Beam search is only supported with" \
158
+ "AsyncLLMEngine and MQLLMEngineClient."
159
generator = self.engine_client.beam_search(
160
- prompt_inputs["prompt_token_ids"], request_id_item,
161
- sampling_params)
160
+ prompt_inputs["prompt_token_ids"],
161
+ request_id_item,
162
+ sampling_params,
163
+ )
164
else:
165
generator = self.engine_client.generate(
166
{
vllm/utils.py
-19
@@ -1370,22 +1370,3 @@ class AtomicCounter:
1370
@property
1371
def value(self):
1372
return self._value
1373
-
1374
-
1375
-def get_beam_search_score(
1376
- tokens: List[int],
1377
- cumulative_logprob: float,
1378
- eos_token_id: int,
1379
- length_penalty: float = 1.0,
1380
-) -> float:
1381
- """Calculate the beam search score with length penalty.
1382
-
1383
- Adapted from
1384
-
1385
- https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
1386
- """
1387
- seq_len = len(tokens)
1388
- if tokens[-1] == eos_token_id:
1389
- seq_len -= 1
1390
-
1391
- return cumulative_logprob / (seq_len**length_penalty)