A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Frontend] merge beam search implementations (#9296)
Brendan Wong committed
Oct 14, 2024 at 15:05 UTC
4d31cd424bdd5935cefa8f03e137bba127be31dd
5 files changed
+145
-234
vllm/engine/async_llm_engine.py
+6
-104
@@ -7,7 +7,6 @@ from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
7
from weakref import ReferenceType
8
9
import vllm.envs as envs
10
-from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
10
from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
11
ParallelConfig, SchedulerConfig)
12
from vllm.core.scheduler import SchedulerOutputs
@@ -15,25 +14,24 @@ from vllm.engine.arg_utils import AsyncEngineArgs
14
from vllm.engine.async_timeout import asyncio_timeout
15
from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
16
from vllm.engine.metrics_types import StatLoggerBase
17
+from vllm.engine.protocol import EngineClient
18
from vllm.executor.executor_base import ExecutorAsyncBase
19
from vllm.executor.gpu_executor import GPUExecutorAsync
20
from vllm.executor.ray_utils import initialize_ray_cluster
21
-from vllm.inputs import PromptType, TokensPrompt
21
+from vllm.inputs import PromptType
22
from vllm.logger import init_logger
23
from vllm.lora.request import LoRARequest
24
from vllm.model_executor.guided_decoding import (
25
get_guided_decoding_logits_processor)
26
from vllm.model_executor.layers.sampler import SamplerOutput
27
-from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
28
- RequestOutput)
27
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
28
from vllm.pooling_params import PoolingParams
29
from vllm.prompt_adapter.request import PromptAdapterRequest
31
-from vllm.sampling_params import BeamSearchParams, SamplingParams
30
+from vllm.sampling_params import SamplingParams
31
from vllm.sequence import ExecuteModelRequest
32
from vllm.transformers_utils.tokenizer import AnyTokenizer
33
from vllm.usage.usage_lib import UsageContext
35
-from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36
- random_uuid, weak_bind)
34
+from vllm.utils import deprecate_kwargs, weak_bind
35
36
logger = init_logger(__name__)
37
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -541,7 +539,7 @@ async def build_guided_decoding_logits_processor_async(
539
return sampling_params
540
541
544
-class AsyncLLMEngine:
542
+class AsyncLLMEngine(EngineClient):
543
"""An asynchronous wrapper for :class:`LLMEngine`.
544
545
This class is used to wrap the :class:`LLMEngine` class to make it
@@ -1039,102 +1037,6 @@ class AsyncLLMEngine:
1037
):
1038
yield LLMEngine.validate_output(output, RequestOutput)
1039
1042
- async def beam_search(
1043
- self,
1044
- prompt: Union[PromptType, List[int]],
1045
- request_id: str,
1046
- params: BeamSearchParams,
1047
- ) -> AsyncGenerator[RequestOutput, None]:
1048
-
1049
- beam_width = params.beam_width
1050
- max_tokens = params.max_tokens
1051
- ignore_eos = params.ignore_eos
1052
- temperature = params.temperature
1053
- length_penalty = params.length_penalty
1054
-
1055
- tokenizer = await self.get_tokenizer()
1056
- tokenizedPrompt = prompt if isinstance(
1057
- prompt, list) else tokenizer.encode(prompt)
1058
- tokenizedLength = len(tokenizedPrompt)
1059
-
1060
- sort_beams_key = create_sort_beams_key_function(
1061
- tokenizer.eos_token_id, length_penalty)
1062
-
1063
- beam_search_params = SamplingParams(logprobs=2 * beam_width,
1064
- max_tokens=1,
1065
- temperature=temperature)
1066
- all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
1067
- completed = []
1068
-
1069
- for _ in range(max_tokens):
1070
- prompts_batch = [
1071
- TokensPrompt(prompt_token_ids=beam.tokens)
1072
- for beam in all_beams
1073
- ]
1074
-
1075
- tasks = []
1076
-
1077
- request_id = f"beam_search-{random_uuid()}"
1078
- for i, individual_prompt in enumerate(prompts_batch):
1079
- request_id_item = f"{request_id}-{i}"
1080
- task = asyncio.create_task(
1081
- collect_from_async_generator(
1082
- self.generate(individual_prompt, beam_search_params,
1083
- request_id_item)))
1084
- tasks.append(task)
1085
-
1086
- output = await asyncio.gather(*tasks)
1087
-
1088
- output = [x[0] for x in output]
1089
-
1090
- logger.info(output)
1091
-
1092
- new_beams = []
1093
- for i, current_beam in enumerate(all_beams):
1094
- result = output[i]
1095
-
1096
- if result.outputs[0].logprobs is not None:
1097
- logprobs = result.outputs[0].logprobs[0]
1098
- for token_id, logprob_obj in logprobs.items():
1099
- new_beam = BeamSearchSequence(
1100
- tokens=current_beam.tokens + [token_id],
1101
- cum_logprob=current_beam.cum_logprob +
1102
- logprob_obj.logprob)
1103
-
1104
- if token_id == tokenizer.eos_token_id and \
1105
- not ignore_eos:
1106
- completed.append(new_beam)
1107
- else:
1108
- new_beams.append(new_beam)
1109
-
1110
- sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
1111
- all_beams = sorted_beams[:beam_width]
1112
-
1113
- completed.extend(all_beams)
1114
- sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
1115
- best_beams = sorted_completed[:beam_width]
1116
-
1117
- for beam in best_beams:
1118
- beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
1119
-
1120
- beam_search_output = RequestOutput(
1121
- request_id=request_id,
1122
- prompt=prompt,
1123
- outputs=[
1124
- CompletionOutput(
1125
- text=beam.text,
1126
- cumulative_logprob=beam.cum_logprob,
1127
- token_ids=beam.tokens,
1128
- index=i,
1129
- logprobs=beam.cum_logprob,
1130
- ) for (i, beam) in enumerate(best_beams)
1131
- ],
1132
- finished=True,
1133
- prompt_token_ids=tokenizedPrompt,
1134
- prompt_logprobs=None)
1135
-
1136
- yield LLMEngine.validate_output(beam_search_output, RequestOutput)
1137
-
1040
async def encode(
1041
self,
1042
prompt: PromptType,
vllm/engine/multiprocessing/client.py
+17
-109
@@ -12,8 +12,8 @@ from zmq import Frame # type: ignore[attr-defined]
12
from zmq.asyncio import Socket
13
14
from vllm import PoolingParams
15
-from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
15
from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16
+from vllm.core.scheduler import SchedulerOutputs
17
from vllm.engine.arg_utils import AsyncEngineArgs
18
# yapf conflicts with isort for this block
19
# yapf: disable
@@ -26,18 +26,18 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
26
RPCError, RPCProcessRequest,
27
RPCStartupRequest, RPCStartupResponse,
28
RPCUProfileRequest)
29
+from vllm.engine.protocol import EngineClient
30
# yapf: enable
31
from vllm.envs import VLLM_RPC_TIMEOUT
31
-from vllm.inputs import PromptType, TokensPrompt
32
+from vllm.inputs import PromptType
33
from vllm.logger import init_logger
34
from vllm.lora.request import LoRARequest
34
-from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
35
- RequestOutput)
35
+from vllm.model_executor.layers.sampler import SamplerOutput
36
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
37
from vllm.prompt_adapter.request import PromptAdapterRequest
37
-from vllm.sampling_params import BeamSearchParams, SamplingParams
38
+from vllm.sampling_params import SamplingParams
39
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
39
-from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
40
- random_uuid)
40
+from vllm.utils import deprecate_kwargs
41
42
logger = init_logger(__name__)
43
@@ -53,7 +53,7 @@ class MQClientClosedError(Exception):
53
"""
54
55
56
-class MQLLMEngineClient:
56
+class MQLLMEngineClient(EngineClient):
57
"""A client wrapper for MQLLMEngine that conforms to the
58
EngineClient protocol.
59
@@ -316,7 +316,7 @@ class MQLLMEngineClient:
316
or response != VLLM_RPC_SUCCESS_STR):
317
raise ValueError(error_message)
318
319
- async def get_tokenizer(self, lora_request: LoRARequest):
319
+ async def get_tokenizer(self, lora_request: Optional[LoRARequest] = None):
320
return await self.tokenizer.get_lora_tokenizer_async(lora_request)
321
322
async def get_decoding_config(self) -> DecodingConfig:
@@ -344,8 +344,14 @@ class MQLLMEngineClient:
344
await self._send_one_way_rpc_request(
345
request=RPCAbortRequest(request_id), socket=self.input_socket)
346
347
- async def do_log_stats(self):
348
- """Ignore do_log_stats (handled on MQLLMEngine polling)"""
347
+ async def do_log_stats(
348
+ self,
349
+ scheduler_outputs: Optional[SchedulerOutputs] = None,
350
+ model_output: Optional[List[SamplerOutput]] = None,
351
+ ) -> None:
352
+ """
353
+ Ignore do_log_stats (handled on MQLLMEngine polling)
354
+ """
355
pass
356
357
async def check_health(self):
@@ -444,104 +450,6 @@ class MQLLMEngineClient:
450
lora_request, trace_headers,
451
prompt_adapter_request, priority)
452
447
- async def beam_search(
448
- self,
449
- prompt: Union[PromptType, List[int]],
450
- request_id: str,
451
- params: BeamSearchParams,
452
- ) -> AsyncGenerator[RequestOutput, None]:
453
-
454
- beam_width = params.beam_width
455
- max_tokens = params.max_tokens
456
- ignore_eos = params.ignore_eos
457
- temperature = params.temperature
458
- length_penalty = params.length_penalty
459
-
460
- tokenizer = await self.get_tokenizer(lora_request=None)
461
- tokenizedPrompt = prompt if isinstance(
462
- prompt, list) else tokenizer.encode(prompt)
463
- tokenizedLength = len(tokenizedPrompt)
464
-
465
- sort_beams_key = create_sort_beams_key_function(
466
- tokenizer.eos_token_id, length_penalty)
467
-
468
- beam_search_params = SamplingParams(logprobs=2 * beam_width,
469
- max_tokens=1,
470
- temperature=temperature)
471
- all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
472
- completed = []
473
-
474
- for _ in range(max_tokens):
475
- prompts_batch = [
476
- TokensPrompt(prompt_token_ids=beam.tokens)
477
- for beam in all_beams
478
- ]
479
-
480
- tasks = []
481
-
482
- request_id = f"beam_search-{random_uuid()}"
483
- for i, individual_prompt in enumerate(prompts_batch):
484
- request_id_item = f"{request_id}-{i}"
485
- task = asyncio.create_task(
486
- collect_from_async_generator(
487
- self.generate(individual_prompt, beam_search_params,
488
- request_id_item)))
489
- tasks.append(task)
490
-
491
- output = await asyncio.gather(*tasks)
492
-
493
- output = [x[0] for x in output]
494
-
495
- logger.info(output)
496
-
497
- new_beams = []
498
- for i, current_beam in enumerate(all_beams):
499
- result = output[i]
500
-
501
- if result.outputs[0].logprobs is not None:
502
- logprobs = result.outputs[0].logprobs[0]
503
- for token_id, logprob_obj in logprobs.items():
504
- new_beam = BeamSearchSequence(
505
- tokens=current_beam.tokens + [token_id],
506
- cum_logprob=current_beam.cum_logprob +
507
- logprob_obj.logprob)
508
-
509
- if token_id == tokenizer.eos_token_id and \
510
- not ignore_eos:
511
- completed.append(new_beam)
512
- else:
513
- new_beams.append(new_beam)
514
-
515
- sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
516
- all_beams = sorted_beams[:beam_width]
517
-
518
- completed.extend(all_beams)
519
- sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
520
- best_beams = sorted_completed[:beam_width]
521
-
522
- for beam in best_beams:
523
- beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
524
-
525
- beam_search_output = RequestOutput(
526
- request_id=request_id,
527
- prompt=prompt,
528
- outputs=[
529
- CompletionOutput(
530
- text=beam.text,
531
- cumulative_logprob=beam.cum_logprob,
532
- token_ids=beam.tokens,
533
- index=i,
534
- logprobs=beam.cum_logprob,
535
- ) for (i, beam) in enumerate(best_beams)
536
- ],
537
- finished=True,
538
- prompt_token_ids=tokenizedPrompt,
539
- prompt_logprobs=None)
540
-
541
- logger.info(beam_search_output)
542
-
543
- yield beam_search_output
544
-
453
@overload # DEPRECATED
454
def encode(
455
self,
vllm/engine/protocol.py
+122
-7
@@ -1,38 +1,49 @@
1
-from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
2
- runtime_checkable)
1
+import asyncio
2
+from abc import ABC, abstractmethod
3
+from typing import AsyncGenerator, List, Mapping, Optional, Union
4
5
+from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
6
from vllm.config import DecodingConfig, ModelConfig
7
from vllm.core.scheduler import SchedulerOutputs
6
-from vllm.inputs.data import PromptType
8
+from vllm.inputs.data import PromptType, TokensPrompt
9
+from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
11
from vllm.model_executor.layers.sampler import SamplerOutput
9
-from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12
+from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
13
+ RequestOutput)
14
from vllm.pooling_params import PoolingParams
15
from vllm.prompt_adapter.request import PromptAdapterRequest
12
-from vllm.sampling_params import SamplingParams
16
+from vllm.sampling_params import BeamSearchParams, SamplingParams
17
from vllm.transformers_utils.tokenizer import AnyTokenizer
18
+from vllm.utils import collect_from_async_generator, random_uuid
19
20
+logger = init_logger(__name__)
21
16
-@runtime_checkable
17
-class EngineClient(Protocol):
22
+
23
+class EngineClient(ABC):
24
"""Protocol class for Clients to Engine"""
25
26
@property
27
+ @abstractmethod
28
def is_running(self) -> bool:
29
...
30
31
@property
32
+ @abstractmethod
33
def is_stopped(self) -> bool:
34
...
35
36
@property
37
+ @abstractmethod
38
def errored(self) -> bool:
39
...
40
41
@property
42
+ @abstractmethod
43
def dead_error(self) -> BaseException:
44
...
45
46
+ @abstractmethod
47
def generate(
48
self,
49
prompt: PromptType,
@@ -46,6 +57,101 @@ class EngineClient(Protocol):
57
"""Generate outputs for a request."""
58
...
59
60
+ async def beam_search(
61
+ self,
62
+ prompt: Union[PromptType, List[int]],
63
+ request_id: str,
64
+ params: BeamSearchParams,
65
+ ) -> AsyncGenerator[RequestOutput, None]:
66
+
67
+ beam_width = params.beam_width
68
+ max_tokens = params.max_tokens
69
+ ignore_eos = params.ignore_eos
70
+ temperature = params.temperature
71
+ length_penalty = params.length_penalty
72
+
73
+ tokenizer = await self.get_tokenizer(lora_request=None)
74
+ tokenizedPrompt = prompt if isinstance(
75
+ prompt, list) else tokenizer.encode(prompt)
76
+ tokenizedLength = len(tokenizedPrompt)
77
+
78
+ sort_beams_key = create_sort_beams_key_function(
79
+ tokenizer.eos_token_id, length_penalty)
80
+
81
+ beam_search_params = SamplingParams(logprobs=2 * beam_width,
82
+ max_tokens=1,
83
+ temperature=temperature)
84
+ all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
85
+ completed = []
86
+
87
+ for _ in range(max_tokens):
88
+ prompts_batch = [
89
+ TokensPrompt(prompt_token_ids=beam.tokens)
90
+ for beam in all_beams
91
+ ]
92
+
93
+ tasks = []
94
+
95
+ request_id = f"beam_search-{random_uuid()}"
96
+ for i, individual_prompt in enumerate(prompts_batch):
97
+ request_id_item = f"{request_id}-{i}"
98
+ task = asyncio.create_task(
99
+ collect_from_async_generator(
100
+ self.generate(individual_prompt, beam_search_params,
101
+ request_id_item)))
102
+ tasks.append(task)
103
+
104
+ output = await asyncio.gather(*tasks)
105
+
106
+ output = [x[0] for x in output]
107
+
108
+ new_beams = []
109
+ for i, current_beam in enumerate(all_beams):
110
+ result = output[i]
111
+
112
+ if result.outputs[0].logprobs is not None:
113
+ logprobs = result.outputs[0].logprobs[0]
114
+ for token_id, logprob_obj in logprobs.items():
115
+ new_beam = BeamSearchSequence(
116
+ tokens=current_beam.tokens + [token_id],
117
+ cum_logprob=current_beam.cum_logprob +
118
+ logprob_obj.logprob)
119
+
120
+ if token_id == tokenizer.eos_token_id and \
121
+ not ignore_eos:
122
+ completed.append(new_beam)
123
+ else:
124
+ new_beams.append(new_beam)
125
+
126
+ sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
127
+ all_beams = sorted_beams[:beam_width]
128
+
129
+ completed.extend(all_beams)
130
+ sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
131
+ best_beams = sorted_completed[:beam_width]
132
+
133
+ for beam in best_beams:
134
+ beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
135
+
136
+ beam_search_output = RequestOutput(
137
+ request_id=request_id,
138
+ prompt=prompt,
139
+ outputs=[
140
+ CompletionOutput(
141
+ text=beam.text,
142
+ cumulative_logprob=beam.cum_logprob,
143
+ token_ids=beam.tokens,
144
+ index=i,
145
+ logprobs=beam.cum_logprob,
146
+ ) for (i, beam) in enumerate(best_beams)
147
+ ],
148
+ finished=True,
149
+ prompt_token_ids=tokenizedPrompt,
150
+ prompt_logprobs=None)
151
+
152
+ yield beam_search_output
153
+
154
+ @abstractmethod
155
def encode(
156
self,
157
prompt: PromptType,
@@ -58,6 +164,7 @@ class EngineClient(Protocol):
164
"""Generate outputs for a request from an embedding model."""
165
...
166
167
+ @abstractmethod
168
async def abort(self, request_id: str) -> None:
169
"""Abort a request.
170
@@ -65,14 +172,17 @@ class EngineClient(Protocol):
172
request_id: The unique id of the request.
173
"""
174
175
+ @abstractmethod
176
async def get_model_config(self) -> ModelConfig:
177
"""Get the model configuration of the vLLM engine."""
178
...
179
180
+ @abstractmethod
181
async def get_decoding_config(self) -> DecodingConfig:
182
...
183
"""Get the decoding configuration of the vLLM engine."""
184
185
+ @abstractmethod
186
async def get_tokenizer(
187
self,
188
lora_request: Optional[LoRARequest] = None,
@@ -80,9 +190,11 @@ class EngineClient(Protocol):
190
"""Get the appropriate tokenizer for the request"""
191
...
192
193
+ @abstractmethod
194
async def is_tracing_enabled(self) -> bool:
195
...
196
197
+ @abstractmethod
198
async def do_log_stats(
199
self,
200
scheduler_outputs: Optional[SchedulerOutputs] = None,
@@ -90,14 +202,17 @@ class EngineClient(Protocol):
202
) -> None:
203
...
204
205
+ @abstractmethod
206
async def check_health(self) -> None:
207
"""Raise if unhealthy"""
208
...
209
210
+ @abstractmethod
211
async def start_profile(self) -> None:
212
"""Start profiling the engine"""
213
...
214
215
+ @abstractmethod
216
async def stop_profile(self) -> None:
217
"""Start profiling the engine"""
218
...
vllm/entrypoints/openai/serving_chat.py
-7
@@ -9,8 +9,6 @@ from typing import Union
9
from fastapi import Request
10
11
from vllm.config import ModelConfig
12
-from vllm.engine.async_llm_engine import AsyncLLMEngine
13
-from vllm.engine.multiprocessing.client import MQLLMEngineClient
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.chat_utils import (ConversationMessage,
14
apply_hf_chat_template,
@@ -237,11 +235,6 @@ class OpenAIServingChat(OpenAIServing):
235
log_tracing_disabled_warning()
236
237
if isinstance(sampling_params, BeamSearchParams):
240
- assert isinstance(self.engine_client,
241
- (AsyncLLMEngine,
242
- MQLLMEngineClient)), \
243
- "Beam search is only supported with" \
244
- "AsyncLLMEngine and MQLLMEngineClient."
238
result_generator = self.engine_client.beam_search(
239
engine_inputs['prompt_token_ids'],
240
request_id,
vllm/entrypoints/openai/serving_completion.py
-7
@@ -8,8 +8,6 @@ from typing import Tuple, Union, cast
8
from fastapi import Request
9
10
from vllm.config import ModelConfig
11
-from vllm.engine.async_llm_engine import AsyncLLMEngine
12
-from vllm.engine.multiprocessing.client import MQLLMEngineClient
11
from vllm.engine.protocol import EngineClient
12
from vllm.entrypoints.logger import RequestLogger
13
# yapf conflicts with isort for this block
@@ -151,11 +149,6 @@ class OpenAIServingCompletion(OpenAIServing):
149
log_tracing_disabled_warning()
150
151
if isinstance(sampling_params, BeamSearchParams):
154
- assert isinstance(self.engine_client,
155
- (AsyncLLMEngine,
156
- MQLLMEngineClient)), \
157
- "Beam search is only supported with" \
158
- "AsyncLLMEngine and MQLLMEngineClient."
152
generator = self.engine_client.beam_search(
153
prompt_inputs["prompt_token_ids"],
154
request_id_item,