@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4d31cd424b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend] merge beam search implementations (#9296)

Brendan Wong committed Oct 14, 2024 at 15:05 UTC 4d31cd424bdd5935cefa8f03e137bba127be31dd
5 files changed +145 -234
vllm/engine/async_llm_engine.py
+6 -104
@@ -7,7 +7,6 @@ from typing import (Any, AsyncGenerator, Callable, Coroutine, Dict, Iterable,
7 from weakref import ReferenceType
8
9 import vllm.envs as envs
10 -from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
10 from vllm.config import (DecodingConfig, EngineConfig, LoRAConfig, ModelConfig,
11 ParallelConfig, SchedulerConfig)
12 from vllm.core.scheduler import SchedulerOutputs
@@ -15,25 +14,24 @@ from vllm.engine.arg_utils import AsyncEngineArgs
14 from vllm.engine.async_timeout import asyncio_timeout
15 from vllm.engine.llm_engine import LLMEngine, SchedulerOutputState
16 from vllm.engine.metrics_types import StatLoggerBase
17 +from vllm.engine.protocol import EngineClient
18 from vllm.executor.executor_base import ExecutorAsyncBase
19 from vllm.executor.gpu_executor import GPUExecutorAsync
20 from vllm.executor.ray_utils import initialize_ray_cluster
21 -from vllm.inputs import PromptType, TokensPrompt
21 +from vllm.inputs import PromptType
22 from vllm.logger import init_logger
23 from vllm.lora.request import LoRARequest
24 from vllm.model_executor.guided_decoding import (
25 get_guided_decoding_logits_processor)
26 from vllm.model_executor.layers.sampler import SamplerOutput
27 -from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
28 - RequestOutput)
27 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
28 from vllm.pooling_params import PoolingParams
29 from vllm.prompt_adapter.request import PromptAdapterRequest
31 -from vllm.sampling_params import BeamSearchParams, SamplingParams
30 +from vllm.sampling_params import SamplingParams
31 from vllm.sequence import ExecuteModelRequest
32 from vllm.transformers_utils.tokenizer import AnyTokenizer
33 from vllm.usage.usage_lib import UsageContext
35 -from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
36 - random_uuid, weak_bind)
34 +from vllm.utils import deprecate_kwargs, weak_bind
35
36 logger = init_logger(__name__)
37 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -541,7 +539,7 @@ async def build_guided_decoding_logits_processor_async(
539 return sampling_params
540
541
544 -class AsyncLLMEngine:
542 +class AsyncLLMEngine(EngineClient):
543 """An asynchronous wrapper for :class:`LLMEngine`.
544
545 This class is used to wrap the :class:`LLMEngine` class to make it
@@ -1039,102 +1037,6 @@ class AsyncLLMEngine:
1037 ):
1038 yield LLMEngine.validate_output(output, RequestOutput)
1039
1042 - async def beam_search(
1043 - self,
1044 - prompt: Union[PromptType, List[int]],
1045 - request_id: str,
1046 - params: BeamSearchParams,
1047 - ) -> AsyncGenerator[RequestOutput, None]:
1048 -
1049 - beam_width = params.beam_width
1050 - max_tokens = params.max_tokens
1051 - ignore_eos = params.ignore_eos
1052 - temperature = params.temperature
1053 - length_penalty = params.length_penalty
1054 -
1055 - tokenizer = await self.get_tokenizer()
1056 - tokenizedPrompt = prompt if isinstance(
1057 - prompt, list) else tokenizer.encode(prompt)
1058 - tokenizedLength = len(tokenizedPrompt)
1059 -
1060 - sort_beams_key = create_sort_beams_key_function(
1061 - tokenizer.eos_token_id, length_penalty)
1062 -
1063 - beam_search_params = SamplingParams(logprobs=2 * beam_width,
1064 - max_tokens=1,
1065 - temperature=temperature)
1066 - all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
1067 - completed = []
1068 -
1069 - for _ in range(max_tokens):
1070 - prompts_batch = [
1071 - TokensPrompt(prompt_token_ids=beam.tokens)
1072 - for beam in all_beams
1073 - ]
1074 -
1075 - tasks = []
1076 -
1077 - request_id = f"beam_search-{random_uuid()}"
1078 - for i, individual_prompt in enumerate(prompts_batch):
1079 - request_id_item = f"{request_id}-{i}"
1080 - task = asyncio.create_task(
1081 - collect_from_async_generator(
1082 - self.generate(individual_prompt, beam_search_params,
1083 - request_id_item)))
1084 - tasks.append(task)
1085 -
1086 - output = await asyncio.gather(*tasks)
1087 -
1088 - output = [x[0] for x in output]
1089 -
1090 - logger.info(output)
1091 -
1092 - new_beams = []
1093 - for i, current_beam in enumerate(all_beams):
1094 - result = output[i]
1095 -
1096 - if result.outputs[0].logprobs is not None:
1097 - logprobs = result.outputs[0].logprobs[0]
1098 - for token_id, logprob_obj in logprobs.items():
1099 - new_beam = BeamSearchSequence(
1100 - tokens=current_beam.tokens + [token_id],
1101 - cum_logprob=current_beam.cum_logprob +
1102 - logprob_obj.logprob)
1103 -
1104 - if token_id == tokenizer.eos_token_id and \
1105 - not ignore_eos:
1106 - completed.append(new_beam)
1107 - else:
1108 - new_beams.append(new_beam)
1109 -
1110 - sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
1111 - all_beams = sorted_beams[:beam_width]
1112 -
1113 - completed.extend(all_beams)
1114 - sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
1115 - best_beams = sorted_completed[:beam_width]
1116 -
1117 - for beam in best_beams:
1118 - beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
1119 -
1120 - beam_search_output = RequestOutput(
1121 - request_id=request_id,
1122 - prompt=prompt,
1123 - outputs=[
1124 - CompletionOutput(
1125 - text=beam.text,
1126 - cumulative_logprob=beam.cum_logprob,
1127 - token_ids=beam.tokens,
1128 - index=i,
1129 - logprobs=beam.cum_logprob,
1130 - ) for (i, beam) in enumerate(best_beams)
1131 - ],
1132 - finished=True,
1133 - prompt_token_ids=tokenizedPrompt,
1134 - prompt_logprobs=None)
1135 -
1136 - yield LLMEngine.validate_output(beam_search_output, RequestOutput)
1137 -
1040 async def encode(
1041 self,
1042 prompt: PromptType,
vllm/engine/multiprocessing/client.py
+17 -109
@@ -12,8 +12,8 @@ from zmq import Frame # type: ignore[attr-defined]
12 from zmq.asyncio import Socket
13
14 from vllm import PoolingParams
15 -from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
15 from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16 +from vllm.core.scheduler import SchedulerOutputs
17 from vllm.engine.arg_utils import AsyncEngineArgs
18 # yapf conflicts with isort for this block
19 # yapf: disable
@@ -26,18 +26,18 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
26 RPCError, RPCProcessRequest,
27 RPCStartupRequest, RPCStartupResponse,
28 RPCUProfileRequest)
29 +from vllm.engine.protocol import EngineClient
30 # yapf: enable
31 from vllm.envs import VLLM_RPC_TIMEOUT
31 -from vllm.inputs import PromptType, TokensPrompt
32 +from vllm.inputs import PromptType
33 from vllm.logger import init_logger
34 from vllm.lora.request import LoRARequest
34 -from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
35 - RequestOutput)
35 +from vllm.model_executor.layers.sampler import SamplerOutput
36 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
37 from vllm.prompt_adapter.request import PromptAdapterRequest
37 -from vllm.sampling_params import BeamSearchParams, SamplingParams
38 +from vllm.sampling_params import SamplingParams
39 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
39 -from vllm.utils import (collect_from_async_generator, deprecate_kwargs,
40 - random_uuid)
40 +from vllm.utils import deprecate_kwargs
41
42 logger = init_logger(__name__)
43
@@ -53,7 +53,7 @@ class MQClientClosedError(Exception):
53 """
54
55
56 -class MQLLMEngineClient:
56 +class MQLLMEngineClient(EngineClient):
57 """A client wrapper for MQLLMEngine that conforms to the
58 EngineClient protocol.
59
@@ -316,7 +316,7 @@ class MQLLMEngineClient:
316 or response != VLLM_RPC_SUCCESS_STR):
317 raise ValueError(error_message)
318
319 - async def get_tokenizer(self, lora_request: LoRARequest):
319 + async def get_tokenizer(self, lora_request: Optional[LoRARequest] = None):
320 return await self.tokenizer.get_lora_tokenizer_async(lora_request)
321
322 async def get_decoding_config(self) -> DecodingConfig:
@@ -344,8 +344,14 @@ class MQLLMEngineClient:
344 await self._send_one_way_rpc_request(
345 request=RPCAbortRequest(request_id), socket=self.input_socket)
346
347 - async def do_log_stats(self):
348 - """Ignore do_log_stats (handled on MQLLMEngine polling)"""
347 + async def do_log_stats(
348 + self,
349 + scheduler_outputs: Optional[SchedulerOutputs] = None,
350 + model_output: Optional[List[SamplerOutput]] = None,
351 + ) -> None:
352 + """
353 + Ignore do_log_stats (handled on MQLLMEngine polling)
354 + """
355 pass
356
357 async def check_health(self):
@@ -444,104 +450,6 @@ class MQLLMEngineClient:
450 lora_request, trace_headers,
451 prompt_adapter_request, priority)
452
447 - async def beam_search(
448 - self,
449 - prompt: Union[PromptType, List[int]],
450 - request_id: str,
451 - params: BeamSearchParams,
452 - ) -> AsyncGenerator[RequestOutput, None]:
453 -
454 - beam_width = params.beam_width
455 - max_tokens = params.max_tokens
456 - ignore_eos = params.ignore_eos
457 - temperature = params.temperature
458 - length_penalty = params.length_penalty
459 -
460 - tokenizer = await self.get_tokenizer(lora_request=None)
461 - tokenizedPrompt = prompt if isinstance(
462 - prompt, list) else tokenizer.encode(prompt)
463 - tokenizedLength = len(tokenizedPrompt)
464 -
465 - sort_beams_key = create_sort_beams_key_function(
466 - tokenizer.eos_token_id, length_penalty)
467 -
468 - beam_search_params = SamplingParams(logprobs=2 * beam_width,
469 - max_tokens=1,
470 - temperature=temperature)
471 - all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
472 - completed = []
473 -
474 - for _ in range(max_tokens):
475 - prompts_batch = [
476 - TokensPrompt(prompt_token_ids=beam.tokens)
477 - for beam in all_beams
478 - ]
479 -
480 - tasks = []
481 -
482 - request_id = f"beam_search-{random_uuid()}"
483 - for i, individual_prompt in enumerate(prompts_batch):
484 - request_id_item = f"{request_id}-{i}"
485 - task = asyncio.create_task(
486 - collect_from_async_generator(
487 - self.generate(individual_prompt, beam_search_params,
488 - request_id_item)))
489 - tasks.append(task)
490 -
491 - output = await asyncio.gather(*tasks)
492 -
493 - output = [x[0] for x in output]
494 -
495 - logger.info(output)
496 -
497 - new_beams = []
498 - for i, current_beam in enumerate(all_beams):
499 - result = output[i]
500 -
501 - if result.outputs[0].logprobs is not None:
502 - logprobs = result.outputs[0].logprobs[0]
503 - for token_id, logprob_obj in logprobs.items():
504 - new_beam = BeamSearchSequence(
505 - tokens=current_beam.tokens + [token_id],
506 - cum_logprob=current_beam.cum_logprob +
507 - logprob_obj.logprob)
508 -
509 - if token_id == tokenizer.eos_token_id and \
510 - not ignore_eos:
511 - completed.append(new_beam)
512 - else:
513 - new_beams.append(new_beam)
514 -
515 - sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
516 - all_beams = sorted_beams[:beam_width]
517 -
518 - completed.extend(all_beams)
519 - sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
520 - best_beams = sorted_completed[:beam_width]
521 -
522 - for beam in best_beams:
523 - beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
524 -
525 - beam_search_output = RequestOutput(
526 - request_id=request_id,
527 - prompt=prompt,
528 - outputs=[
529 - CompletionOutput(
530 - text=beam.text,
531 - cumulative_logprob=beam.cum_logprob,
532 - token_ids=beam.tokens,
533 - index=i,
534 - logprobs=beam.cum_logprob,
535 - ) for (i, beam) in enumerate(best_beams)
536 - ],
537 - finished=True,
538 - prompt_token_ids=tokenizedPrompt,
539 - prompt_logprobs=None)
540 -
541 - logger.info(beam_search_output)
542 -
543 - yield beam_search_output
544 -
453 @overload # DEPRECATED
454 def encode(
455 self,
vllm/engine/protocol.py
+122 -7
@@ -1,38 +1,49 @@
1 -from typing import (AsyncGenerator, List, Mapping, Optional, Protocol,
2 - runtime_checkable)
1 +import asyncio
2 +from abc import ABC, abstractmethod
3 +from typing import AsyncGenerator, List, Mapping, Optional, Union
4
5 +from vllm.beam_search import BeamSearchSequence, create_sort_beams_key_function
6 from vllm.config import DecodingConfig, ModelConfig
7 from vllm.core.scheduler import SchedulerOutputs
6 -from vllm.inputs.data import PromptType
8 +from vllm.inputs.data import PromptType, TokensPrompt
9 +from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
11 from vllm.model_executor.layers.sampler import SamplerOutput
9 -from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12 +from vllm.outputs import (CompletionOutput, EmbeddingRequestOutput,
13 + RequestOutput)
14 from vllm.pooling_params import PoolingParams
15 from vllm.prompt_adapter.request import PromptAdapterRequest
12 -from vllm.sampling_params import SamplingParams
16 +from vllm.sampling_params import BeamSearchParams, SamplingParams
17 from vllm.transformers_utils.tokenizer import AnyTokenizer
18 +from vllm.utils import collect_from_async_generator, random_uuid
19
20 +logger = init_logger(__name__)
21
16 -@runtime_checkable
17 -class EngineClient(Protocol):
22 +
23 +class EngineClient(ABC):
24 """Protocol class for Clients to Engine"""
25
26 @property
27 + @abstractmethod
28 def is_running(self) -> bool:
29 ...
30
31 @property
32 + @abstractmethod
33 def is_stopped(self) -> bool:
34 ...
35
36 @property
37 + @abstractmethod
38 def errored(self) -> bool:
39 ...
40
41 @property
42 + @abstractmethod
43 def dead_error(self) -> BaseException:
44 ...
45
46 + @abstractmethod
47 def generate(
48 self,
49 prompt: PromptType,
@@ -46,6 +57,101 @@ class EngineClient(Protocol):
57 """Generate outputs for a request."""
58 ...
59
60 + async def beam_search(
61 + self,
62 + prompt: Union[PromptType, List[int]],
63 + request_id: str,
64 + params: BeamSearchParams,
65 + ) -> AsyncGenerator[RequestOutput, None]:
66 +
67 + beam_width = params.beam_width
68 + max_tokens = params.max_tokens
69 + ignore_eos = params.ignore_eos
70 + temperature = params.temperature
71 + length_penalty = params.length_penalty
72 +
73 + tokenizer = await self.get_tokenizer(lora_request=None)
74 + tokenizedPrompt = prompt if isinstance(
75 + prompt, list) else tokenizer.encode(prompt)
76 + tokenizedLength = len(tokenizedPrompt)
77 +
78 + sort_beams_key = create_sort_beams_key_function(
79 + tokenizer.eos_token_id, length_penalty)
80 +
81 + beam_search_params = SamplingParams(logprobs=2 * beam_width,
82 + max_tokens=1,
83 + temperature=temperature)
84 + all_beams = [BeamSearchSequence(tokens=tokenizedPrompt, cum_logprob=0)]
85 + completed = []
86 +
87 + for _ in range(max_tokens):
88 + prompts_batch = [
89 + TokensPrompt(prompt_token_ids=beam.tokens)
90 + for beam in all_beams
91 + ]
92 +
93 + tasks = []
94 +
95 + request_id = f"beam_search-{random_uuid()}"
96 + for i, individual_prompt in enumerate(prompts_batch):
97 + request_id_item = f"{request_id}-{i}"
98 + task = asyncio.create_task(
99 + collect_from_async_generator(
100 + self.generate(individual_prompt, beam_search_params,
101 + request_id_item)))
102 + tasks.append(task)
103 +
104 + output = await asyncio.gather(*tasks)
105 +
106 + output = [x[0] for x in output]
107 +
108 + new_beams = []
109 + for i, current_beam in enumerate(all_beams):
110 + result = output[i]
111 +
112 + if result.outputs[0].logprobs is not None:
113 + logprobs = result.outputs[0].logprobs[0]
114 + for token_id, logprob_obj in logprobs.items():
115 + new_beam = BeamSearchSequence(
116 + tokens=current_beam.tokens + [token_id],
117 + cum_logprob=current_beam.cum_logprob +
118 + logprob_obj.logprob)
119 +
120 + if token_id == tokenizer.eos_token_id and \
121 + not ignore_eos:
122 + completed.append(new_beam)
123 + else:
124 + new_beams.append(new_beam)
125 +
126 + sorted_beams = sorted(new_beams, key=sort_beams_key, reverse=True)
127 + all_beams = sorted_beams[:beam_width]
128 +
129 + completed.extend(all_beams)
130 + sorted_completed = sorted(completed, key=sort_beams_key, reverse=True)
131 + best_beams = sorted_completed[:beam_width]
132 +
133 + for beam in best_beams:
134 + beam.text = tokenizer.decode(beam.tokens[tokenizedLength:])
135 +
136 + beam_search_output = RequestOutput(
137 + request_id=request_id,
138 + prompt=prompt,
139 + outputs=[
140 + CompletionOutput(
141 + text=beam.text,
142 + cumulative_logprob=beam.cum_logprob,
143 + token_ids=beam.tokens,
144 + index=i,
145 + logprobs=beam.cum_logprob,
146 + ) for (i, beam) in enumerate(best_beams)
147 + ],
148 + finished=True,
149 + prompt_token_ids=tokenizedPrompt,
150 + prompt_logprobs=None)
151 +
152 + yield beam_search_output
153 +
154 + @abstractmethod
155 def encode(
156 self,
157 prompt: PromptType,
@@ -58,6 +164,7 @@ class EngineClient(Protocol):
164 """Generate outputs for a request from an embedding model."""
165 ...
166
167 + @abstractmethod
168 async def abort(self, request_id: str) -> None:
169 """Abort a request.
170
@@ -65,14 +172,17 @@ class EngineClient(Protocol):
172 request_id: The unique id of the request.
173 """
174
175 + @abstractmethod
176 async def get_model_config(self) -> ModelConfig:
177 """Get the model configuration of the vLLM engine."""
178 ...
179
180 + @abstractmethod
181 async def get_decoding_config(self) -> DecodingConfig:
182 ...
183 """Get the decoding configuration of the vLLM engine."""
184
185 + @abstractmethod
186 async def get_tokenizer(
187 self,
188 lora_request: Optional[LoRARequest] = None,
@@ -80,9 +190,11 @@ class EngineClient(Protocol):
190 """Get the appropriate tokenizer for the request"""
191 ...
192
193 + @abstractmethod
194 async def is_tracing_enabled(self) -> bool:
195 ...
196
197 + @abstractmethod
198 async def do_log_stats(
199 self,
200 scheduler_outputs: Optional[SchedulerOutputs] = None,
@@ -90,14 +202,17 @@ class EngineClient(Protocol):
202 ) -> None:
203 ...
204
205 + @abstractmethod
206 async def check_health(self) -> None:
207 """Raise if unhealthy"""
208 ...
209
210 + @abstractmethod
211 async def start_profile(self) -> None:
212 """Start profiling the engine"""
213 ...
214
215 + @abstractmethod
216 async def stop_profile(self) -> None:
217 """Start profiling the engine"""
218 ...
vllm/entrypoints/openai/serving_chat.py
-7
@@ -9,8 +9,6 @@ from typing import Union
9 from fastapi import Request
10
11 from vllm.config import ModelConfig
12 -from vllm.engine.async_llm_engine import AsyncLLMEngine
13 -from vllm.engine.multiprocessing.client import MQLLMEngineClient
12 from vllm.engine.protocol import EngineClient
13 from vllm.entrypoints.chat_utils import (ConversationMessage,
14 apply_hf_chat_template,
@@ -237,11 +235,6 @@ class OpenAIServingChat(OpenAIServing):
235 log_tracing_disabled_warning()
236
237 if isinstance(sampling_params, BeamSearchParams):
240 - assert isinstance(self.engine_client,
241 - (AsyncLLMEngine,
242 - MQLLMEngineClient)), \
243 - "Beam search is only supported with" \
244 - "AsyncLLMEngine and MQLLMEngineClient."
238 result_generator = self.engine_client.beam_search(
239 engine_inputs['prompt_token_ids'],
240 request_id,
vllm/entrypoints/openai/serving_completion.py
-7
@@ -8,8 +8,6 @@ from typing import Tuple, Union, cast
8 from fastapi import Request
9
10 from vllm.config import ModelConfig
11 -from vllm.engine.async_llm_engine import AsyncLLMEngine
12 -from vllm.engine.multiprocessing.client import MQLLMEngineClient
11 from vllm.engine.protocol import EngineClient
12 from vllm.entrypoints.logger import RequestLogger
13 # yapf conflicts with isort for this block
@@ -151,11 +149,6 @@ class OpenAIServingCompletion(OpenAIServing):
149 log_tracing_disabled_warning()
150
151 if isinstance(sampling_params, BeamSearchParams):
154 - assert isinstance(self.engine_client,
155 - (AsyncLLMEngine,
156 - MQLLMEngineClient)), \
157 - "Beam search is only supported with" \
158 - "AsyncLLMEngine and MQLLMEngineClient."
152 generator = self.engine_client.beam_search(
153 prompt_inputs["prompt_token_ids"],
154 request_id_item,