A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
add option to shorten prompt print in log (#991)
Signed-off-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
leiwen83 committed
Sep 13, 2023 at 06:10 UTC
d6545ad22ee89c8b6e3eb6cfcf8ff914a06ccee1
2 files changed
+20
-2
vllm/engine/arg_utils.py
+7
@@ -171,6 +171,7 @@ class AsyncEngineArgs(EngineArgs):
171
"""Arguments for asynchronous vLLM engine."""
172
engine_use_ray: bool = False
173
disable_log_requests: bool = False
174
+ max_log_len: Optional[int] = None
175
176
@staticmethod
177
def add_cli_args(
@@ -183,4 +184,10 @@ class AsyncEngineArgs(EngineArgs):
184
parser.add_argument('--disable-log-requests',
185
action='store_true',
186
help='disable logging requests')
187
+ parser.add_argument('--max-log-len',
188
+ type=int,
189
+ default=None,
190
+ help='max number of prompt characters or prompt '
191
+ 'ID numbers being printed in log. '
192
+ 'Default: unlimited.')
193
return parser
vllm/engine/async_llm_engine.py
+13
-2
@@ -242,11 +242,13 @@ class AsyncLLMEngine:
242
engine_use_ray: bool,
243
*args,
244
log_requests: bool = True,
245
+ max_log_len: Optional[int] = None,
246
start_engine_loop: bool = True,
247
**kwargs) -> None:
248
self.worker_use_ray = worker_use_ray
249
self.engine_use_ray = engine_use_ray
250
self.log_requests = log_requests
251
+ self.max_log_len = max_log_len
252
self.engine = self._init_engine(*args, **kwargs)
253
254
self.request_tracker: RequestTracker = RequestTracker()
@@ -325,10 +327,18 @@ class AsyncLLMEngine:
327
arrival_time: Optional[float] = None,
328
) -> AsyncStream:
329
if self.log_requests:
330
+ shortened_prompt = prompt
331
+ shortened_token_ids = prompt_token_ids
332
+ if self.max_log_len is not None:
333
+ if shortened_prompt is not None:
334
+ shortened_prompt = shortened_prompt[:self.max_log_len]
335
+ if shortened_token_ids is not None:
336
+ shortened_token_ids = shortened_token_ids[:self.
337
+ max_log_len]
338
logger.info(f"Received request {request_id}: "
329
- f"prompt: {prompt!r}, "
339
+ f"prompt: {shortened_prompt!r}, "
340
f"sampling params: {sampling_params}, "
331
- f"prompt token ids: {prompt_token_ids}.")
341
+ f"prompt token ids: {shortened_token_ids}.")
342
343
if not self.is_running:
344
if self.start_engine_loop:
@@ -446,5 +456,6 @@ class AsyncLLMEngine:
456
placement_group,
457
log_requests=not engine_args.disable_log_requests,
458
log_stats=not engine_args.disable_log_stats,
459
+ max_log_len=engine_args.max_log_len,
460
start_engine_loop=start_engine_loop)
461
return engine