@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6545ad22e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

add option to shorten prompt print in log (#991)

Signed-off-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Lei Wen <wenlei03@qiyi.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

leiwen83 committed Sep 13, 2023 at 06:10 UTC d6545ad22ee89c8b6e3eb6cfcf8ff914a06ccee1
2 files changed +20 -2
vllm/engine/arg_utils.py
+7
@@ -171,6 +171,7 @@ class AsyncEngineArgs(EngineArgs):
171 """Arguments for asynchronous vLLM engine."""
172 engine_use_ray: bool = False
173 disable_log_requests: bool = False
174 + max_log_len: Optional[int] = None
175
176 @staticmethod
177 def add_cli_args(
@@ -183,4 +184,10 @@ class AsyncEngineArgs(EngineArgs):
184 parser.add_argument('--disable-log-requests',
185 action='store_true',
186 help='disable logging requests')
187 + parser.add_argument('--max-log-len',
188 + type=int,
189 + default=None,
190 + help='max number of prompt characters or prompt '
191 + 'ID numbers being printed in log. '
192 + 'Default: unlimited.')
193 return parser
vllm/engine/async_llm_engine.py
+13 -2
@@ -242,11 +242,13 @@ class AsyncLLMEngine:
242 engine_use_ray: bool,
243 *args,
244 log_requests: bool = True,
245 + max_log_len: Optional[int] = None,
246 start_engine_loop: bool = True,
247 **kwargs) -> None:
248 self.worker_use_ray = worker_use_ray
249 self.engine_use_ray = engine_use_ray
250 self.log_requests = log_requests
251 + self.max_log_len = max_log_len
252 self.engine = self._init_engine(*args, **kwargs)
253
254 self.request_tracker: RequestTracker = RequestTracker()
@@ -325,10 +327,18 @@ class AsyncLLMEngine:
327 arrival_time: Optional[float] = None,
328 ) -> AsyncStream:
329 if self.log_requests:
330 + shortened_prompt = prompt
331 + shortened_token_ids = prompt_token_ids
332 + if self.max_log_len is not None:
333 + if shortened_prompt is not None:
334 + shortened_prompt = shortened_prompt[:self.max_log_len]
335 + if shortened_token_ids is not None:
336 + shortened_token_ids = shortened_token_ids[:self.
337 + max_log_len]
338 logger.info(f"Received request {request_id}: "
329 - f"prompt: {prompt!r}, "
339 + f"prompt: {shortened_prompt!r}, "
340 f"sampling params: {sampling_params}, "
331 - f"prompt token ids: {prompt_token_ids}.")
341 + f"prompt token ids: {shortened_token_ids}.")
342
343 if not self.is_running:
344 if self.start_engine_loop:
@@ -446,5 +456,6 @@ class AsyncLLMEngine:
456 placement_group,
457 log_requests=not engine_args.disable_log_requests,
458 log_stats=not engine_args.disable_log_stats,
459 + max_log_len=engine_args.max_log_len,
460 start_engine_loop=start_engine_loop)
461 return engine