A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Minor] Minor optimizations for update_from_output (#12454)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jan 27, 2025 at 01:09 UTC
624a1e4711cb9cfdd7e336980668e64744a84863
1 file changed
+13
-7
vllm/v1/core/scheduler.py
+13
-7
@@ -411,6 +411,10 @@ class Scheduler:
411
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
412
new_running: List[Request] = []
413
outputs: List[EngineCoreOutput] = []
414
+
415
+ # NOTE(woosuk): As len(self.running) can be up to 1K or more, the below
416
+ # loop can be a performance bottleneck. We should do our best to avoid
417
+ # expensive operations inside the loop.
418
for request in self.running:
419
req_id = request.request_id
420
request.num_computed_tokens += num_scheduled_tokens[req_id]
@@ -421,13 +425,15 @@ class Scheduler:
425
426
cached_encoder_input_ids = (
427
self.encoder_cache_manager.get_cached_input_ids(request))
424
- for input_id in list(cached_encoder_input_ids):
425
- start_pos = request.mm_positions[input_id]["offset"]
426
- num_tokens = request.mm_positions[input_id]["length"]
427
- if start_pos + num_tokens <= request.num_computed_tokens:
428
- # The encoder output is already processed and stored
429
- # in the decoder's KV cache.
430
- self.encoder_cache_manager.free(request, input_id)
428
+ # OPTIMIZATION: Avoid list(set) if the set is empty.
429
+ if cached_encoder_input_ids:
430
+ for input_id in list(cached_encoder_input_ids):
431
+ start_pos = request.mm_positions[input_id]["offset"]
432
+ num_tokens = request.mm_positions[input_id]["length"]
433
+ if start_pos + num_tokens <= request.num_computed_tokens:
434
+ # The encoder output is already processed and stored
435
+ # in the decoder's KV cache.
436
+ self.encoder_cache_manager.free(request, input_id)
437
438
if request.num_computed_tokens == request.num_tokens:
439
req_index = model_runner_output.req_id_to_index[req_id]