@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 624a1e4711
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Minor] Minor optimizations for update_from_output (#12454)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 27, 2025 at 01:09 UTC 624a1e4711cb9cfdd7e336980668e64744a84863
1 file changed +13 -7
vllm/v1/core/scheduler.py
+13 -7
@@ -411,6 +411,10 @@ class Scheduler:
411 num_scheduled_tokens = scheduler_output.num_scheduled_tokens
412 new_running: List[Request] = []
413 outputs: List[EngineCoreOutput] = []
414 +
415 + # NOTE(woosuk): As len(self.running) can be up to 1K or more, the below
416 + # loop can be a performance bottleneck. We should do our best to avoid
417 + # expensive operations inside the loop.
418 for request in self.running:
419 req_id = request.request_id
420 request.num_computed_tokens += num_scheduled_tokens[req_id]
@@ -421,13 +425,15 @@ class Scheduler:
425
426 cached_encoder_input_ids = (
427 self.encoder_cache_manager.get_cached_input_ids(request))
424 - for input_id in list(cached_encoder_input_ids):
425 - start_pos = request.mm_positions[input_id]["offset"]
426 - num_tokens = request.mm_positions[input_id]["length"]
427 - if start_pos + num_tokens <= request.num_computed_tokens:
428 - # The encoder output is already processed and stored
429 - # in the decoder's KV cache.
430 - self.encoder_cache_manager.free(request, input_id)
428 + # OPTIMIZATION: Avoid list(set) if the set is empty.
429 + if cached_encoder_input_ids:
430 + for input_id in list(cached_encoder_input_ids):
431 + start_pos = request.mm_positions[input_id]["offset"]
432 + num_tokens = request.mm_positions[input_id]["length"]
433 + if start_pos + num_tokens <= request.num_computed_tokens:
434 + # The encoder output is already processed and stored
435 + # in the decoder's KV cache.
436 + self.encoder_cache_manager.free(request, input_id)
437
438 if request.num_computed_tokens == request.num_tokens:
439 req_index = model_runner_output.req_id_to_index[req_id]