A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1] Simplify GPUModelRunner._update_states check (#13265)
Nick Hill committed
Feb 14, 2025 at 04:33 UTC
085b7b2d6c1062303a4c1c9c66132140bc9ccbe6
1 file changed
+4
-2
vllm/v1/worker/gpu_model_runner.py
+4
-2
@@ -347,6 +347,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
347
self.input_batch.block_table.append_row(req_index, start_index,
348
req_data.new_block_ids)
349
350
+ batch_changed = len(removed_req_indices) > 0 or len(req_ids_to_add) > 0
351
+
352
# Add the new or resumed requests to the persistent batch.
353
# The smaller empty indices are filled first.
354
removed_req_indices = sorted(removed_req_indices, reverse=True)
@@ -363,8 +365,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
365
# Condense the batched states if there are empty indices.
366
if removed_req_indices:
367
self.input_batch.condense(removed_req_indices)
366
- return (len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
367
- or len(scheduler_output.finished_req_ids) > 0)
368
+
369
+ return batch_changed
370
371
def _prepare_inputs(self, scheduler_output: "SchedulerOutput"):
372
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens