@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6821020109
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix async log stats (#8417)

Alexander Matveev committed Sep 12, 2024 at 23:48 UTC 68210201099e6ce1c0a1453633c77fc0185af488
2 files changed +17 -4
tests/basic_correctness/test_preemption.py
+1
@@ -64,6 +64,7 @@ def test_chunked_prefill_recompute(
64 enable_chunked_prefill=enable_chunked_prefill,
65 max_num_seqs=max_num_seqs,
66 worker_use_ray=worker_use_ray,
67 + disable_log_stats=False,
68 ) as vllm_model:
69 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
70 assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
vllm/engine/llm_engine.py
+16 -4
@@ -1056,7 +1056,8 @@ class LLMEngine:
1056 # LLMEngine/AsyncLLMEngine directly
1057 if is_async:
1058 # Log stats.
1059 - self.do_log_stats(scheduler_outputs, outputs, finished_before)
1059 + self.do_log_stats(scheduler_outputs, outputs, finished_before,
1060 + skip)
1061
1062 # Tracing
1063 self.do_tracing(scheduler_outputs)
@@ -1363,18 +1364,20 @@ class LLMEngine:
1364 def do_log_stats(self,
1365 scheduler_outputs: Optional[SchedulerOutputs] = None,
1366 model_output: Optional[List[SamplerOutput]] = None,
1366 - finished_before: Optional[List[int]] = None) -> None:
1367 + finished_before: Optional[List[int]] = None,
1368 + skip: Optional[List[int]] = None) -> None:
1369 """Forced log when no requests active."""
1370 if self.log_stats:
1371 stats = self._get_stats(scheduler_outputs, model_output,
1370 - finished_before)
1372 + finished_before, skip)
1373 for logger in self.stat_loggers.values():
1374 logger.log(stats)
1375
1376 def _get_stats(self,
1377 scheduler_outputs: Optional[SchedulerOutputs],
1378 model_output: Optional[List[SamplerOutput]] = None,
1377 - finished_before: Optional[List[int]] = None) -> Stats:
1379 + finished_before: Optional[List[int]] = None,
1380 + skip: Optional[List[int]] = None) -> Stats:
1381 """Get Stats to be Logged to Prometheus.
1382
1383 Args:
@@ -1382,6 +1385,10 @@ class LLMEngine:
1385 the scheduled batch,
1386 model_output: Optional, used to emit speculative decoding metrics
1387 which are created by the workers.
1388 + finished_before: Optional, indices of sequences that were finished
1389 + before. These sequences will be ignored.
1390 + skip: Optional, indices of sequences that were preempted. These
1391 + sequences will be ignored.
1392 """
1393 now = time.time()
1394
@@ -1456,6 +1463,11 @@ class LLMEngine:
1463 actual_num_batched_tokens -= 1
1464 continue
1465
1466 + # Currently, skip == preempted sequences, so we need to skip
1467 + # their log stats
1468 + if skip and idx in skip:
1469 + continue
1470 +
1471 group_was_prefill = idx < scheduler_outputs.num_prefill_groups
1472 seq_group = scheduled_seq_group.seq_group
1473