A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix async log stats (#8417)
Alexander Matveev committed
Sep 12, 2024 at 23:48 UTC
68210201099e6ce1c0a1453633c77fc0185af488
2 files changed
+17
-4
tests/basic_correctness/test_preemption.py
+1
@@ -64,6 +64,7 @@ def test_chunked_prefill_recompute(
64
enable_chunked_prefill=enable_chunked_prefill,
65
max_num_seqs=max_num_seqs,
66
worker_use_ray=worker_use_ray,
67
+ disable_log_stats=False,
68
) as vllm_model:
69
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
70
assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
vllm/engine/llm_engine.py
+16
-4
@@ -1056,7 +1056,8 @@ class LLMEngine:
1056
# LLMEngine/AsyncLLMEngine directly
1057
if is_async:
1058
# Log stats.
1059
- self.do_log_stats(scheduler_outputs, outputs, finished_before)
1059
+ self.do_log_stats(scheduler_outputs, outputs, finished_before,
1060
+ skip)
1061
1062
# Tracing
1063
self.do_tracing(scheduler_outputs)
@@ -1363,18 +1364,20 @@ class LLMEngine:
1364
def do_log_stats(self,
1365
scheduler_outputs: Optional[SchedulerOutputs] = None,
1366
model_output: Optional[List[SamplerOutput]] = None,
1366
- finished_before: Optional[List[int]] = None) -> None:
1367
+ finished_before: Optional[List[int]] = None,
1368
+ skip: Optional[List[int]] = None) -> None:
1369
"""Forced log when no requests active."""
1370
if self.log_stats:
1371
stats = self._get_stats(scheduler_outputs, model_output,
1370
- finished_before)
1372
+ finished_before, skip)
1373
for logger in self.stat_loggers.values():
1374
logger.log(stats)
1375
1376
def _get_stats(self,
1377
scheduler_outputs: Optional[SchedulerOutputs],
1378
model_output: Optional[List[SamplerOutput]] = None,
1377
- finished_before: Optional[List[int]] = None) -> Stats:
1379
+ finished_before: Optional[List[int]] = None,
1380
+ skip: Optional[List[int]] = None) -> Stats:
1381
"""Get Stats to be Logged to Prometheus.
1382
1383
Args:
@@ -1382,6 +1385,10 @@ class LLMEngine:
1385
the scheduled batch,
1386
model_output: Optional, used to emit speculative decoding metrics
1387
which are created by the workers.
1388
+ finished_before: Optional, indices of sequences that were finished
1389
+ before. These sequences will be ignored.
1390
+ skip: Optional, indices of sequences that were preempted. These
1391
+ sequences will be ignored.
1392
"""
1393
now = time.time()
1394
@@ -1456,6 +1463,11 @@ class LLMEngine:
1463
actual_num_batched_tokens -= 1
1464
continue
1465
1466
+ # Currently, skip == preempted sequences, so we need to skip
1467
+ # their log stats
1468
+ if skip and idx in skip:
1469
+ continue
1470
+
1471
group_was_prefill = idx < scheduler_outputs.num_prefill_groups
1472
seq_group = scheduled_seq_group.seq_group
1473