A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[BugFix] Fix metrics error for --num-scheduler-steps > 1 (#8234)
yulei committed
Oct 23, 2024 at 06:43 UTC
b17046e2982cad4cc205851c5af98375e0d1c3f3
2 files changed
+48
tests/metrics/test_metrics.py
+39
@@ -84,6 +84,45 @@ def test_metric_counter_generation_tokens(
84
f"metric: {metric_count!r}")
85
86
87
+@pytest.mark.parametrize("model", MODELS)
88
+@pytest.mark.parametrize("max_tokens", [128, 129])
89
+@pytest.mark.parametrize("disable_async_output_proc", [True, False])
90
+def test_metric_counter_generation_tokens_multi_step(
91
+ vllm_runner,
92
+ example_prompts,
93
+ model: str,
94
+ max_tokens: int,
95
+ disable_async_output_proc: bool,
96
+) -> None:
97
+ num_scheduler_steps = 8
98
+ with vllm_runner(
99
+ model,
100
+ disable_log_stats=False,
101
+ gpu_memory_utilization=0.4,
102
+ num_scheduler_steps=num_scheduler_steps,
103
+ disable_async_output_proc=disable_async_output_proc,
104
+ ) as vllm_model:
105
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
106
+ tokenizer = vllm_model.model.get_tokenizer()
107
+ stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
108
+ metric_count = stat_logger.metrics.counter_generation_tokens.labels(
109
+ **stat_logger.labels)._value.get()
110
+ vllm_generation_count = 0
111
+ for i in range(len(example_prompts)):
112
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
113
+ prompt_ids = tokenizer.encode(example_prompts[i])
114
+ # vllm_output_ids contains both prompt tokens and generation tokens.
115
+ # We're interested only in the count of the generation tokens.
116
+ vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
117
+
118
+ # The multi-step scheduling will continue to execute forward even when
119
+ # encountering EOS, leading to slightly imprecise metrics.
120
+ assert abs(vllm_generation_count - metric_count) <\
121
+ len(example_prompts) * num_scheduler_steps, \
122
+ (f"generation token count: {vllm_generation_count!r}\n"
123
+ f"metric: {metric_count!r}")
124
+
125
+
126
@pytest.mark.parametrize("model", MODELS)
127
@pytest.mark.parametrize("dtype", ["float"])
128
@pytest.mark.parametrize(
vllm/engine/llm_engine.py
+9
@@ -1718,6 +1718,15 @@ class LLMEngine:
1718
# TPOTs.
1719
latency = seq_group.get_last_latency(now)
1720
time_per_output_tokens_iter.append(latency)
1721
+ if seq_group.state.current_step == 0:
1722
+ # For async_output_proc, the do_log_stats()
1723
+ # is called following init_multi_step(), which
1724
+ # sets the current_step to zero.
1725
+ actual_num_batched_tokens +=\
1726
+ seq_group.state.num_steps - 1
1727
+ else:
1728
+ actual_num_batched_tokens +=\
1729
+ seq_group.state.current_step - 1
1730
1731
# Because of chunked prefill, we can have a single sequence
1732
# group that does multiple prompt_runs. To prevent logging