@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b17046e298
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[BugFix] Fix metrics error for --num-scheduler-steps > 1 (#8234)

yulei committed Oct 23, 2024 at 06:43 UTC b17046e2982cad4cc205851c5af98375e0d1c3f3
2 files changed +48
tests/metrics/test_metrics.py
+39
@@ -84,6 +84,45 @@ def test_metric_counter_generation_tokens(
84 f"metric: {metric_count!r}")
85
86
87 +@pytest.mark.parametrize("model", MODELS)
88 +@pytest.mark.parametrize("max_tokens", [128, 129])
89 +@pytest.mark.parametrize("disable_async_output_proc", [True, False])
90 +def test_metric_counter_generation_tokens_multi_step(
91 + vllm_runner,
92 + example_prompts,
93 + model: str,
94 + max_tokens: int,
95 + disable_async_output_proc: bool,
96 +) -> None:
97 + num_scheduler_steps = 8
98 + with vllm_runner(
99 + model,
100 + disable_log_stats=False,
101 + gpu_memory_utilization=0.4,
102 + num_scheduler_steps=num_scheduler_steps,
103 + disable_async_output_proc=disable_async_output_proc,
104 + ) as vllm_model:
105 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
106 + tokenizer = vllm_model.model.get_tokenizer()
107 + stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
108 + metric_count = stat_logger.metrics.counter_generation_tokens.labels(
109 + **stat_logger.labels)._value.get()
110 + vllm_generation_count = 0
111 + for i in range(len(example_prompts)):
112 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
113 + prompt_ids = tokenizer.encode(example_prompts[i])
114 + # vllm_output_ids contains both prompt tokens and generation tokens.
115 + # We're interested only in the count of the generation tokens.
116 + vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
117 +
118 + # The multi-step scheduling will continue to execute forward even when
119 + # encountering EOS, leading to slightly imprecise metrics.
120 + assert abs(vllm_generation_count - metric_count) <\
121 + len(example_prompts) * num_scheduler_steps, \
122 + (f"generation token count: {vllm_generation_count!r}\n"
123 + f"metric: {metric_count!r}")
124 +
125 +
126 @pytest.mark.parametrize("model", MODELS)
127 @pytest.mark.parametrize("dtype", ["float"])
128 @pytest.mark.parametrize(
vllm/engine/llm_engine.py
+9
@@ -1718,6 +1718,15 @@ class LLMEngine:
1718 # TPOTs.
1719 latency = seq_group.get_last_latency(now)
1720 time_per_output_tokens_iter.append(latency)
1721 + if seq_group.state.current_step == 0:
1722 + # For async_output_proc, the do_log_stats()
1723 + # is called following init_multi_step(), which
1724 + # sets the current_step to zero.
1725 + actual_num_batched_tokens +=\
1726 + seq_group.state.num_steps - 1
1727 + else:
1728 + actual_num_batched_tokens +=\
1729 + seq_group.state.current_step - 1
1730
1731 # Because of chunked prefill, we can have a single sequence
1732 # group that does multiple prompt_runs. To prevent logging