@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 160e1d8c99
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Misc] Log spec decode metrics (#6454)

Cody Yu committed Jul 16, 2024 at 13:37 UTC 160e1d8c99567872ef3e683e87c09f96a3a2e7c4
4 files changed +137 -14
tests/metrics/test_metrics.py
+49
@@ -168,6 +168,55 @@ def test_engine_log_metrics_regression(
168 assert_metrics(engine, disable_log_stats, len(example_prompts))
169
170
171 +@pytest.mark.parametrize("model", MODELS)
172 +@pytest.mark.parametrize("dtype", ["half"])
173 +@pytest.mark.parametrize("max_tokens", [10])
174 +def test_metric_spec_decode(
175 + vllm_runner,
176 + example_prompts,
177 + model: str,
178 + dtype: str,
179 + max_tokens: int,
180 +) -> None:
181 + k = 5
182 +
183 + with vllm_runner(model,
184 + dtype=dtype,
185 + disable_log_stats=False,
186 + gpu_memory_utilization=0.4,
187 + speculative_model=model,
188 + num_speculative_tokens=k,
189 + use_v2_block_manager=True) as vllm_model:
190 +
191 + # Force log interval to be 0 to catch all metrics.
192 + stat_logger = vllm_model.model.llm_engine.stat_loggers['prometheus']
193 + stat_logger.local_interval = 0
194 +
195 + # Note that the purpose of this test is to verify spec decode
196 + # metrics instead of functional correctness, so the expected values
197 + # are intended to be loose.
198 + metric_name_to_expected_fn = {
199 + "gauge_spec_decode_draft_acceptance_rate": lambda v: 0 <= v <= 1,
200 + "gauge_spec_decode_efficiency": lambda v: 0 <= v <= 1,
201 + "counter_spec_decode_num_accepted_tokens": lambda v: 0 <= v <= k,
202 + "counter_spec_decode_num_draft_tokens": lambda v: v == k,
203 + "counter_spec_decode_num_emitted_tokens":
204 + lambda v: 0 <= v <= k + 1,
205 + }
206 +
207 + # Use one request to better inspect the metrics.
208 + prompts = example_prompts[:1]
209 +
210 + _ = vllm_model.generate_greedy(prompts, max_tokens)
211 + for metric_name, is_expected in metric_name_to_expected_fn.items():
212 + metric_val = getattr(
213 + stat_logger.metrics,
214 + metric_name).labels(**stat_logger.labels)._value.get()
215 + assert is_expected(metric_val), (
216 + f"the value of metric {metric_name} ({metric_val}) "
217 + "does not meet expectation")
218 +
219 +
220 def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
221 num_requests: int) -> None:
222 if disable_log_stats:
tests/spec_decode/e2e/conftest.py
+36 -8
@@ -162,6 +162,11 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
162 }
163 test_name = request.node.name
164
165 + model = kwargs["model"]
166 + draft_model = kwargs.get("speculative_model", None)
167 + same_draft_target_model = (draft_model is not None
168 + and draft_model == model)
169 +
170 def generator_inner():
171
172 wait_for_gpu_memory_to_clear(
@@ -177,6 +182,13 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
182
183 print(f'Creating {baseline_or_test=} LLM for {test_name=}. {kwargs=}')
184 llm = AsyncLLM(**kwargs) if use_async else LLM(**kwargs)
185 +
186 + # Override logging interval to 0 for spec decode test run to
187 + # log all metrics in time.
188 + if (baseline_or_test == "test" and not use_async
189 + and llm.llm_engine.log_stats):
190 + for sate_logger in llm.llm_engine.stat_loggers.values():
191 + sate_logger.local_interval = 0
192 set_random_seed(seed)
193
194 yield llm
@@ -188,6 +200,9 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
200 yield llm
201 del llm
202
203 + # Set an attribute to the generator_outer function to allow us to
204 + # determine whether to further check the acceptance rate in tests.
205 + generator_outer.same_draft_target_model = same_draft_target_model # type: ignore
206 return generator_outer
207
208
@@ -204,18 +219,26 @@ def maybe_assert_ngram_worker(llm):
219
220 def get_output_from_llm_generator(
221 llm_generator, prompts,
207 - sampling_params) -> Tuple[List[str], List[List[int]]]:
222 + sampling_params) -> Tuple[List[str], List[List[int]], float]:
223 tokens: List[str] = []
224 token_ids: List[List[int]] = []
225 + acceptance_rate: float = -1.0
226 for llm in llm_generator():
227 maybe_assert_ngram_worker(llm)
228
229 outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
230 token_ids = [output.outputs[0].token_ids for output in outputs]
231 tokens = [output.outputs[0].text for output in outputs]
232 +
233 + # Fetch acceptance rate if logging is enabled.
234 + if stat_loggers := getattr(llm.llm_engine, "stat_loggers", None):
235 + stat_logger = stat_loggers["prometheus"]
236 + acceptance_rate = (stat_logger.metrics.
237 + gauge_spec_decode_draft_acceptance_rate.labels(
238 + **stat_logger.labels)._value.get())
239 del llm
240
218 - return tokens, token_ids
241 + return tokens, token_ids, acceptance_rate
242
243
244 def get_logprobs_from_llm_generator(
@@ -237,7 +260,8 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
260 batch_size,
261 max_output_len,
262 force_output_len: bool,
240 - print_tokens: bool = False):
263 + print_tokens: bool = False,
264 + ensure_all_accepted: bool = False):
265 """Helper method that compares the outputs of both the baseline LLM and
266 the test LLM. It asserts greedy equality, e.g. that the outputs are exactly
267 the same when temperature is zero.
@@ -267,12 +291,13 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
291 temperature=temperature,
292 )
293
270 - spec_batch_tokens, spec_batch_token_ids = get_output_from_llm_generator(
271 - test_llm_generator, prompts, sampling_params)
294 + (spec_batch_tokens, spec_batch_token_ids,
295 + acceptance_rate) = get_output_from_llm_generator(test_llm_generator,
296 + prompts, sampling_params)
297
273 - (baseline_batch_tokens,
274 - baseline_batch_token_ids) = get_output_from_llm_generator(
275 - baseline_llm_generator, prompts, sampling_params)
298 + (baseline_batch_tokens, baseline_batch_token_ids,
299 + _) = get_output_from_llm_generator(baseline_llm_generator, prompts,
300 + sampling_params)
301
302 assert len(baseline_batch_token_ids) == len(prompts)
303 assert len(spec_batch_token_ids) == len(prompts)
@@ -287,3 +312,6 @@ def run_greedy_equality_correctness_test(baseline_llm_generator,
312 print(f'{i=} {baseline_token_ids=}')
313 print(f'{i=} {spec_token_ids=}')
314 assert baseline_token_ids == spec_token_ids
315 +
316 + if ensure_all_accepted:
317 + assert acceptance_rate == 1.0
tests/spec_decode/e2e/test_multistep_correctness.py
+12 -6
@@ -97,7 +97,7 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
97 temperature=temperature,
98 )
99
100 - batch_tokens, batch_token_ids = get_output_from_llm_generator(
100 + batch_tokens, batch_token_ids, _ = get_output_from_llm_generator(
101 test_llm_generator, prompts, sampling_params)
102
103 # Expect a generation for each prompt in the batch.
@@ -200,12 +200,18 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
200
201 Since this test is cheaper than other e2e correctness tests, we generate
202 with a higher output_len.
203 +
204 + When the draft model is the same as the target model, we further check
205 + whether all speculative tokens are accepted.
206 """
204 - run_greedy_equality_correctness_test(baseline_llm_generator,
205 - test_llm_generator,
206 - batch_size,
207 - max_output_len=output_len,
208 - force_output_len=True)
207 + ensure_all_accepted = test_llm_generator.same_draft_target_model
208 + run_greedy_equality_correctness_test(
209 + baseline_llm_generator,
210 + test_llm_generator,
211 + batch_size,
212 + max_output_len=output_len,
213 + force_output_len=True,
214 + ensure_all_accepted=ensure_all_accepted)
215
216
217 @pytest.mark.parametrize(
vllm/engine/metrics.py
+40
@@ -133,6 +133,30 @@ class Metrics:
133 documentation="Count of successfully processed requests.",
134 labelnames=labelnames + [Metrics.labelname_finish_reason])
135
136 + # Speculatie decoding stats
137 + self.gauge_spec_decode_draft_acceptance_rate = self._base_library.Gauge(
138 + name="vllm:spec_decode_draft_acceptance_rate",
139 + documentation="Speulative token acceptance rate.",
140 + labelnames=labelnames)
141 + self.gauge_spec_decode_efficiency = self._base_library.Gauge(
142 + name="vllm:spec_decode_efficiency",
143 + documentation="Speculative decoding system efficiency.",
144 + labelnames=labelnames)
145 + self.counter_spec_decode_num_accepted_tokens = (
146 + self._base_library.Counter(
147 + name="vllm:spec_decode_num_accepted_tokens_total",
148 + documentation="Number of accepted tokens.",
149 + labelnames=labelnames))
150 + self.counter_spec_decode_num_draft_tokens = self._base_library.Counter(
151 + name="vllm:spec_decode_num_draft_tokens_total",
152 + documentation="Number of draft tokens.",
153 + labelnames=labelnames)
154 + self.counter_spec_decode_num_emitted_tokens = (
155 + self._base_library.Counter(
156 + name="vllm:spec_decode_num_emitted_tokens_total",
157 + documentation="Number of emitted tokens.",
158 + labelnames=labelnames))
159 +
160 # Deprecated in favor of vllm:prompt_tokens_total
161 self.gauge_avg_prompt_throughput = self._base_library.Gauge(
162 name="vllm:avg_prompt_throughput_toks_per_s",
@@ -454,6 +478,22 @@ class PrometheusStatLogger(StatLoggerBase):
478 self.num_generation_tokens = []
479 self.last_local_log = stats.now
480
481 + if stats.spec_decode_metrics is not None:
482 + self._log_gauge(
483 + self.metrics.gauge_spec_decode_draft_acceptance_rate,
484 + stats.spec_decode_metrics.draft_acceptance_rate)
485 + self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
486 + stats.spec_decode_metrics.system_efficiency)
487 + self._log_counter(
488 + self.metrics.counter_spec_decode_num_accepted_tokens,
489 + stats.spec_decode_metrics.accepted_tokens)
490 + self._log_counter(
491 + self.metrics.counter_spec_decode_num_draft_tokens,
492 + stats.spec_decode_metrics.draft_tokens)
493 + self._log_counter(
494 + self.metrics.counter_spec_decode_num_emitted_tokens,
495 + stats.spec_decode_metrics.emitted_tokens)
496 +
497
498 class RayPrometheusStatLogger(PrometheusStatLogger):
499 """RayPrometheusStatLogger uses Ray metrics instead."""