@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5e401bce17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[CI]Add regression tests to ensure the async engine generates metrics (#4524)

Ronen Schaffer committed May 2, 2024 at 05:57 UTC 5e401bce17ae9b327020ade6ba0ddceea2853451
1 file changed +94
tests/metrics/test_metrics.py
+94
@@ -1,4 +1,10 @@
1 import pytest
2 +from prometheus_client import REGISTRY
3 +
4 +from vllm import EngineArgs, LLMEngine
5 +from vllm.engine.arg_utils import AsyncEngineArgs
6 +from vllm.engine.async_llm_engine import AsyncLLMEngine
7 +from vllm.sampling_params import SamplingParams
8
9 MODELS = [
10 "facebook/opt-125m",
@@ -68,3 +74,91 @@ def test_metric_counter_generation_tokens(
74 assert vllm_generation_count == metric_count, (
75 f"generation token count: {vllm_generation_count!r}\n"
76 f"metric: {metric_count!r}")
77 +
78 +
79 +@pytest.mark.parametrize("model", MODELS)
80 +@pytest.mark.parametrize("dtype", ["half"])
81 +@pytest.mark.parametrize("max_tokens", [4])
82 +@pytest.mark.parametrize("disable_log_stats", [True, False])
83 +@pytest.mark.asyncio
84 +async def test_async_engine_log_metrics_regression(
85 + example_prompts,
86 + model: str,
87 + dtype: str,
88 + max_tokens: int,
89 + disable_log_stats: bool,
90 +) -> None:
91 + """
92 + Regression test ensuring async engine generates metrics
93 + when disable_log_stats=False
94 + (see: https://github.com/vllm-project/vllm/pull/4150#pullrequestreview-2008176678)
95 + """
96 + engine_args = AsyncEngineArgs(model=model,
97 + dtype=dtype,
98 + disable_log_stats=disable_log_stats)
99 + async_engine = AsyncLLMEngine.from_engine_args(engine_args)
100 + for i, prompt in enumerate(example_prompts):
101 + results = async_engine.generate(
102 + prompt,
103 + SamplingParams(max_tokens=max_tokens),
104 + f"request-id-{i}",
105 + )
106 + # Exhaust the async iterator to make the async engine work
107 + async for _ in results:
108 + pass
109 +
110 + assert_metrics(async_engine.engine, disable_log_stats,
111 + len(example_prompts))
112 +
113 +
114 +@pytest.mark.parametrize("model", MODELS)
115 +@pytest.mark.parametrize("dtype", ["half"])
116 +@pytest.mark.parametrize("max_tokens", [4])
117 +@pytest.mark.parametrize("disable_log_stats", [True, False])
118 +def test_engine_log_metrics_regression(
119 + example_prompts,
120 + model: str,
121 + dtype: str,
122 + max_tokens: int,
123 + disable_log_stats: bool,
124 +) -> None:
125 + engine_args = EngineArgs(model=model,
126 + dtype=dtype,
127 + disable_log_stats=disable_log_stats)
128 + engine = LLMEngine.from_engine_args(engine_args)
129 + for i, prompt in enumerate(example_prompts):
130 + engine.add_request(
131 + f"request-id-{i}",
132 + prompt,
133 + SamplingParams(max_tokens=max_tokens),
134 + )
135 + while engine.has_unfinished_requests():
136 + engine.step()
137 +
138 + assert_metrics(engine, disable_log_stats, len(example_prompts))
139 +
140 +
141 +def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
142 + num_requests: int) -> None:
143 + if disable_log_stats:
144 + with pytest.raises(AttributeError):
145 + _ = engine.stat_logger
146 + else:
147 + assert (engine.stat_logger
148 + is not None), "engine.stat_logger should be set"
149 + # Ensure the count bucket of request-level histogram metrics matches
150 + # the number of requests as a simple sanity check to ensure metrics are
151 + # generated
152 + labels = {'model_name': engine.model_config.model}
153 + request_histogram_metrics = [
154 + "vllm:e2e_request_latency_seconds",
155 + "vllm:request_prompt_tokens",
156 + "vllm:request_generation_tokens",
157 + "vllm:request_params_best_of",
158 + "vllm:request_params_n",
159 + ]
160 + for metric_name in request_histogram_metrics:
161 + metric_value = REGISTRY.get_sample_value(f"{metric_name}_count",
162 + labels)
163 + assert (
164 + metric_value == num_requests), "Metrics should be collected"