A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[CI]Add regression tests to ensure the async engine generates metrics (#4524)
Ronen Schaffer committed
May 2, 2024 at 05:57 UTC
5e401bce17ae9b327020ade6ba0ddceea2853451
1 file changed
+94
tests/metrics/test_metrics.py
+94
@@ -1,4 +1,10 @@
1
import pytest
2
+from prometheus_client import REGISTRY
3
+
4
+from vllm import EngineArgs, LLMEngine
5
+from vllm.engine.arg_utils import AsyncEngineArgs
6
+from vllm.engine.async_llm_engine import AsyncLLMEngine
7
+from vllm.sampling_params import SamplingParams
8
9
MODELS = [
10
"facebook/opt-125m",
@@ -68,3 +74,91 @@ def test_metric_counter_generation_tokens(
74
assert vllm_generation_count == metric_count, (
75
f"generation token count: {vllm_generation_count!r}\n"
76
f"metric: {metric_count!r}")
77
+
78
+
79
+@pytest.mark.parametrize("model", MODELS)
80
+@pytest.mark.parametrize("dtype", ["half"])
81
+@pytest.mark.parametrize("max_tokens", [4])
82
+@pytest.mark.parametrize("disable_log_stats", [True, False])
83
+@pytest.mark.asyncio
84
+async def test_async_engine_log_metrics_regression(
85
+ example_prompts,
86
+ model: str,
87
+ dtype: str,
88
+ max_tokens: int,
89
+ disable_log_stats: bool,
90
+) -> None:
91
+ """
92
+ Regression test ensuring async engine generates metrics
93
+ when disable_log_stats=False
94
+ (see: https://github.com/vllm-project/vllm/pull/4150#pullrequestreview-2008176678)
95
+ """
96
+ engine_args = AsyncEngineArgs(model=model,
97
+ dtype=dtype,
98
+ disable_log_stats=disable_log_stats)
99
+ async_engine = AsyncLLMEngine.from_engine_args(engine_args)
100
+ for i, prompt in enumerate(example_prompts):
101
+ results = async_engine.generate(
102
+ prompt,
103
+ SamplingParams(max_tokens=max_tokens),
104
+ f"request-id-{i}",
105
+ )
106
+ # Exhaust the async iterator to make the async engine work
107
+ async for _ in results:
108
+ pass
109
+
110
+ assert_metrics(async_engine.engine, disable_log_stats,
111
+ len(example_prompts))
112
+
113
+
114
+@pytest.mark.parametrize("model", MODELS)
115
+@pytest.mark.parametrize("dtype", ["half"])
116
+@pytest.mark.parametrize("max_tokens", [4])
117
+@pytest.mark.parametrize("disable_log_stats", [True, False])
118
+def test_engine_log_metrics_regression(
119
+ example_prompts,
120
+ model: str,
121
+ dtype: str,
122
+ max_tokens: int,
123
+ disable_log_stats: bool,
124
+) -> None:
125
+ engine_args = EngineArgs(model=model,
126
+ dtype=dtype,
127
+ disable_log_stats=disable_log_stats)
128
+ engine = LLMEngine.from_engine_args(engine_args)
129
+ for i, prompt in enumerate(example_prompts):
130
+ engine.add_request(
131
+ f"request-id-{i}",
132
+ prompt,
133
+ SamplingParams(max_tokens=max_tokens),
134
+ )
135
+ while engine.has_unfinished_requests():
136
+ engine.step()
137
+
138
+ assert_metrics(engine, disable_log_stats, len(example_prompts))
139
+
140
+
141
+def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
142
+ num_requests: int) -> None:
143
+ if disable_log_stats:
144
+ with pytest.raises(AttributeError):
145
+ _ = engine.stat_logger
146
+ else:
147
+ assert (engine.stat_logger
148
+ is not None), "engine.stat_logger should be set"
149
+ # Ensure the count bucket of request-level histogram metrics matches
150
+ # the number of requests as a simple sanity check to ensure metrics are
151
+ # generated
152
+ labels = {'model_name': engine.model_config.model}
153
+ request_histogram_metrics = [
154
+ "vllm:e2e_request_latency_seconds",
155
+ "vllm:request_prompt_tokens",
156
+ "vllm:request_generation_tokens",
157
+ "vllm:request_params_best_of",
158
+ "vllm:request_params_n",
159
+ ]
160
+ for metric_name in request_histogram_metrics:
161
+ metric_value = REGISTRY.get_sample_value(f"{metric_name}_count",
162
+ labels)
163
+ assert (
164
+ metric_value == num_requests), "Metrics should be collected"