A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V0 Deprecation] Remove V0 Tracing & Metrics tests (#25115)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Sep 17, 2025 at 19:40 UTC
2fc24e94f964368491a994641fb2921ed74cb4d4
5 files changed
+1
-508
.buildkite/test-pipeline.yaml
+1
-3
@@ -217,16 +217,14 @@ steps:
217
num_gpus: 2
218
source_file_dependencies:
219
- vllm/
220
- - tests/metrics
220
- tests/v1/tracing
221
commands:
223
- - pytest -v -s metrics
222
- "pip install \
223
'opentelemetry-sdk>=1.26.0' \
224
'opentelemetry-api>=1.26.0' \
225
'opentelemetry-exporter-otlp>=1.26.0' \
226
'opentelemetry-semantic-conventions-ai>=0.4.1'"
229
- - pytest -v -s tracing
227
+ - pytest -v -s v1/tracing
228
229
##### fast check tests #####
230
##### 1 GPU test #####
tests/metrics/test_metrics.py
deleted
-268
@@ -1,268 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-import pytest
5
-import ray
6
-from prometheus_client import REGISTRY
7
-
8
-import vllm.envs as envs
9
-from vllm import EngineArgs, LLMEngine
10
-from vllm.engine.arg_utils import AsyncEngineArgs
11
-from vllm.engine.async_llm_engine import AsyncLLMEngine
12
-from vllm.engine.metrics import RayPrometheusStatLogger
13
-from vllm.sampling_params import SamplingParams
14
-from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET
15
-
16
-
17
-@pytest.fixture(scope="function", autouse=True)
18
-def use_v0_only(monkeypatch):
19
- """
20
- This module tests V0 internals, so set VLLM_USE_V1=0.
21
- """
22
- monkeypatch.setenv('VLLM_USE_V1', '0')
23
-
24
-
25
-MODELS = [
26
- "distilbert/distilgpt2",
27
-]
28
-
29
-
30
-@pytest.mark.parametrize("model", MODELS)
31
-@pytest.mark.parametrize("dtype", ["float"])
32
-@pytest.mark.parametrize("max_tokens", [128])
33
-def test_metric_counter_prompt_tokens(
34
- vllm_runner,
35
- example_prompts,
36
- model: str,
37
- dtype: str,
38
- max_tokens: int,
39
-) -> None:
40
- with vllm_runner(model,
41
- dtype=dtype,
42
- disable_log_stats=False,
43
- gpu_memory_utilization=0.4) as vllm_model:
44
- tokenizer = vllm_model.llm.get_tokenizer()
45
- prompt_token_counts = [
46
- len(tokenizer.encode(p)) for p in example_prompts
47
- ]
48
- # This test needs at least 2 prompts in a batch of different lengths to
49
- # verify their token count is correct despite padding.
50
- assert len(example_prompts) > 1, "at least 2 prompts are required"
51
- assert prompt_token_counts[0] != prompt_token_counts[1], (
52
- "prompts of different lengths are required")
53
- vllm_prompt_token_count = sum(prompt_token_counts)
54
-
55
- _ = vllm_model.generate_greedy(example_prompts, max_tokens)
56
- stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']
57
- metric_count = stat_logger.metrics.counter_prompt_tokens.labels(
58
- **stat_logger.labels)._value.get()
59
-
60
- assert vllm_prompt_token_count == metric_count, (
61
- f"prompt token count: {vllm_prompt_token_count!r}\n"
62
- f"metric: {metric_count!r}")
63
-
64
-
65
-@pytest.mark.parametrize("model", MODELS)
66
-@pytest.mark.parametrize("dtype", ["float"])
67
-@pytest.mark.parametrize("max_tokens", [128])
68
-def test_metric_counter_generation_tokens(
69
- vllm_runner,
70
- example_prompts,
71
- model: str,
72
- dtype: str,
73
- max_tokens: int,
74
-) -> None:
75
- with vllm_runner(model,
76
- dtype=dtype,
77
- disable_log_stats=False,
78
- gpu_memory_utilization=0.4) as vllm_model:
79
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
80
- tokenizer = vllm_model.llm.get_tokenizer()
81
- stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']
82
- metric_count = stat_logger.metrics.counter_generation_tokens.labels(
83
- **stat_logger.labels)._value.get()
84
- vllm_generation_count = 0
85
- for i in range(len(example_prompts)):
86
- vllm_output_ids, vllm_output_str = vllm_outputs[i]
87
- prompt_ids = tokenizer.encode(example_prompts[i])
88
- # vllm_output_ids contains both prompt tokens and generation tokens.
89
- # We're interested only in the count of the generation tokens.
90
- vllm_generation_count += len(vllm_output_ids) - len(prompt_ids)
91
-
92
- assert vllm_generation_count == metric_count, (
93
- f"generation token count: {vllm_generation_count!r}\n"
94
- f"metric: {metric_count!r}")
95
-
96
-
97
-@pytest.mark.parametrize("model", MODELS)
98
-@pytest.mark.parametrize("dtype", ["float"])
99
-@pytest.mark.parametrize(
100
- "served_model_name",
101
- [None, [], ["ModelName0"], ["ModelName0", "ModelName1", "ModelName2"]])
102
-def test_metric_set_tag_model_name(vllm_runner, model: str, dtype: str,
103
- served_model_name: list[str]) -> None:
104
- with vllm_runner(model,
105
- dtype=dtype,
106
- disable_log_stats=False,
107
- gpu_memory_utilization=0.3,
108
- served_model_name=served_model_name) as vllm_model:
109
- stat_logger = vllm_model.llm.llm_engine.stat_loggers['prometheus']
110
- metrics_tag_content = stat_logger.labels["model_name"]
111
-
112
- if envs.VLLM_CI_USE_S3:
113
- model = f"{MODEL_WEIGHTS_S3_BUCKET}/{model}"
114
- if served_model_name is None or served_model_name == []:
115
- assert metrics_tag_content == model, (
116
- f"Metrics tag model_name is wrong! expect: {model!r}\n"
117
- f"actual: {metrics_tag_content!r}")
118
- else:
119
- assert metrics_tag_content == served_model_name[0], (
120
- f"Metrics tag model_name is wrong! expect: "
121
- f"{served_model_name[0]!r}\n"
122
- f"actual: {metrics_tag_content!r}")
123
-
124
-
125
-@pytest.mark.parametrize("model", MODELS)
126
-@pytest.mark.parametrize("dtype", ["half"])
127
-@pytest.mark.parametrize("max_tokens", [4])
128
-@pytest.mark.parametrize("disable_log_stats", [True, False])
129
-@pytest.mark.asyncio
130
-async def test_async_engine_log_metrics_regression(
131
- example_prompts,
132
- model: str,
133
- dtype: str,
134
- max_tokens: int,
135
- disable_log_stats: bool,
136
-) -> None:
137
- """
138
- Regression test ensuring async engine generates metrics
139
- when disable_log_stats=False
140
- (see: https://github.com/vllm-project/vllm/pull/4150#pullrequestreview-2008176678)
141
- """
142
- engine_args = AsyncEngineArgs(
143
- model=model,
144
- dtype=dtype,
145
- disable_log_stats=disable_log_stats,
146
- )
147
- async_engine = AsyncLLMEngine.from_engine_args(engine_args)
148
- for i, prompt in enumerate(example_prompts):
149
- results = async_engine.generate(
150
- prompt,
151
- SamplingParams(max_tokens=max_tokens),
152
- f"request-id-{i}",
153
- )
154
- # Exhaust the async iterator to make the async engine work
155
- async for _ in results:
156
- pass
157
-
158
- assert_metrics(model, async_engine.engine, disable_log_stats,
159
- len(example_prompts))
160
-
161
-
162
-@pytest.mark.parametrize("model", MODELS)
163
-@pytest.mark.parametrize("dtype", ["half"])
164
-@pytest.mark.parametrize("max_tokens", [4])
165
-@pytest.mark.parametrize("disable_log_stats", [True, False])
166
-def test_engine_log_metrics_regression(
167
- example_prompts,
168
- model: str,
169
- dtype: str,
170
- max_tokens: int,
171
- disable_log_stats: bool,
172
-) -> None:
173
- engine_args = EngineArgs(
174
- model=model,
175
- dtype=dtype,
176
- disable_log_stats=disable_log_stats,
177
- )
178
- engine = LLMEngine.from_engine_args(engine_args)
179
- for i, prompt in enumerate(example_prompts):
180
- engine.add_request(
181
- f"request-id-{i}",
182
- prompt,
183
- SamplingParams(max_tokens=max_tokens),
184
- )
185
- while engine.has_unfinished_requests():
186
- engine.step()
187
-
188
- if envs.VLLM_CI_USE_S3:
189
- model = f"{MODEL_WEIGHTS_S3_BUCKET}/{model}"
190
- assert_metrics(model, engine, disable_log_stats, len(example_prompts))
191
-
192
-
193
-def assert_metrics(model: str, engine: LLMEngine, disable_log_stats: bool,
194
- num_requests: int) -> None:
195
- if disable_log_stats:
196
- with pytest.raises(AttributeError):
197
- _ = engine.stat_loggers
198
- else:
199
- assert (engine.stat_loggers
200
- is not None), "engine.stat_loggers should be set"
201
- # Ensure the count bucket of request-level histogram metrics matches
202
- # the number of requests as a simple sanity check to ensure metrics are
203
- # generated
204
- labels = {'model_name': model}
205
- request_histogram_metrics = [
206
- "vllm:e2e_request_latency_seconds",
207
- "vllm:request_prompt_tokens",
208
- "vllm:request_generation_tokens",
209
- "vllm:request_params_n",
210
- "vllm:request_params_max_tokens",
211
- ]
212
- for metric_name in request_histogram_metrics:
213
- metric_value = REGISTRY.get_sample_value(f"{metric_name}_count",
214
- labels)
215
- assert (
216
- metric_value == num_requests), "Metrics should be collected"
217
-
218
-
219
-@pytest.mark.parametrize("model", MODELS)
220
-@pytest.mark.parametrize("dtype", ["half"])
221
-@pytest.mark.parametrize("max_tokens", [16])
222
-def test_engine_log_metrics_ray(
223
- example_prompts,
224
- model: str,
225
- dtype: str,
226
- max_tokens: int,
227
-) -> None:
228
- # This test is quite weak - it only checks that we can use
229
- # RayPrometheusStatLogger without exceptions.
230
- # Checking whether the metrics are actually emitted is unfortunately
231
- # non-trivial.
232
-
233
- # We have to run in a Ray task for Ray metrics to be emitted correctly
234
- @ray.remote(num_gpus=1)
235
- def _inner():
236
-
237
- class _RayPrometheusStatLogger(RayPrometheusStatLogger):
238
-
239
- def __init__(self, *args, **kwargs):
240
- self._i = 0
241
- super().__init__(*args, **kwargs)
242
-
243
- def log(self, *args, **kwargs):
244
- self._i += 1
245
- return super().log(*args, **kwargs)
246
-
247
- engine_args = EngineArgs(
248
- model=model,
249
- dtype=dtype,
250
- disable_log_stats=False,
251
- )
252
- engine = LLMEngine.from_engine_args(engine_args)
253
- logger = _RayPrometheusStatLogger(
254
- local_interval=0.5,
255
- labels=dict(model_name=engine.model_config.served_model_name),
256
- vllm_config=engine.vllm_config)
257
- engine.add_logger("ray", logger)
258
- for i, prompt in enumerate(example_prompts):
259
- engine.add_request(
260
- f"request-id-{i}",
261
- prompt,
262
- SamplingParams(max_tokens=max_tokens),
263
- )
264
- while engine.has_unfinished_requests():
265
- engine.step()
266
- assert logger._i > 0, ".log must be called at least once"
267
-
268
- ray.get(_inner.remote())
tests/tracing/__init__.py
tests/tracing/test_tracing.py
deleted
-237
@@ -1,237 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# ruff: noqa
4
-# type: ignore
5
-from __future__ import annotations
6
-
7
-import threading
8
-from collections.abc import Iterable
9
-from concurrent import futures
10
-from typing import Callable, Generator, Literal
11
-
12
-import grpc
13
-import pytest
14
-from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
15
- ExportTraceServiceResponse)
16
-from opentelemetry.proto.collector.trace.v1.trace_service_pb2_grpc import (
17
- TraceServiceServicer, add_TraceServiceServicer_to_server)
18
-from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue
19
-from opentelemetry.sdk.environment_variables import (
20
- OTEL_EXPORTER_OTLP_TRACES_INSECURE)
21
-
22
-from vllm import LLM, SamplingParams
23
-from vllm.tracing import SpanAttributes
24
-
25
-
26
-@pytest.fixture(scope="function", autouse=True)
27
-def use_v0_only(monkeypatch: pytest.MonkeyPatch):
28
- """
29
- Since this module is V0 only, set VLLM_USE_V1=0 for
30
- all tests in the module.
31
- """
32
- with monkeypatch.context() as m:
33
- m.setenv('VLLM_USE_V1', '0')
34
- yield
35
-
36
-
37
-FAKE_TRACE_SERVER_ADDRESS = "localhost:4317"
38
-
39
-FieldName = Literal['bool_value', 'string_value', 'int_value', 'double_value',
40
- 'array_value']
41
-
42
-
43
-def decode_value(value: AnyValue):
44
- field_decoders: dict[FieldName, Callable] = {
45
- "bool_value": (lambda v: v.bool_value),
46
- "string_value": (lambda v: v.string_value),
47
- "int_value": (lambda v: v.int_value),
48
- "double_value": (lambda v: v.double_value),
49
- "array_value":
50
- (lambda v: [decode_value(item) for item in v.array_value.values]),
51
- }
52
- for field, decoder in field_decoders.items():
53
- if value.HasField(field):
54
- return decoder(value)
55
- raise ValueError(f"Couldn't decode value: {value}")
56
-
57
-
58
-def decode_attributes(attributes: Iterable[KeyValue]):
59
- return {kv.key: decode_value(kv.value) for kv in attributes}
60
-
61
-
62
-class FakeTraceService(TraceServiceServicer):
63
-
64
- def __init__(self):
65
- self.request = None
66
- self.evt = threading.Event()
67
-
68
- def Export(self, request, context):
69
- self.request = request
70
- self.evt.set()
71
- return ExportTraceServiceResponse()
72
-
73
-
74
-@pytest.fixture
75
-def trace_service() -> Generator[FakeTraceService, None, None]:
76
- """Fixture to set up a fake gRPC trace service"""
77
- server = grpc.server(futures.ThreadPoolExecutor(max_workers=1))
78
- service = FakeTraceService()
79
- add_TraceServiceServicer_to_server(service, server)
80
- server.add_insecure_port(FAKE_TRACE_SERVER_ADDRESS)
81
- server.start()
82
-
83
- yield service
84
-
85
- server.stop(None)
86
-
87
-
88
-def test_traces(
89
- monkeypatch: pytest.MonkeyPatch,
90
- trace_service: FakeTraceService,
91
-):
92
- with monkeypatch.context() as m:
93
- m.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
94
-
95
- sampling_params = SamplingParams(
96
- temperature=0.01,
97
- top_p=0.1,
98
- max_tokens=256,
99
- )
100
- model = "facebook/opt-125m"
101
- llm = LLM(
102
- model=model,
103
- otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
104
- )
105
- prompts = ["This is a short prompt"]
106
- outputs = llm.generate(prompts, sampling_params=sampling_params)
107
-
108
- timeout = 5
109
- if not trace_service.evt.wait(timeout):
110
- raise TimeoutError(
111
- f"The fake trace service didn't receive a trace within "
112
- f"the {timeout} seconds timeout")
113
-
114
- request = trace_service.request
115
- assert len(request.resource_spans) == 1, (
116
- f"Expected 1 resource span, "
117
- f"but got {len(request.resource_spans)}")
118
- assert len(request.resource_spans[0].scope_spans) == 1, (
119
- f"Expected 1 scope span, "
120
- f"but got {len(request.resource_spans[0].scope_spans)}")
121
- assert len(request.resource_spans[0].scope_spans[0].spans) == 1, (
122
- f"Expected 1 span, "
123
- f"but got {len(request.resource_spans[0].scope_spans[0].spans)}")
124
-
125
- attributes = decode_attributes(
126
- request.resource_spans[0].scope_spans[0].spans[0].attributes)
127
- assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
128
- assert attributes.get(
129
- SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
130
- assert attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE
131
- ) == sampling_params.temperature
132
- assert attributes.get(
133
- SpanAttributes.GEN_AI_REQUEST_TOP_P) == sampling_params.top_p
134
- assert attributes.get(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS
135
- ) == sampling_params.max_tokens
136
- assert attributes.get(
137
- SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
138
- assert attributes.get(
139
- SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
140
- outputs[0].prompt_token_ids)
141
- completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
142
- assert attributes.get(
143
- SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS) == completion_tokens
144
- metrics = outputs[0].metrics
145
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE
146
- ) == metrics.time_in_queue
147
- ttft = metrics.first_token_time - metrics.arrival_time
148
- assert attributes.get(
149
- SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
150
- e2e_time = metrics.finished_time - metrics.arrival_time
151
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) == e2e_time
152
- assert metrics.scheduler_time > 0
153
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_SCHEDULER
154
- ) == metrics.scheduler_time
155
- # Model forward and model execute should be none, since detailed traces is
156
- # not enabled.
157
- assert metrics.model_forward_time is None
158
- assert metrics.model_execute_time is None
159
-
160
-
161
-def test_traces_with_detailed_steps(
162
- monkeypatch: pytest.MonkeyPatch,
163
- trace_service: FakeTraceService,
164
-):
165
- with monkeypatch.context() as m:
166
- m.setenv(OTEL_EXPORTER_OTLP_TRACES_INSECURE, "true")
167
-
168
- sampling_params = SamplingParams(
169
- temperature=0.01,
170
- top_p=0.1,
171
- max_tokens=256,
172
- )
173
- model = "facebook/opt-125m"
174
- llm = LLM(
175
- model=model,
176
- otlp_traces_endpoint=FAKE_TRACE_SERVER_ADDRESS,
177
- collect_detailed_traces=["all"],
178
- )
179
- prompts = ["This is a short prompt"]
180
- outputs = llm.generate(prompts, sampling_params=sampling_params)
181
-
182
- timeout = 5
183
- if not trace_service.evt.wait(timeout):
184
- raise TimeoutError(
185
- f"The fake trace service didn't receive a trace within "
186
- f"the {timeout} seconds timeout")
187
-
188
- request = trace_service.request
189
- assert len(request.resource_spans) == 1, (
190
- f"Expected 1 resource span, "
191
- f"but got {len(request.resource_spans)}")
192
- assert len(request.resource_spans[0].scope_spans) == 1, (
193
- f"Expected 1 scope span, "
194
- f"but got {len(request.resource_spans[0].scope_spans)}")
195
- assert len(request.resource_spans[0].scope_spans[0].spans) == 1, (
196
- f"Expected 1 span, "
197
- f"but got {len(request.resource_spans[0].scope_spans[0].spans)}")
198
-
199
- attributes = decode_attributes(
200
- request.resource_spans[0].scope_spans[0].spans[0].attributes)
201
- assert attributes.get(SpanAttributes.GEN_AI_RESPONSE_MODEL) == model
202
- assert attributes.get(
203
- SpanAttributes.GEN_AI_REQUEST_ID) == outputs[0].request_id
204
- assert attributes.get(SpanAttributes.GEN_AI_REQUEST_TEMPERATURE
205
- ) == sampling_params.temperature
206
- assert attributes.get(
207
- SpanAttributes.GEN_AI_REQUEST_TOP_P) == sampling_params.top_p
208
- assert attributes.get(SpanAttributes.GEN_AI_REQUEST_MAX_TOKENS
209
- ) == sampling_params.max_tokens
210
- assert attributes.get(
211
- SpanAttributes.GEN_AI_REQUEST_N) == sampling_params.n
212
- assert attributes.get(
213
- SpanAttributes.GEN_AI_USAGE_PROMPT_TOKENS) == len(
214
- outputs[0].prompt_token_ids)
215
- completion_tokens = sum(len(o.token_ids) for o in outputs[0].outputs)
216
- assert attributes.get(
217
- SpanAttributes.GEN_AI_USAGE_COMPLETION_TOKENS) == completion_tokens
218
- metrics = outputs[0].metrics
219
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_QUEUE
220
- ) == metrics.time_in_queue
221
- ttft = metrics.first_token_time - metrics.arrival_time
222
- assert attributes.get(
223
- SpanAttributes.GEN_AI_LATENCY_TIME_TO_FIRST_TOKEN) == ttft
224
- e2e_time = metrics.finished_time - metrics.arrival_time
225
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_E2E) == e2e_time
226
- assert metrics.scheduler_time > 0
227
- assert attributes.get(SpanAttributes.GEN_AI_LATENCY_TIME_IN_SCHEDULER
228
- ) == metrics.scheduler_time
229
- assert metrics.model_forward_time > 0
230
- assert attributes.get(
231
- SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_FORWARD
232
- ) == pytest.approx(metrics.model_forward_time / 1000)
233
- assert metrics.model_execute_time > 0
234
- assert attributes.get(
235
- SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE
236
- ) == metrics.model_execute_time
237
- assert metrics.model_forward_time < 1000 * metrics.model_execute_time