A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix `vllm:prompt_tokens_total` metric calculation (#2869)
Ronen Schaffer committed
Feb 19, 2024 at 09:55 UTC
e433c115bce2bf27f7b1abdde7029566007d9eee
3 files changed
+41
-6
tests/conftest.py
+5
-5
@@ -13,12 +13,10 @@ _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
13
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
14
15
16
-def _read_prompts(filename: str) -> str:
17
- prompts = []
16
+def _read_prompts(filename: str) -> List[str]:
17
with open(filename, "r") as f:
19
- prompt = f.readline()
20
- prompts.append(prompt)
21
- return prompts
18
+ prompts = f.readlines()
19
+ return prompts
20
21
22
@pytest.fixture
@@ -165,6 +163,7 @@ class VllmRunner:
163
model_name: str,
164
tokenizer_name: Optional[str] = None,
165
dtype: str = "half",
166
+ disable_log_stats: bool = True,
167
tensor_parallel_size: int = 1,
168
) -> None:
169
self.model = LLM(
@@ -173,6 +172,7 @@ class VllmRunner:
172
trust_remote_code=True,
173
dtype=dtype,
174
swap_space=0,
175
+ disable_log_stats=disable_log_stats,
176
tensor_parallel_size=tensor_parallel_size,
177
)
178
tests/metrics/test_metrics.py
new
+33
@@ -0,0 +1,33 @@
1
+import pytest
2
+import vllm.engine.metrics
3
+
4
+MODELS = [
5
+ "facebook/opt-125m",
6
+]
7
+
8
+
9
+@pytest.mark.parametrize("model", MODELS)
10
+@pytest.mark.parametrize("dtype", ["float"])
11
+@pytest.mark.parametrize("max_tokens", [128])
12
+def test_metrics(
13
+ vllm_runner,
14
+ example_prompts,
15
+ model: str,
16
+ dtype: str,
17
+ max_tokens: int,
18
+) -> None:
19
+ vllm_model = vllm_runner(model, dtype=dtype, disable_log_stats=False)
20
+ tokenizer = vllm_model.model.get_tokenizer()
21
+ prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
22
+ # This test needs at least 2 prompts in a batch of different lengths to verify their token count is correct despite padding.
23
+ assert len(example_prompts) > 1, "at least 2 prompts are required"
24
+ assert prompt_token_counts[0] != prompt_token_counts[1], (
25
+ "prompts of different lengths are required")
26
+ vllm_prompt_token_count = sum(prompt_token_counts)
27
+
28
+ _ = vllm_model.generate_greedy(example_prompts, max_tokens)
29
+ metric_count = vllm.engine.metrics.counter_prompt_tokens.get_value({})
30
+
31
+ assert vllm_prompt_token_count == metric_count, (
32
+ f"prompt token count: {vllm_prompt_token_count!r}\nmetric: {metric_count!r}"
33
+ )
vllm/engine/llm_engine.py
+3
-1
@@ -867,7 +867,9 @@ class LLMEngine:
867
868
# Number of Tokens.
869
if prompt_run:
870
- num_prompt_tokens = scheduler_outputs.num_batched_tokens
870
+ num_prompt_tokens = sum(
871
+ len(seq_group.prompt_token_ids)
872
+ for seq_group in scheduler_outputs.scheduled_seq_groups)
873
else:
874
num_generation_tokens = scheduler_outputs.num_batched_tokens
875