@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e433c115bc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix `vllm:prompt_tokens_total` metric calculation (#2869)

Ronen Schaffer committed Feb 19, 2024 at 09:55 UTC e433c115bce2bf27f7b1abdde7029566007d9eee
3 files changed +41 -6
tests/conftest.py
+5 -5
@@ -13,12 +13,10 @@ _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
13 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
14
15
16 -def _read_prompts(filename: str) -> str:
17 - prompts = []
16 +def _read_prompts(filename: str) -> List[str]:
17 with open(filename, "r") as f:
19 - prompt = f.readline()
20 - prompts.append(prompt)
21 - return prompts
18 + prompts = f.readlines()
19 + return prompts
20
21
22 @pytest.fixture
@@ -165,6 +163,7 @@ class VllmRunner:
163 model_name: str,
164 tokenizer_name: Optional[str] = None,
165 dtype: str = "half",
166 + disable_log_stats: bool = True,
167 tensor_parallel_size: int = 1,
168 ) -> None:
169 self.model = LLM(
@@ -173,6 +172,7 @@ class VllmRunner:
172 trust_remote_code=True,
173 dtype=dtype,
174 swap_space=0,
175 + disable_log_stats=disable_log_stats,
176 tensor_parallel_size=tensor_parallel_size,
177 )
178
tests/metrics/test_metrics.py new
+33
@@ -0,0 +1,33 @@
1 +import pytest
2 +import vllm.engine.metrics
3 +
4 +MODELS = [
5 + "facebook/opt-125m",
6 +]
7 +
8 +
9 +@pytest.mark.parametrize("model", MODELS)
10 +@pytest.mark.parametrize("dtype", ["float"])
11 +@pytest.mark.parametrize("max_tokens", [128])
12 +def test_metrics(
13 + vllm_runner,
14 + example_prompts,
15 + model: str,
16 + dtype: str,
17 + max_tokens: int,
18 +) -> None:
19 + vllm_model = vllm_runner(model, dtype=dtype, disable_log_stats=False)
20 + tokenizer = vllm_model.model.get_tokenizer()
21 + prompt_token_counts = [len(tokenizer.encode(p)) for p in example_prompts]
22 + # This test needs at least 2 prompts in a batch of different lengths to verify their token count is correct despite padding.
23 + assert len(example_prompts) > 1, "at least 2 prompts are required"
24 + assert prompt_token_counts[0] != prompt_token_counts[1], (
25 + "prompts of different lengths are required")
26 + vllm_prompt_token_count = sum(prompt_token_counts)
27 +
28 + _ = vllm_model.generate_greedy(example_prompts, max_tokens)
29 + metric_count = vllm.engine.metrics.counter_prompt_tokens.get_value({})
30 +
31 + assert vllm_prompt_token_count == metric_count, (
32 + f"prompt token count: {vllm_prompt_token_count!r}\nmetric: {metric_count!r}"
33 + )
vllm/engine/llm_engine.py
+3 -1
@@ -867,7 +867,9 @@ class LLMEngine:
867
868 # Number of Tokens.
869 if prompt_run:
870 - num_prompt_tokens = scheduler_outputs.num_batched_tokens
870 + num_prompt_tokens = sum(
871 + len(seq_group.prompt_token_ids)
872 + for seq_group in scheduler_outputs.scheduled_seq_groups)
873 else:
874 num_generation_tokens = scheduler_outputs.num_batched_tokens
875