| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """This file test accuracy of the vLLM server via LMEval. |
| 4 | It uses local-completions, which interacts with vLLM |
| 5 | through the OAI API with N concurrent connections. |
| 6 | This simulates real work usage of the API and makes |
| 7 | sure that the zmq frontend mp RPC message passing and |
| 8 | AsyncLLMEngine are working correctly. |
| 9 | """ |
| 10 | |
| 11 | import lm_eval |
| 12 | import pytest |
| 13 | |
| 14 | from vllm.platforms import current_platform |
| 15 | |
| 16 | MODEL_NAMES = [ |
| 17 | "Qwen/Qwen3-1.7B", |
| 18 | "google/gemma-3-1b-it", |
| 19 | ] |
| 20 | FP8_KV_MODEL_NAMES = [ |
| 21 | "Qwen/Qwen3-1.7B", |
| 22 | ] |
| 23 | NUM_CONCURRENT = 500 |
| 24 | TASK = "gsm8k" |
| 25 | FILTER = "exact_match,strict-match" |
| 26 | RTOL = 0.03 |
| 27 | EXPECTED_VALUES = { |
| 28 | "Qwen/Qwen3-1.7B": 0.68, |
| 29 | "google/gemma-3-1b-it": 0.25, |
| 30 | } |
| 31 | |
| 32 | |
| 33 | def run_test(model_name, more_args=None): |
| 34 | """Run the end to end accuracy test.""" |
| 35 | model_args = f"pretrained={model_name},max_model_len=4096" |
| 36 | |
| 37 | if more_args is not None: |
| 38 | model_args = "{},{}".format(model_args, more_args) |
| 39 | |
| 40 | results = lm_eval.simple_evaluate( |
| 41 | model="vllm", |
| 42 | model_args=model_args, |
| 43 | tasks="gsm8k", |
| 44 | batch_size="auto", |
| 45 | ) |
| 46 | |
| 47 | measured_value = results["results"][TASK][FILTER] |
| 48 | assert model_name in EXPECTED_VALUES, ( |
| 49 | f"Cannot find the expected value for the model {model_name=}" |
| 50 | ) |
| 51 | expected_value = EXPECTED_VALUES[model_name] |
| 52 | assert ( |
| 53 | measured_value - RTOL < expected_value |
| 54 | and measured_value + RTOL > expected_value |
| 55 | ), f"Expected: {expected_value} | Measured: {measured_value}" |
| 56 | |
| 57 | |
| 58 | # TODO: [AlexM] Fix it with new CI/CD tests |
| 59 | TPU_TP_TEST_STR = "" # "tensor_parallel_size=4" |
| 60 | |
| 61 | |
| 62 | @pytest.mark.parametrize("model", MODEL_NAMES) |
| 63 | def test_lm_eval_accuracy_v1_engine(model): |
| 64 | """Run with the V1 Engine.""" |
| 65 | more_args = None |
| 66 | if current_platform.is_tpu(): |
| 67 | # Limit compilation time for TPU V1 |
| 68 | |
| 69 | more_args = "max_model_len=2048,max_num_seqs=64" |
| 70 | |
| 71 | # Add TP test (if provided) |
| 72 | if TPU_TP_TEST_STR: |
| 73 | more_args += ",{}".format(TPU_TP_TEST_STR) |
| 74 | |
| 75 | run_test(model, more_args) |
| 76 | |
| 77 | |
| 78 | @pytest.mark.parametrize("model", FP8_KV_MODEL_NAMES) |
| 79 | def test_lm_eval_accuracy_v1_engine_fp8_kv_cache(model): |
| 80 | """Run with the V1 Engine.""" |
| 81 | more_args = None |
| 82 | if current_platform.is_tpu(): |
| 83 | # Limit compilation time for TPU V1 |
| 84 | more_args = "max_model_len=2048,max_num_seqs=128,kv_cache_dtype=fp8" |
| 85 | |
| 86 | # Add TP test (if provided) |
| 87 | if TPU_TP_TEST_STR: |
| 88 | more_args += ",{}".format(TPU_TP_TEST_STR) |
| 89 | |
| 90 | run_test(model, more_args) |