| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """This file test accuracy of the vLLM server via LMEval. |
| 4 | It uses local-completions, which interacts with vLLM |
| 5 | through the OAI API with N concurrent connections. |
| 6 | This simulates real work usage of the API and makes |
| 7 | sure that the zmq frontend mp RPC message passing and |
| 8 | AsyncLLMEngine are working correctly. |
| 9 | """ |
| 10 | |
| 11 | import lm_eval |
| 12 | |
| 13 | from vllm.platforms import current_platform |
| 14 | |
| 15 | from ....utils import RemoteOpenAIServer |
| 16 | |
| 17 | MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct" |
| 18 | NUM_CONCURRENT = 500 |
| 19 | TASK = "gsm8k" |
| 20 | FILTER = "exact_match,strict-match" |
| 21 | RTOL = 0.03 |
| 22 | EXPECTED_VALUE = 0.54 |
| 23 | DEFAULT_ARGS = ["--max-model-len", "4096"] |
| 24 | MORE_ARGS_LIST = [ |
| 25 | [], # Default |
| 26 | ["--enable-chunked-prefill"], # Chunked |
| 27 | ] |
| 28 | MAX_WAIT_SECONDS = None |
| 29 | |
| 30 | if current_platform.is_tpu(): |
| 31 | MORE_ARGS_LIST = [ |
| 32 | [], # Default |
| 33 | ] |
| 34 | MAX_WAIT_SECONDS = 600 |
| 35 | |
| 36 | |
| 37 | def run_test(more_args): |
| 38 | """Run the end to end accuracy test.""" |
| 39 | args = list(DEFAULT_ARGS) |
| 40 | args.extend(more_args) |
| 41 | print(f"Running with: {args}") |
| 42 | |
| 43 | with RemoteOpenAIServer( |
| 44 | MODEL_NAME, args, max_wait_seconds=MAX_WAIT_SECONDS |
| 45 | ) as remote_server: |
| 46 | url = f"{remote_server.url_for('v1')}/completions" |
| 47 | |
| 48 | model_args = ( |
| 49 | f"model={MODEL_NAME}," |
| 50 | f"base_url={url}," |
| 51 | f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False" |
| 52 | ) |
| 53 | |
| 54 | results = lm_eval.simple_evaluate( |
| 55 | model="local-completions", |
| 56 | model_args=model_args, |
| 57 | tasks=TASK, |
| 58 | ) |
| 59 | |
| 60 | measured_value = results["results"][TASK][FILTER] |
| 61 | assert ( |
| 62 | measured_value - RTOL < EXPECTED_VALUE |
| 63 | and measured_value + RTOL > EXPECTED_VALUE |
| 64 | ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}" |
| 65 | |
| 66 | |
| 67 | def test_lm_eval_accuracy_v1_engine(): |
| 68 | """Run with the V1 Engine.""" |
| 69 | more_args = [] |
| 70 | |
| 71 | # Limit compilation time for V1 on TPU |
| 72 | # Avoid OOM on XPU |
| 73 | if current_platform.is_tpu() or current_platform.is_xpu(): |
| 74 | more_args = ["--max-num-seqs", "64"] |
| 75 | |
| 76 | run_test(more_args) |