| 1 | # For vllm script, with -t option (tensor parallel size) |
| 2 | # bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -l 1319 -t 1 |
| 3 | model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic" |
| 4 | required_gpu_arch: |
| 5 | - gfx942 |
| 6 | - gfx950 |
| 7 | tasks: |
| 8 | - name: "gsm8k" |
| 9 | metrics: |
| 10 | - name: "exact_match,strict-match" |
| 11 | value: 0.47 |
| 12 | - name: "exact_match,flexible-extract" |
| 13 | value: 0.64 |
| 14 | limit: 1319 |
| 15 | num_fewshot: 5 |