@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c882a7f5b3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[SpecDecoding] Update MLPSpeculator CI tests to use smaller model (#6714)

Nick Hill committed Jul 24, 2024 at 00:34 UTC c882a7f5b3ce5c98efb52c911ea15ca565d10cd7
1 file changed +3 -3
tests/spec_decode/e2e/test_mlp_correctness.py
+3 -3
@@ -24,14 +24,14 @@ import pytest
24 from .conftest import run_greedy_equality_correctness_test
25
26 # main model
27 -MAIN_MODEL = "ibm-granite/granite-3b-code-instruct"
27 +MAIN_MODEL = "JackFram/llama-160m"
28
29 # speculative model
30 -SPEC_MODEL = "ibm-granite/granite-3b-code-instruct-accelerator"
30 +SPEC_MODEL = "ibm-fms/llama-160m-accelerator"
31
32 # max. number of speculative tokens: this corresponds to
33 # n_predict in the config.json of the speculator model.
34 -MAX_SPEC_TOKENS = 5
34 +MAX_SPEC_TOKENS = 3
35
36 # precision
37 PRECISION = "float32"