A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[SpecDecoding] Update MLPSpeculator CI tests to use smaller model (#6714)
Nick Hill committed
Jul 24, 2024 at 00:34 UTC
c882a7f5b3ce5c98efb52c911ea15ca565d10cd7
1 file changed
+3
-3
tests/spec_decode/e2e/test_mlp_correctness.py
+3
-3
@@ -24,14 +24,14 @@ import pytest
24
from .conftest import run_greedy_equality_correctness_test
25
26
# main model
27
-MAIN_MODEL = "ibm-granite/granite-3b-code-instruct"
27
+MAIN_MODEL = "JackFram/llama-160m"
28
29
# speculative model
30
-SPEC_MODEL = "ibm-granite/granite-3b-code-instruct-accelerator"
30
+SPEC_MODEL = "ibm-fms/llama-160m-accelerator"
31
32
# max. number of speculative tokens: this corresponds to
33
# n_predict in the config.json of the speculator model.
34
-MAX_SPEC_TOKENS = 5
34
+MAX_SPEC_TOKENS = 3
35
36
# precision
37
PRECISION = "float32"