A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
:bug: Fixup more test failures from memory profiling (#9563)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Oct 21, 2024 at 19:10 UTC
ef7faad1b8e6473556b732a7e8d5bc9be5df556f
3 files changed
+13
-1
.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
2
+model_name: "neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.356
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.358
10
+limit: 1000
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1
-1
@@ -1,6 +1,6 @@
1
Meta-Llama-3-8B-Instruct.yaml
2
Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
3
-Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
3
+Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
4
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
5
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
tests/lora/test_minicpmv.py
+1
@@ -61,6 +61,7 @@ def test_minicpmv_lora(minicpmv_lora_files):
61
max_loras=4,
62
max_lora_rank=64,
63
trust_remote_code=True,
64
+ gpu_memory_utilization=0.97 # This model is pretty big for CI gpus
65
)
66
67
output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)