@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef7faad1b8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

:bug: Fixup more test failures from memory profiling (#9563)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Oct 21, 2024 at 19:10 UTC ef7faad1b8e6473556b732a7e8d5bc9be5df556f
3 files changed +13 -1
.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
2 +model_name: "neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.356
8 + - name: "exact_match,flexible-extract"
9 + value: 0.358
10 +limit: 1000
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-small.txt
+1 -1
@@ -1,6 +1,6 @@
1 Meta-Llama-3-8B-Instruct.yaml
2 Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
3 -Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
3 +Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
4 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
5 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6 Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
tests/lora/test_minicpmv.py
+1
@@ -61,6 +61,7 @@ def test_minicpmv_lora(minicpmv_lora_files):
61 max_loras=4,
62 max_lora_rank=64,
63 trust_remote_code=True,
64 + gpu_memory_utilization=0.97 # This model is pretty big for CI gpus
65 )
66
67 output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)