A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[CI/Build] Replaced some models on tests for smaller ones (#9570)
Signed-off-by: Wallas Santos <wallashss@ibm.com>
Wallas Henrique committed
Oct 22, 2024 at 01:52 UTC
c0292211cea53dc5a761b3e51ce37a6c6aecd593
8 files changed
+15
-15
tests/basic_correctness/test_basic_correctness.py
+1
-1
@@ -19,7 +19,7 @@ from ..utils import multi_gpu_test
19
20
MODELS = [
21
"facebook/opt-125m",
22
- "meta-llama/Llama-2-7b-hf",
22
+ "meta-llama/Llama-3.2-1B",
23
]
24
25
TARGET_TEST_SUITE = os.environ.get("TARGET_TEST_SUITE", "L4")
tests/basic_correctness/test_chunked_prefill.py
+1
-1
@@ -16,7 +16,7 @@ from ..utils import multi_gpu_test
16
17
MODELS = [
18
"facebook/opt-125m",
19
- "meta-llama/Llama-2-7b-hf",
19
+ "meta-llama/Llama-3.2-1B",
20
]
21
22
tests/basic_correctness/test_cpu_offload.py
+2
-2
@@ -2,5 +2,5 @@ from ..utils import compare_two_settings
2
3
4
def test_cpu_offload():
5
- compare_two_settings("meta-llama/Llama-2-7b-hf", [],
6
- ["--cpu-offload-gb", "4"])
5
+ compare_two_settings("meta-llama/Llama-3.2-1B", [],
6
+ ["--cpu-offload-gb", "1"])
tests/compile/test_basic_correctness.py
+1
-2
@@ -13,8 +13,7 @@ from ..utils import compare_all_settings
13
@pytest.mark.parametrize(
14
"model, model_args, pp_size, tp_size, attn_backend, method, fullgraph",
15
[
16
- ("meta-llama/Meta-Llama-3-8B", [], 2, 2, "FLASH_ATTN", "generate",
17
- True),
16
+ ("meta-llama/Llama-3.2-1B", [], 2, 2, "FLASH_ATTN", "generate", True),
17
("nm-testing/Meta-Llama-3-8B-Instruct-W8A8-Dyn-Per-Token-2048-Samples",
18
["--quantization", "compressed-tensors"
19
], 1, 1, "FLASH_ATTN", "generate", True),
tests/entrypoints/llm/test_chat.py
+2
-2
@@ -8,7 +8,7 @@ from ..openai.test_vision import TEST_IMAGE_URLS
8
9
10
def test_chat():
11
- llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
11
+ llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct")
12
13
prompt1 = "Explain the concept of entropy."
14
messages = [
@@ -26,7 +26,7 @@ def test_chat():
26
27
28
def test_multi_chat():
29
- llm = LLM(model="meta-llama/Meta-Llama-3-8B-Instruct")
29
+ llm = LLM(model="meta-llama/Llama-3.2-1B-Instruct")
30
31
prompt1 = "Explain the concept of entropy."
32
prompt2 = "Explain what among us is."
tests/entrypoints/openai/test_chat.py
-3
@@ -16,9 +16,6 @@ from .test_completion import zephyr_lora_files # noqa: F401
16
17
# any model with a chat template should work here
18
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
19
-# technically this needs Mistral-7B-v0.1 as base, but we're not testing
20
-# generation quality here
21
-LORA_NAME = "typeof/zephyr-7b-beta-lora"
19
20
21
@pytest.fixture(scope="module")
tests/entrypoints/openai/test_shutdown.py
+1
-1
@@ -6,7 +6,7 @@ import pytest
6
7
from ...utils import RemoteOpenAIServer
8
9
-MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
9
+MODEL_NAME = "meta-llama/Llama-3.2-1B"
10
11
12
@pytest.mark.asyncio
tests/test_sharded_state_loader.py
+7
-3
@@ -46,9 +46,10 @@ def test_filter_subtensors():
46
@pytest.fixture(scope="module")
47
def llama_2_7b_files():
48
with TemporaryDirectory() as cache_dir:
49
- input_dir = snapshot_download("meta-llama/Llama-2-7b-hf",
49
+ input_dir = snapshot_download("meta-llama/Llama-3.2-1B",
50
cache_dir=cache_dir,
51
- ignore_patterns="*.bin*")
51
+ ignore_patterns=["*.bin*", "original/*"])
52
+
53
yield input_dir
54
55
@@ -58,9 +59,12 @@ def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
59
# Dump worker states to output directory
60
llm_sharded_writer.llm_engine.model_executor.save_sharded_state(
61
path=output_dir)
62
+
63
# Copy metadata files to output directory
64
for file in os.listdir(input_dir):
63
- if not any(file.endswith(ext) for ext in weights_patterns):
65
+ if not any(
66
+ file.endswith(ext) and not os.path.isdir(file)
67
+ for ext in weights_patterns):
68
shutil.copy(f"{input_dir}/{file}", output_dir)
69
70