@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32b6816e55
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add tests for models (#922)

Woosuk Kwon committed Sep 1, 2023 at 11:19 UTC 32b6816e556f69f1672085a6267e8516bcb8e622
3 files changed +178
requirements-dev.txt
+1
@@ -10,3 +10,4 @@ types-setuptools
10
11 # testing
12 pytest
13 +pytest-forked
tests/conftest.py new
+132
@@ -0,0 +1,132 @@
1 +from typing import List, Optional, Tuple
2 +
3 +import pytest
4 +import torch
5 +from transformers import AutoModelForCausalLM
6 +
7 +from vllm import LLM, SamplingParams
8 +from vllm.transformers_utils.tokenizer import get_tokenizer
9 +
10 +_TEST_PROMPTS = [
11 + "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs.",
12 + "Briefly describe the major milestones in the development of artificial intelligence from 1950 to 2020.",
13 + "Compare and contrast artificial intelligence with human intelligence in terms of processing information.",
14 + "Describe the basic components of a neural network and how it can be trained.",
15 + "Write a short story about a robot that dreams for the first time.",
16 + "Analyze the impact of the COVID-19 pandemic on global economic structures and future business models.",
17 + "Explain the cultural significance of the Mona Lisa painting, and how its perception might vary in Western versus Eastern societies.",
18 + "Translate the following English sentence into Japanese, French, and Swahili: 'The early bird catches the worm.'",
19 +]
20 +
21 +
22 +@pytest.fixture
23 +def example_prompts() -> List[str]:
24 + return _TEST_PROMPTS
25 +
26 +
27 +_STR_DTYPE_TO_TORCH_DTYPE = {
28 + "half": torch.half,
29 + "bfloat16": torch.bfloat16,
30 + "float": torch.float,
31 +}
32 +
33 +
34 +class HfRunner:
35 +
36 + def __init__(
37 + self,
38 + model_name: str,
39 + tokenizer_name: Optional[str] = None,
40 + dtype: str = "half",
41 + ) -> None:
42 + assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
43 + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
44 + self.model = AutoModelForCausalLM.from_pretrained(
45 + model_name,
46 + torch_dtype=torch_dtype,
47 + trust_remote_code=True,
48 + ).cuda()
49 + if tokenizer_name is None:
50 + tokenizer_name = model_name
51 + self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
52 +
53 + def generate(
54 + self,
55 + prompts: List[str],
56 + **kwargs,
57 + ) -> List[Tuple[List[int], str]]:
58 + outputs: List[Tuple[List[int], str]] = []
59 + for prompt in prompts:
60 + input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
61 + output_ids = self.model.generate(
62 + input_ids.cuda(),
63 + use_cache=True,
64 + **kwargs,
65 + )
66 + output_str = self.tokenizer.batch_decode(
67 + output_ids,
68 + skip_special_tokens=True,
69 + clean_up_tokenization_spaces=False,
70 + )[0]
71 + output_ids = output_ids[0].cpu().tolist()
72 + outputs.append((output_ids, output_str))
73 + return outputs
74 +
75 + def generate_greedy(
76 + self,
77 + prompts: List[str],
78 + max_tokens: int,
79 + ) -> List[Tuple[List[int], str]]:
80 + return self.generate(prompts, do_sample=False,
81 + max_new_tokens=max_tokens)
82 +
83 +
84 +@pytest.fixture
85 +def hf_runner():
86 + return HfRunner
87 +
88 +
89 +class VllmRunner:
90 +
91 + def __init__(
92 + self,
93 + model_name: str,
94 + tokenizer_name: Optional[str] = None,
95 + dtype: str = "half",
96 + ) -> None:
97 + self.model = LLM(
98 + model=model_name,
99 + tokenizer=tokenizer_name,
100 + trust_remote_code=True,
101 + dtype=dtype,
102 + swap_space=0,
103 + )
104 +
105 + def generate(
106 + self,
107 + prompts: List[str],
108 + sampling_params: SamplingParams,
109 + ) -> List[Tuple[List[int], str]]:
110 + req_outputs = self.model.generate(
111 + prompts, sampling_params=sampling_params)
112 + outputs = []
113 + for req_output in req_outputs:
114 + prompt_str = req_output.prompt
115 + prompt_ids = req_output.prompt_token_ids
116 + output_str = req_output.outputs[0].text
117 + output_ids = req_output.outputs[0].token_ids
118 + outputs.append((prompt_ids + output_ids, prompt_str + output_str))
119 + return outputs
120 +
121 + def generate_greedy(
122 + self,
123 + prompts: List[str],
124 + max_tokens: int,
125 + ) -> List[Tuple[List[int], str]]:
126 + greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
127 + return self.generate(prompts, greedy_params)
128 +
129 +
130 +@pytest.fixture
131 +def vllm_runner():
132 + return VllmRunner
tests/models/test_models.py new
+45
@@ -0,0 +1,45 @@
1 +"""Compare the outputs of HF and vLLM when using greedy sampling.
2 +
3 +Run `pytest tests/models/test_models.py --forked`.
4 +"""
5 +import pytest
6 +
7 +MODELS = [
8 + "facebook/opt-125m",
9 + "gpt2",
10 + "bigcode/tiny_starcoder_py",
11 + "EleutherAI/gpt-j-6b",
12 + "EleutherAI/pythia-70m",
13 + "bigscience/bloom-560m",
14 + "mosaicml/mpt-7b",
15 + "tiiuae/falcon-7b",
16 + "meta-llama/Llama-2-7b-hf",
17 +]
18 +
19 +
20 +@pytest.mark.parametrize("model", MODELS)
21 +@pytest.mark.parametrize("dtype", ["half"])
22 +@pytest.mark.parametrize("max_tokens", [128])
23 +def test_models(
24 + hf_runner,
25 + vllm_runner,
26 + example_prompts,
27 + model: str,
28 + dtype: str,
29 + max_tokens: int,
30 +) -> None:
31 + hf_model = hf_runner(model, dtype=dtype)
32 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
33 + del hf_model
34 +
35 + vllm_model = vllm_runner(model, dtype=dtype)
36 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
37 + del vllm_model
38 +
39 + for i in range(len(example_prompts)):
40 + hf_output_ids, hf_output_str = hf_outputs[i]
41 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
42 + assert hf_output_str == vllm_output_str, (
43 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
44 + assert hf_output_ids == vllm_output_ids, (
45 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")