A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add tests for models (#922)
Woosuk Kwon committed
Sep 1, 2023 at 11:19 UTC
32b6816e556f69f1672085a6267e8516bcb8e622
3 files changed
+178
requirements-dev.txt
+1
@@ -10,3 +10,4 @@ types-setuptools
10
11
# testing
12
pytest
13
+pytest-forked
tests/conftest.py
new
+132
@@ -0,0 +1,132 @@
1
+from typing import List, Optional, Tuple
2
+
3
+import pytest
4
+import torch
5
+from transformers import AutoModelForCausalLM
6
+
7
+from vllm import LLM, SamplingParams
8
+from vllm.transformers_utils.tokenizer import get_tokenizer
9
+
10
+_TEST_PROMPTS = [
11
+ "vLLM is a high-throughput and memory-efficient inference and serving engine for LLMs.",
12
+ "Briefly describe the major milestones in the development of artificial intelligence from 1950 to 2020.",
13
+ "Compare and contrast artificial intelligence with human intelligence in terms of processing information.",
14
+ "Describe the basic components of a neural network and how it can be trained.",
15
+ "Write a short story about a robot that dreams for the first time.",
16
+ "Analyze the impact of the COVID-19 pandemic on global economic structures and future business models.",
17
+ "Explain the cultural significance of the Mona Lisa painting, and how its perception might vary in Western versus Eastern societies.",
18
+ "Translate the following English sentence into Japanese, French, and Swahili: 'The early bird catches the worm.'",
19
+]
20
+
21
+
22
+@pytest.fixture
23
+def example_prompts() -> List[str]:
24
+ return _TEST_PROMPTS
25
+
26
+
27
+_STR_DTYPE_TO_TORCH_DTYPE = {
28
+ "half": torch.half,
29
+ "bfloat16": torch.bfloat16,
30
+ "float": torch.float,
31
+}
32
+
33
+
34
+class HfRunner:
35
+
36
+ def __init__(
37
+ self,
38
+ model_name: str,
39
+ tokenizer_name: Optional[str] = None,
40
+ dtype: str = "half",
41
+ ) -> None:
42
+ assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
43
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
44
+ self.model = AutoModelForCausalLM.from_pretrained(
45
+ model_name,
46
+ torch_dtype=torch_dtype,
47
+ trust_remote_code=True,
48
+ ).cuda()
49
+ if tokenizer_name is None:
50
+ tokenizer_name = model_name
51
+ self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
52
+
53
+ def generate(
54
+ self,
55
+ prompts: List[str],
56
+ **kwargs,
57
+ ) -> List[Tuple[List[int], str]]:
58
+ outputs: List[Tuple[List[int], str]] = []
59
+ for prompt in prompts:
60
+ input_ids = self.tokenizer(prompt, return_tensors="pt").input_ids
61
+ output_ids = self.model.generate(
62
+ input_ids.cuda(),
63
+ use_cache=True,
64
+ **kwargs,
65
+ )
66
+ output_str = self.tokenizer.batch_decode(
67
+ output_ids,
68
+ skip_special_tokens=True,
69
+ clean_up_tokenization_spaces=False,
70
+ )[0]
71
+ output_ids = output_ids[0].cpu().tolist()
72
+ outputs.append((output_ids, output_str))
73
+ return outputs
74
+
75
+ def generate_greedy(
76
+ self,
77
+ prompts: List[str],
78
+ max_tokens: int,
79
+ ) -> List[Tuple[List[int], str]]:
80
+ return self.generate(prompts, do_sample=False,
81
+ max_new_tokens=max_tokens)
82
+
83
+
84
+@pytest.fixture
85
+def hf_runner():
86
+ return HfRunner
87
+
88
+
89
+class VllmRunner:
90
+
91
+ def __init__(
92
+ self,
93
+ model_name: str,
94
+ tokenizer_name: Optional[str] = None,
95
+ dtype: str = "half",
96
+ ) -> None:
97
+ self.model = LLM(
98
+ model=model_name,
99
+ tokenizer=tokenizer_name,
100
+ trust_remote_code=True,
101
+ dtype=dtype,
102
+ swap_space=0,
103
+ )
104
+
105
+ def generate(
106
+ self,
107
+ prompts: List[str],
108
+ sampling_params: SamplingParams,
109
+ ) -> List[Tuple[List[int], str]]:
110
+ req_outputs = self.model.generate(
111
+ prompts, sampling_params=sampling_params)
112
+ outputs = []
113
+ for req_output in req_outputs:
114
+ prompt_str = req_output.prompt
115
+ prompt_ids = req_output.prompt_token_ids
116
+ output_str = req_output.outputs[0].text
117
+ output_ids = req_output.outputs[0].token_ids
118
+ outputs.append((prompt_ids + output_ids, prompt_str + output_str))
119
+ return outputs
120
+
121
+ def generate_greedy(
122
+ self,
123
+ prompts: List[str],
124
+ max_tokens: int,
125
+ ) -> List[Tuple[List[int], str]]:
126
+ greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
127
+ return self.generate(prompts, greedy_params)
128
+
129
+
130
+@pytest.fixture
131
+def vllm_runner():
132
+ return VllmRunner
tests/models/test_models.py
new
+45
@@ -0,0 +1,45 @@
1
+"""Compare the outputs of HF and vLLM when using greedy sampling.
2
+
3
+Run `pytest tests/models/test_models.py --forked`.
4
+"""
5
+import pytest
6
+
7
+MODELS = [
8
+ "facebook/opt-125m",
9
+ "gpt2",
10
+ "bigcode/tiny_starcoder_py",
11
+ "EleutherAI/gpt-j-6b",
12
+ "EleutherAI/pythia-70m",
13
+ "bigscience/bloom-560m",
14
+ "mosaicml/mpt-7b",
15
+ "tiiuae/falcon-7b",
16
+ "meta-llama/Llama-2-7b-hf",
17
+]
18
+
19
+
20
+@pytest.mark.parametrize("model", MODELS)
21
+@pytest.mark.parametrize("dtype", ["half"])
22
+@pytest.mark.parametrize("max_tokens", [128])
23
+def test_models(
24
+ hf_runner,
25
+ vllm_runner,
26
+ example_prompts,
27
+ model: str,
28
+ dtype: str,
29
+ max_tokens: int,
30
+) -> None:
31
+ hf_model = hf_runner(model, dtype=dtype)
32
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
33
+ del hf_model
34
+
35
+ vllm_model = vllm_runner(model, dtype=dtype)
36
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
37
+ del vllm_model
38
+
39
+ for i in range(len(example_prompts)):
40
+ hf_output_ids, hf_output_str = hf_outputs[i]
41
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
42
+ assert hf_output_str == vllm_output_str, (
43
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
44
+ assert hf_output_ids == vllm_output_ids, (
45
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")