A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Misc][VLM][Doc] Consolidate offline examples for vision language models (#6858)
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Roger Wang committed
Jul 26, 2024 at 22:44 UTC
aa46953a20685377fc51dcde172114ddd7ffdc68
7 files changed
+174
-212
examples/fuyu_example.py
deleted
-31
@@ -1,31 +0,0 @@
1
-import requests
2
-from PIL import Image
3
-
4
-from vllm import LLM, SamplingParams
5
-
6
-
7
-def run_fuyu():
8
- llm = LLM(model="adept/fuyu-8b", max_model_len=4096)
9
-
10
- # single-image prompt
11
- prompt = "What is the highest life expectancy at of male?\n"
12
- url = "https://huggingface.co/adept/fuyu-8b/resolve/main/chart.png"
13
- image = Image.open(requests.get(url, stream=True).raw)
14
- sampling_params = SamplingParams(temperature=0, max_tokens=64)
15
-
16
- outputs = llm.generate(
17
- {
18
- "prompt": prompt,
19
- "multi_modal_data": {
20
- "image": image
21
- },
22
- },
23
- sampling_params=sampling_params)
24
-
25
- for o in outputs:
26
- generated_text = o.outputs[0].text
27
- print(generated_text)
28
-
29
-
30
-if __name__ == "__main__":
31
- run_fuyu()
examples/llava_example.py
deleted
-25
@@ -1,25 +0,0 @@
1
-from vllm import LLM
2
-from vllm.assets.image import ImageAsset
3
-
4
-
5
-def run_llava():
6
- llm = LLM(model="llava-hf/llava-1.5-7b-hf")
7
-
8
- prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
9
-
10
- image = ImageAsset("stop_sign").pil_image
11
-
12
- outputs = llm.generate({
13
- "prompt": prompt,
14
- "multi_modal_data": {
15
- "image": image
16
- },
17
- })
18
-
19
- for o in outputs:
20
- generated_text = o.outputs[0].text
21
- print(generated_text)
22
-
23
-
24
-if __name__ == "__main__":
25
- run_llava()
examples/llava_next_example.py
deleted
-36
@@ -1,36 +0,0 @@
1
-from io import BytesIO
2
-
3
-import requests
4
-from PIL import Image
5
-
6
-from vllm import LLM, SamplingParams
7
-
8
-
9
-def run_llava_next():
10
- llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=4096)
11
-
12
- prompt = "[INST] <image>\nWhat is shown in this image? [/INST]"
13
- url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
14
- image = Image.open(BytesIO(requests.get(url).content))
15
- sampling_params = SamplingParams(temperature=0.8,
16
- top_p=0.95,
17
- max_tokens=100)
18
-
19
- outputs = llm.generate(
20
- {
21
- "prompt": prompt,
22
- "multi_modal_data": {
23
- "image": image
24
- }
25
- },
26
- sampling_params=sampling_params)
27
-
28
- generated_text = ""
29
- for o in outputs:
30
- generated_text += o.outputs[0].text
31
-
32
- print(f"LLM output:{generated_text}")
33
-
34
-
35
-if __name__ == "__main__":
36
- run_llava_next()
examples/minicpmv_example.py
deleted
-55
@@ -1,55 +0,0 @@
1
-from transformers import AutoTokenizer
2
-
3
-from vllm import LLM, SamplingParams
4
-from vllm.assets.image import ImageAsset
5
-
6
-# 2.0
7
-# The official repo doesn't work yet, so we need to use a fork for now
8
-# For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
9
-# MODEL_NAME = "HwwwH/MiniCPM-V-2"
10
-# 2.5
11
-MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"
12
-
13
-image = ImageAsset("stop_sign").pil_image.convert("RGB")
14
-
15
-tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
16
-llm = LLM(model=MODEL_NAME,
17
- gpu_memory_utilization=1,
18
- trust_remote_code=True,
19
- max_model_len=4096)
20
-
21
-messages = [{
22
- 'role':
23
- 'user',
24
- 'content':
25
- '(<image>./</image>)\n' + "What's the content of the image?"
26
-}]
27
-prompt = tokenizer.apply_chat_template(messages,
28
- tokenize=False,
29
- add_generation_prompt=True)
30
-# 2.0
31
-# stop_token_ids = [tokenizer.eos_id]
32
-# 2.5
33
-stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
34
-
35
-sampling_params = SamplingParams(
36
- stop_token_ids=stop_token_ids,
37
- # temperature=0.7,
38
- # top_p=0.8,
39
- # top_k=100,
40
- # seed=3472,
41
- max_tokens=1024,
42
- # min_tokens=150,
43
- temperature=0,
44
- use_beam_search=True,
45
- # length_penalty=1.2,
46
- best_of=3)
47
-
48
-outputs = llm.generate({
49
- "prompt": prompt,
50
- "multi_modal_data": {
51
- "image": image
52
- }
53
-},
54
- sampling_params=sampling_params)
55
-print(outputs[0].outputs[0].text)
examples/offline_inference_vision_language.py
new
+174
@@ -0,0 +1,174 @@
1
+"""
2
+This example shows how to use vLLM for running offline inference
3
+with the correct prompt format on vision language models.
4
+
5
+For most models, the prompt format should follow corresponding examples
6
+on HuggingFace model repository.
7
+"""
8
+from transformers import AutoTokenizer
9
+
10
+from vllm import LLM, SamplingParams
11
+from vllm.assets.image import ImageAsset
12
+from vllm.utils import FlexibleArgumentParser
13
+
14
+# Input image and question
15
+image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
16
+question = "What is the content of this image?"
17
+
18
+
19
+# LLaVA-1.5
20
+def run_llava(question):
21
+
22
+ prompt = f"USER: <image>\n{question}\nASSISTANT:"
23
+
24
+ llm = LLM(model="llava-hf/llava-1.5-7b-hf")
25
+
26
+ return llm, prompt
27
+
28
+
29
+# LLaVA-1.6/LLaVA-NeXT
30
+def run_llava_next(question):
31
+
32
+ prompt = f"[INST] <image>\n{question} [/INST]"
33
+ llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf")
34
+
35
+ return llm, prompt
36
+
37
+
38
+# Fuyu
39
+def run_fuyu(question):
40
+
41
+ prompt = f"{question}\n"
42
+ llm = LLM(model="adept/fuyu-8b")
43
+
44
+ return llm, prompt
45
+
46
+
47
+# Phi-3-Vision
48
+def run_phi3v(question):
49
+
50
+ prompt = f"<|user|>\n<|image_1|>\n{question}<|end|>\n<|assistant|>\n" # noqa: E501
51
+ # Note: The default setting of max_num_seqs (256) and
52
+ # max_model_len (128k) for this model may cause OOM.
53
+ # You may lower either to run this example on lower-end GPUs.
54
+
55
+ # In this example, we override max_num_seqs to 5 while
56
+ # keeping the original context length of 128k.
57
+ llm = LLM(
58
+ model="microsoft/Phi-3-vision-128k-instruct",
59
+ trust_remote_code=True,
60
+ max_num_seqs=5,
61
+ )
62
+ return llm, prompt
63
+
64
+
65
+# PaliGemma
66
+def run_paligemma(question):
67
+
68
+ prompt = question
69
+ llm = LLM(model="google/paligemma-3b-mix-224")
70
+
71
+ return llm, prompt
72
+
73
+
74
+# Chameleon
75
+def run_chameleon(question):
76
+
77
+ prompt = f"{question}<image>"
78
+ llm = LLM(model="facebook/chameleon-7b")
79
+ return llm, prompt
80
+
81
+
82
+# MiniCPM-V
83
+def run_minicpmv(question):
84
+
85
+ # 2.0
86
+ # The official repo doesn't work yet, so we need to use a fork for now
87
+ # For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # noqa
88
+ # model_name = "HwwwH/MiniCPM-V-2"
89
+
90
+ # 2.5
91
+ model_name = "openbmb/MiniCPM-Llama3-V-2_5"
92
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
93
+ trust_remote_code=True)
94
+ llm = LLM(
95
+ model=model_name,
96
+ trust_remote_code=True,
97
+ )
98
+
99
+ messages = [{
100
+ 'role': 'user',
101
+ 'content': f'(<image>./</image>)\n{question}'
102
+ }]
103
+ prompt = tokenizer.apply_chat_template(messages,
104
+ tokenize=False,
105
+ add_generation_prompt=True)
106
+ return llm, prompt
107
+
108
+
109
+model_example_map = {
110
+ "llava": run_llava,
111
+ "llava-next": run_llava_next,
112
+ "fuyu": run_fuyu,
113
+ "phi3_v": run_phi3v,
114
+ "paligemma": run_paligemma,
115
+ "chameleon": run_chameleon,
116
+ "minicpmv": run_minicpmv,
117
+}
118
+
119
+
120
+def main(args):
121
+ model = args.model_type
122
+ if model not in model_example_map:
123
+ raise ValueError(f"Model type {model} is not supported.")
124
+
125
+ llm, prompt = model_example_map[model](question)
126
+
127
+ # We set temperature to 0.2 so that outputs can be different
128
+ # even when all prompts are identical when running batch inference.
129
+ sampling_params = SamplingParams(temperature=0.2, max_tokens=64)
130
+
131
+ assert args.num_prompts > 0
132
+ if args.num_prompts == 1:
133
+ # Single inference
134
+ inputs = {
135
+ "prompt": prompt,
136
+ "multi_modal_data": {
137
+ "image": image
138
+ },
139
+ }
140
+
141
+ else:
142
+ # Batch inference
143
+ inputs = [{
144
+ "prompt": prompt,
145
+ "multi_modal_data": {
146
+ "image": image
147
+ },
148
+ } for _ in range(args.num_prompts)]
149
+
150
+ outputs = llm.generate(inputs, sampling_params=sampling_params)
151
+
152
+ for o in outputs:
153
+ generated_text = o.outputs[0].text
154
+ print(generated_text)
155
+
156
+
157
+if __name__ == "__main__":
158
+ parser = FlexibleArgumentParser(
159
+ description='Demo on using vLLM for offline inference with '
160
+ 'vision language models')
161
+ args = parser.parse_args()
162
+ parser.add_argument('--model-type',
163
+ '-m',
164
+ type=str,
165
+ default="llava",
166
+ choices=model_example_map.keys(),
167
+ help='Huggingface "model_type".')
168
+ parser.add_argument('--num-prompts',
169
+ type=int,
170
+ default=1,
171
+ help='Number of prompts to run.')
172
+
173
+ args = parser.parse_args()
174
+ main(args)
examples/paligemma_example.py
deleted
-25
@@ -1,25 +0,0 @@
1
-from vllm import LLM
2
-from vllm.assets.image import ImageAsset
3
-
4
-
5
-def run_paligemma():
6
- llm = LLM(model="google/paligemma-3b-mix-224")
7
-
8
- prompt = "caption es"
9
-
10
- image = ImageAsset("stop_sign").pil_image
11
-
12
- outputs = llm.generate({
13
- "prompt": prompt,
14
- "multi_modal_data": {
15
- "image": image
16
- },
17
- })
18
-
19
- for o in outputs:
20
- generated_text = o.outputs[0].text
21
- print(generated_text)
22
-
23
-
24
-if __name__ == "__main__":
25
- run_paligemma()
examples/phi3v_example.py
deleted
-40
@@ -1,40 +0,0 @@
1
-from vllm import LLM, SamplingParams
2
-from vllm.assets.image import ImageAsset
3
-
4
-
5
-def run_phi3v():
6
- model_path = "microsoft/Phi-3-vision-128k-instruct"
7
-
8
- # Note: The default setting of max_num_seqs (256) and
9
- # max_model_len (128k) for this model may cause OOM.
10
- # You may lower either to run this example on lower-end GPUs.
11
-
12
- # In this example, we override max_num_seqs to 5 while
13
- # keeping the original context length of 128k.
14
- llm = LLM(
15
- model=model_path,
16
- trust_remote_code=True,
17
- max_num_seqs=5,
18
- )
19
-
20
- image = ImageAsset("cherry_blossom").pil_image
21
-
22
- # single-image prompt
23
- prompt = "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n" # noqa: E501
24
- sampling_params = SamplingParams(temperature=0, max_tokens=64)
25
-
26
- outputs = llm.generate(
27
- {
28
- "prompt": prompt,
29
- "multi_modal_data": {
30
- "image": image
31
- },
32
- },
33
- sampling_params=sampling_params)
34
- for o in outputs:
35
- generated_text = o.outputs[0].text
36
- print(generated_text)
37
-
38
-
39
-if __name__ == "__main__":
40
- run_phi3v()