@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6ae996a873
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 14:45 UTC

[Misc] refactor argument parsing in examples (#16635)

Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>

Reid committed Apr 15, 2025 at 16:05 UTC 6ae996a8733269a10cbbc25b8b45b921d81eb362
25 files changed +606 -422
examples/offline_inference/audio_language.py
+28 -24
@@ -187,6 +187,33 @@ model_example_map = {
187 }
188
189
190 +def parse_args():
191 + parser = FlexibleArgumentParser(
192 + description='Demo on using vLLM for offline inference with '
193 + 'audio language models')
194 + parser.add_argument('--model-type',
195 + '-m',
196 + type=str,
197 + default="ultravox",
198 + choices=model_example_map.keys(),
199 + help='Huggingface "model_type".')
200 + parser.add_argument('--num-prompts',
201 + type=int,
202 + default=1,
203 + help='Number of prompts to run.')
204 + parser.add_argument("--num-audios",
205 + type=int,
206 + default=1,
207 + choices=[0, 1, 2],
208 + help="Number of audio items per prompt.")
209 + parser.add_argument("--seed",
210 + type=int,
211 + default=None,
212 + help="Set the seed when initializing `vllm.LLM`.")
213 +
214 + return parser.parse_args()
215 +
216 +
217 def main(args):
218 model = args.model_type
219 if model not in model_example_map:
@@ -240,28 +267,5 @@ def main(args):
267
268
269 if __name__ == "__main__":
243 - parser = FlexibleArgumentParser(
244 - description='Demo on using vLLM for offline inference with '
245 - 'audio language models')
246 - parser.add_argument('--model-type',
247 - '-m',
248 - type=str,
249 - default="ultravox",
250 - choices=model_example_map.keys(),
251 - help='Huggingface "model_type".')
252 - parser.add_argument('--num-prompts',
253 - type=int,
254 - default=1,
255 - help='Number of prompts to run.')
256 - parser.add_argument("--num-audios",
257 - type=int,
258 - default=1,
259 - choices=[0, 1, 2],
260 - help="Number of audio items per prompt.")
261 - parser.add_argument("--seed",
262 - type=int,
263 - default=None,
264 - help="Set the seed when initializing `vllm.LLM`.")
265 -
266 - args = parser.parse_args()
270 + args = parse_args()
271 main(args)
examples/offline_inference/basic/basic.py
+20 -13
@@ -12,16 +12,23 @@ prompts = [
12 # Create a sampling params object.
13 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
14
15 -# Create an LLM.
16 -llm = LLM(model="facebook/opt-125m")
17 -# Generate texts from the prompts. The output is a list of RequestOutput objects
18 -# that contain the prompt, generated text, and other information.
19 -outputs = llm.generate(prompts, sampling_params)
20 -# Print the outputs.
21 -print("\nGenerated Outputs:\n" + "-" * 60)
22 -for output in outputs:
23 - prompt = output.prompt
24 - generated_text = output.outputs[0].text
25 - print(f"Prompt: {prompt!r}")
26 - print(f"Output: {generated_text!r}")
27 - print("-" * 60)
\ No newline at end of file
15 +
16 +def main():
17 + # Create an LLM.
18 + llm = LLM(model="facebook/opt-125m")
19 + # Generate texts from the prompts.
20 + # The output is a list of RequestOutput objects
21 + # that contain the prompt, generated text, and other information.
22 + outputs = llm.generate(prompts, sampling_params)
23 + # Print the outputs.
24 + print("\nGenerated Outputs:\n" + "-" * 60)
25 + for output in outputs:
26 + prompt = output.prompt
27 + generated_text = output.outputs[0].text
28 + print(f"Prompt: {prompt!r}")
29 + print(f"Output: {generated_text!r}")
30 + print("-" * 60)
31 +
32 +
33 +if __name__ == "__main__":
34 + main()
examples/offline_inference/basic/chat.py
+19 -13
@@ -4,6 +4,24 @@ from vllm import LLM, EngineArgs
4 from vllm.utils import FlexibleArgumentParser
5
6
7 +def create_parser():
8 + parser = FlexibleArgumentParser()
9 + # Add engine args
10 + engine_group = parser.add_argument_group("Engine arguments")
11 + EngineArgs.add_cli_args(engine_group)
12 + engine_group.set_defaults(model="meta-llama/Llama-3.2-1B-Instruct")
13 + # Add sampling params
14 + sampling_group = parser.add_argument_group("Sampling parameters")
15 + sampling_group.add_argument("--max-tokens", type=int)
16 + sampling_group.add_argument("--temperature", type=float)
17 + sampling_group.add_argument("--top-p", type=float)
18 + sampling_group.add_argument("--top-k", type=int)
19 + # Add example params
20 + parser.add_argument("--chat-template-path", type=str)
21 +
22 + return parser
23 +
24 +
25 def main(args: dict):
26 # Pop arguments not used by LLM
27 max_tokens = args.pop("max_tokens")
@@ -82,18 +100,6 @@ def main(args: dict):
100
101
102 if __name__ == "__main__":
85 - parser = FlexibleArgumentParser()
86 - # Add engine args
87 - engine_group = parser.add_argument_group("Engine arguments")
88 - EngineArgs.add_cli_args(engine_group)
89 - engine_group.set_defaults(model="meta-llama/Llama-3.2-1B-Instruct")
90 - # Add sampling params
91 - sampling_group = parser.add_argument_group("Sampling parameters")
92 - sampling_group.add_argument("--max-tokens", type=int)
93 - sampling_group.add_argument("--temperature", type=float)
94 - sampling_group.add_argument("--top-p", type=float)
95 - sampling_group.add_argument("--top-k", type=int)
96 - # Add example params
97 - parser.add_argument("--chat-template-path", type=str)
103 + parser = create_parser()
104 args: dict = vars(parser.parse_args())
105 main(args)
examples/offline_inference/basic/classify.py
+11 -7
@@ -6,6 +6,16 @@ from vllm import LLM, EngineArgs
6 from vllm.utils import FlexibleArgumentParser
7
8
9 +def parse_args():
10 + parser = FlexibleArgumentParser()
11 + parser = EngineArgs.add_cli_args(parser)
12 + # Set example specific arguments
13 + parser.set_defaults(model="jason9693/Qwen2.5-1.5B-apeach",
14 + task="classify",
15 + enforce_eager=True)
16 + return parser.parse_args()
17 +
18 +
19 def main(args: Namespace):
20 # Sample prompts.
21 prompts = [
@@ -34,11 +44,5 @@ def main(args: Namespace):
44
45
46 if __name__ == "__main__":
37 - parser = FlexibleArgumentParser()
38 - parser = EngineArgs.add_cli_args(parser)
39 - # Set example specific arguments
40 - parser.set_defaults(model="jason9693/Qwen2.5-1.5B-apeach",
41 - task="classify",
42 - enforce_eager=True)
43 - args = parser.parse_args()
47 + args = parse_args()
48 main(args)
examples/offline_inference/basic/embed.py
+11 -7
@@ -6,6 +6,16 @@ from vllm import LLM, EngineArgs
6 from vllm.utils import FlexibleArgumentParser
7
8
9 +def parse_args():
10 + parser = FlexibleArgumentParser()
11 + parser = EngineArgs.add_cli_args(parser)
12 + # Set example specific arguments
13 + parser.set_defaults(model="intfloat/e5-mistral-7b-instruct",
14 + task="embed",
15 + enforce_eager=True)
16 + return parser.parse_args()
17 +
18 +
19 def main(args: Namespace):
20 # Sample prompts.
21 prompts = [
@@ -34,11 +44,5 @@ def main(args: Namespace):
44
45
46 if __name__ == "__main__":
37 - parser = FlexibleArgumentParser()
38 - parser = EngineArgs.add_cli_args(parser)
39 - # Set example specific arguments
40 - parser.set_defaults(model="intfloat/e5-mistral-7b-instruct",
41 - task="embed",
42 - enforce_eager=True)
43 - args = parser.parse_args()
47 + args = parse_args()
48 main(args)
examples/offline_inference/basic/generate.py
+20 -12
@@ -4,6 +4,22 @@ from vllm import LLM, EngineArgs
4 from vllm.utils import FlexibleArgumentParser
5
6
7 +def create_parser():
8 + parser = FlexibleArgumentParser()
9 + # Add engine args
10 + engine_group = parser.add_argument_group("Engine arguments")
11 + EngineArgs.add_cli_args(engine_group)
12 + engine_group.set_defaults(model="meta-llama/Llama-3.2-1B-Instruct")
13 + # Add sampling params
14 + sampling_group = parser.add_argument_group("Sampling parameters")
15 + sampling_group.add_argument("--max-tokens", type=int)
16 + sampling_group.add_argument("--temperature", type=float)
17 + sampling_group.add_argument("--top-p", type=float)
18 + sampling_group.add_argument("--top-k", type=int)
19 +
20 + return parser
21 +
22 +
23 def main(args: dict):
24 # Pop arguments not used by LLM
25 max_tokens = args.pop("max_tokens")
@@ -35,23 +51,15 @@ def main(args: dict):
51 ]
52 outputs = llm.generate(prompts, sampling_params)
53 # Print the outputs.
54 + print("-" * 50)
55 for output in outputs:
56 prompt = output.prompt
57 generated_text = output.outputs[0].text
41 - print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
58 + print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
59 + print("-" * 50)
60
61
62 if __name__ == "__main__":
45 - parser = FlexibleArgumentParser()
46 - # Add engine args
47 - engine_group = parser.add_argument_group("Engine arguments")
48 - EngineArgs.add_cli_args(engine_group)
49 - engine_group.set_defaults(model="meta-llama/Llama-3.2-1B-Instruct")
50 - # Add sampling params
51 - sampling_group = parser.add_argument_group("Sampling parameters")
52 - sampling_group.add_argument("--max-tokens", type=int)
53 - sampling_group.add_argument("--temperature", type=float)
54 - sampling_group.add_argument("--top-p", type=float)
55 - sampling_group.add_argument("--top-k", type=int)
63 + parser = create_parser()
64 args: dict = vars(parser.parse_args())
65 main(args)
examples/offline_inference/basic/score.py
+11 -7
@@ -6,6 +6,16 @@ from vllm import LLM, EngineArgs
6 from vllm.utils import FlexibleArgumentParser
7
8
9 +def parse_args():
10 + parser = FlexibleArgumentParser()
11 + parser = EngineArgs.add_cli_args(parser)
12 + # Set example specific arguments
13 + parser.set_defaults(model="BAAI/bge-reranker-v2-m3",
14 + task="score",
15 + enforce_eager=True)
16 + return parser.parse_args()
17 +
18 +
19 def main(args: Namespace):
20 # Sample prompts.
21 text_1 = "What is the capital of France?"
@@ -30,11 +40,5 @@ def main(args: Namespace):
40
41
42 if __name__ == "__main__":
33 - parser = FlexibleArgumentParser()
34 - parser = EngineArgs.add_cli_args(parser)
35 - # Set example specific arguments
36 - parser.set_defaults(model="BAAI/bge-reranker-v2-m3",
37 - task="score",
38 - enforce_eager=True)
39 - args = parser.parse_args()
43 + args = parse_args()
44 main(args)
examples/offline_inference/data_parallel.py
+36 -31
@@ -34,6 +34,40 @@ from vllm import LLM, SamplingParams
34 from vllm.utils import get_open_port
35
36
37 +def parse_args():
38 + import argparse
39 + parser = argparse.ArgumentParser(description="Data Parallel Inference")
40 + parser.add_argument("--model",
41 + type=str,
42 + default="ibm-research/PowerMoE-3b",
43 + help="Model name or path")
44 + parser.add_argument("--dp-size",
45 + type=int,
46 + default=2,
47 + help="Data parallel size")
48 + parser.add_argument("--tp-size",
49 + type=int,
50 + default=2,
51 + help="Tensor parallel size")
52 + parser.add_argument("--node-size",
53 + type=int,
54 + default=1,
55 + help="Total number of nodes")
56 + parser.add_argument("--node-rank",
57 + type=int,
58 + default=0,
59 + help="Rank of the current node")
60 + parser.add_argument("--master-addr",
61 + type=str,
62 + default="",
63 + help="Master node IP address")
64 + parser.add_argument("--master-port",
65 + type=int,
66 + default=0,
67 + help="Master node port")
68 + return parser.parse_args()
69 +
70 +
71 def main(model, dp_size, local_dp_rank, global_dp_rank, dp_master_ip,
72 dp_master_port, GPUs_per_dp_rank):
73 os.environ["VLLM_DP_RANK"] = str(global_dp_rank)
@@ -95,37 +129,8 @@ def main(model, dp_size, local_dp_rank, global_dp_rank, dp_master_ip,
129
130
131 if __name__ == "__main__":
98 - import argparse
99 - parser = argparse.ArgumentParser(description="Data Parallel Inference")
100 - parser.add_argument("--model",
101 - type=str,
102 - default="ibm-research/PowerMoE-3b",
103 - help="Model name or path")
104 - parser.add_argument("--dp-size",
105 - type=int,
106 - default=2,
107 - help="Data parallel size")
108 - parser.add_argument("--tp-size",
109 - type=int,
110 - default=2,
111 - help="Tensor parallel size")
112 - parser.add_argument("--node-size",
113 - type=int,
114 - default=1,
115 - help="Total number of nodes")
116 - parser.add_argument("--node-rank",
117 - type=int,
118 - default=0,
119 - help="Rank of the current node")
120 - parser.add_argument("--master-addr",
121 - type=str,
122 - default="",
123 - help="Master node IP address")
124 - parser.add_argument("--master-port",
125 - type=int,
126 - default=0,
127 - help="Master node port")
128 - args = parser.parse_args()
132 +
133 + args = parse_args()
134
135 dp_size = args.dp_size
136 tp_size = args.tp_size
examples/offline_inference/eagle.py
+7 -2
@@ -27,7 +27,7 @@ def load_prompts(dataset_path, num_prompts):
27 return prompts[:num_prompts]
28
29
30 -def main():
30 +def parse_args():
31 parser = argparse.ArgumentParser()
32 parser.add_argument(
33 "--dataset",
@@ -45,7 +45,12 @@ def main():
45 parser.add_argument("--enable_chunked_prefill", action='store_true')
46 parser.add_argument("--max_num_batched_tokens", type=int, default=2048)
47 parser.add_argument("--temp", type=float, default=0)
48 - args = parser.parse_args()
48 + return parser.parse_args()
49 +
50 +
51 +def main():
52 +
53 + args = parse_args()
54
55 model_dir = "meta-llama/Meta-Llama-3-8B-Instruct"
56 eagle_dir = "abhigoyal/EAGLE-LLaMA3-Instruct-8B-vllm"
examples/offline_inference/embed_jina_embeddings_v3.py
+11 -7
@@ -6,6 +6,16 @@ from vllm import LLM, EngineArgs
6 from vllm.utils import FlexibleArgumentParser
7
8
9 +def parse_args():
10 + parser = FlexibleArgumentParser()
11 + parser = EngineArgs.add_cli_args(parser)
12 + # Set example specific arguments
13 + parser.set_defaults(model="jinaai/jina-embeddings-v3",
14 + task="embed",
15 + trust_remote_code=True)
16 + return parser.parse_args()
17 +
18 +
19 def main(args: Namespace):
20 # Sample prompts.
21 prompts = [
@@ -40,11 +50,5 @@ def main(args: Namespace):
50
51
52 if __name__ == "__main__":
43 - parser = FlexibleArgumentParser()
44 - parser = EngineArgs.add_cli_args(parser)
45 - # Set example specific arguments
46 - parser.set_defaults(model="jinaai/jina-embeddings-v3",
47 - task="embed",
48 - trust_remote_code=True)
49 - args = parser.parse_args()
53 + args = parse_args()
54 main(args)
examples/offline_inference/embed_matryoshka_fy.py
+11 -7
@@ -6,6 +6,16 @@ from vllm import LLM, EngineArgs, PoolingParams
6 from vllm.utils import FlexibleArgumentParser
7
8
9 +def parse_args():
10 + parser = FlexibleArgumentParser()
11 + parser = EngineArgs.add_cli_args(parser)
12 + # Set example specific arguments
13 + parser.set_defaults(model="jinaai/jina-embeddings-v3",
14 + task="embed",
15 + trust_remote_code=True)
16 + return parser.parse_args()
17 +
18 +
19 def main(args: Namespace):
20 # Sample prompts.
21 prompts = [
@@ -38,11 +48,5 @@ def main(args: Namespace):
48
49
50 if __name__ == "__main__":
41 - parser = FlexibleArgumentParser()
42 - parser = EngineArgs.add_cli_args(parser)
43 - # Set example specific arguments
44 - parser.set_defaults(model="jinaai/jina-embeddings-v3",
45 - task="embed",
46 - trust_remote_code=True)
47 - args = parser.parse_args()
51 + args = parse_args()
52 main(args)
examples/offline_inference/encoder_decoder.py
+104 -86
@@ -8,94 +8,112 @@ from vllm import LLM, SamplingParams
8 from vllm.inputs import (ExplicitEncoderDecoderPrompt, TextPrompt,
9 TokensPrompt, zip_enc_dec_prompts)
10
11 -dtype = "float"
12 -
13 -# Create a BART encoder/decoder model instance
14 -llm = LLM(
15 - model="facebook/bart-large-cnn",
16 - dtype=dtype,
17 -)
18 -
19 -# Get BART tokenizer
20 -tokenizer = llm.llm_engine.get_tokenizer_group()
21 -
22 -# Test prompts
23 -#
24 -# This section shows all of the valid ways to prompt an
25 -# encoder/decoder model.
26 -#
27 -# - Helpers for building prompts
28 -text_prompt_raw = "Hello, my name is"
29 -text_prompt = TextPrompt(prompt="The president of the United States is")
30 -tokens_prompt = TokensPrompt(prompt_token_ids=tokenizer.encode(
31 - prompt="The capital of France is"))
32 -# - Pass a single prompt to encoder/decoder model
33 -# (implicitly encoder input prompt);
34 -# decoder input prompt is assumed to be None
35 -
36 -single_text_prompt_raw = text_prompt_raw # Pass a string directly
37 -single_text_prompt = text_prompt # Pass a TextPrompt
38 -single_tokens_prompt = tokens_prompt # Pass a TokensPrompt
39 -
40 -# - Pass explicit encoder and decoder input prompts within one data structure.
41 -# Encoder and decoder prompts can both independently be text or tokens, with
42 -# no requirement that they be the same prompt type. Some example prompt-type
43 -# combinations are shown below, note that these are not exhaustive.
44 -
45 -enc_dec_prompt1 = ExplicitEncoderDecoderPrompt(
46 - # Pass encoder prompt string directly, &
47 - # pass decoder prompt tokens
48 - encoder_prompt=single_text_prompt_raw,
49 - decoder_prompt=single_tokens_prompt,
50 -)
51 -enc_dec_prompt2 = ExplicitEncoderDecoderPrompt(
52 - # Pass TextPrompt to encoder, and
53 - # pass decoder prompt string directly
54 - encoder_prompt=single_text_prompt,
55 - decoder_prompt=single_text_prompt_raw,
56 -)
57 -enc_dec_prompt3 = ExplicitEncoderDecoderPrompt(
58 - # Pass encoder prompt tokens directly, and
59 - # pass TextPrompt to decoder
60 - encoder_prompt=single_tokens_prompt,
61 - decoder_prompt=single_text_prompt,
62 -)
63 -
64 -# - Finally, here's a useful helper function for zipping encoder and
65 -# decoder prompts together into a list of ExplicitEncoderDecoderPrompt
66 -# instances
67 -zipped_prompt_list = zip_enc_dec_prompts(
68 - ['An encoder prompt', 'Another encoder prompt'],
69 - ['A decoder prompt', 'Another decoder prompt'])
70 -
71 -# - Let's put all of the above example prompts together into one list
72 -# which we will pass to the encoder/decoder LLM.
73 -prompts = [
74 - single_text_prompt_raw, single_text_prompt, single_tokens_prompt,
75 - enc_dec_prompt1, enc_dec_prompt2, enc_dec_prompt3
76 -] + zipped_prompt_list
11 +
12 +def create_prompts(tokenizer):
13 + # Test prompts
14 + #
15 + # This section shows all of the valid ways to prompt an
16 + # encoder/decoder model.
17 + #
18 + # - Helpers for building prompts
19 + text_prompt_raw = "Hello, my name is"
20 + text_prompt = TextPrompt(prompt="The president of the United States is")
21 + tokens_prompt = TokensPrompt(prompt_token_ids=tokenizer.encode(
22 + prompt="The capital of France is"))
23 + # - Pass a single prompt to encoder/decoder model
24 + # (implicitly encoder input prompt);
25 + # decoder input prompt is assumed to be None
26 +
27 + single_text_prompt_raw = text_prompt_raw # Pass a string directly
28 + single_text_prompt = text_prompt # Pass a TextPrompt
29 + single_tokens_prompt = tokens_prompt # Pass a TokensPrompt
30 +
31 + # ruff: noqa: E501
32 + # - Pass explicit encoder and decoder input prompts within one data structure.
33 + # Encoder and decoder prompts can both independently be text or tokens, with
34 + # no requirement that they be the same prompt type. Some example prompt-type
35 + # combinations are shown below, note that these are not exhaustive.
36 +
37 + enc_dec_prompt1 = ExplicitEncoderDecoderPrompt(
38 + # Pass encoder prompt string directly, &
39 + # pass decoder prompt tokens
40 + encoder_prompt=single_text_prompt_raw,
41 + decoder_prompt=single_tokens_prompt,
42 + )
43 + enc_dec_prompt2 = ExplicitEncoderDecoderPrompt(
44 + # Pass TextPrompt to encoder, and
45 + # pass decoder prompt string directly
46 + encoder_prompt=single_text_prompt,
47 + decoder_prompt=single_text_prompt_raw,
48 + )
49 + enc_dec_prompt3 = ExplicitEncoderDecoderPrompt(
50 + # Pass encoder prompt tokens directly, and
51 + # pass TextPrompt to decoder
52 + encoder_prompt=single_tokens_prompt,
53 + decoder_prompt=single_text_prompt,
54 + )
55 +
56 + # - Finally, here's a useful helper function for zipping encoder and
57 + # decoder prompts together into a list of ExplicitEncoderDecoderPrompt
58 + # instances
59 + zipped_prompt_list = zip_enc_dec_prompts(
60 + ['An encoder prompt', 'Another encoder prompt'],
61 + ['A decoder prompt', 'Another decoder prompt'])
62 +
63 + # - Let's put all of the above example prompts together into one list
64 + # which we will pass to the encoder/decoder LLM.
65 + return [
66 + single_text_prompt_raw, single_text_prompt, single_tokens_prompt,
67 + enc_dec_prompt1, enc_dec_prompt2, enc_dec_prompt3
68 + ] + zipped_prompt_list
69 +
70
71 # Create a sampling params object.
79 -sampling_params = SamplingParams(
80 - temperature=0,
81 - top_p=1.0,
82 - min_tokens=0,
83 - max_tokens=20,
84 -)
85 -
86 -# Generate output tokens from the prompts. The output is a list of
87 -# RequestOutput objects that contain the prompt, generated
88 -# text, and other information.
89 -outputs = llm.generate(prompts, sampling_params)
72 +def create_sampling_params():
73 + return SamplingParams(
74 + temperature=0,
75 + top_p=1.0,
76 + min_tokens=0,
77 + max_tokens=20,
78 + )
79 +
80
81 # Print the outputs.
92 -print("-" * 50)
93 -for i, output in enumerate(outputs):
94 - prompt = output.prompt
95 - encoder_prompt = output.encoder_prompt
96 - generated_text = output.outputs[0].text
97 - print(f"Output {i+1}:")
98 - print(f"Encoder prompt: {encoder_prompt!r}\n"
99 - f"Decoder prompt: {prompt!r}\n"
100 - f"Generated text: {generated_text!r}")
82 +def print_outputs(outputs):
83 print("-" * 50)
84 + for i, output in enumerate(outputs):
85 + prompt = output.prompt
86 + encoder_prompt = output.encoder_prompt
87 + generated_text = output.outputs[0].text
88 + print(f"Output {i+1}:")
89 + print(f"Encoder prompt: {encoder_prompt!r}\n"
90 + f"Decoder prompt: {prompt!r}\n"
91 + f"Generated text: {generated_text!r}")
92 + print("-" * 50)
93 +
94 +
95 +def main():
96 + dtype = "float"
97 +
98 + # Create a BART encoder/decoder model instance
99 + llm = LLM(
100 + model="facebook/bart-large-cnn",
101 + dtype=dtype,
102 + )
103 +
104 + # Get BART tokenizer
105 + tokenizer = llm.llm_engine.get_tokenizer_group()
106 +
107 + prompts = create_prompts(tokenizer)
108 + sampling_params = create_sampling_params()
109 +
110 + # Generate output tokens from the prompts. The output is a list of
111 + # RequestOutput objects that contain the prompt, generated
112 + # text, and other information.
113 + outputs = llm.generate(prompts, sampling_params)
114 +
115 + print_outputs(outputs)
116 +
117 +
118 +if __name__ == "__main__":
119 + main()
examples/offline_inference/encoder_decoder_multimodal.py
+18 -15
@@ -126,6 +126,23 @@ model_example_map = {
126 }
127
128
129 +def parse_args():
130 + parser = FlexibleArgumentParser(
131 + description='Demo on using vLLM for offline inference with '
132 + 'vision language models for text generation')
133 + parser.add_argument('--model-type',
134 + '-m',
135 + type=str,
136 + default="mllama",
137 + choices=model_example_map.keys(),
138 + help='Huggingface "model_type".')
139 + parser.add_argument("--seed",
140 + type=int,
141 + default=None,
142 + help="Set the seed when initializing `vllm.LLM`.")
143 + return parser.parse_args()
144 +
145 +
146 def main(args):
147 model = args.model_type
148 if model not in model_example_map:
@@ -171,19 +188,5 @@ def main(args):
188
189
190 if __name__ == "__main__":
174 - parser = FlexibleArgumentParser(
175 - description='Demo on using vLLM for offline inference with '
176 - 'vision language models for text generation')
177 - parser.add_argument('--model-type',
178 - '-m',
179 - type=str,
180 - default="mllama",
181 - choices=model_example_map.keys(),
182 - help='Huggingface "model_type".')
183 - parser.add_argument("--seed",
184 - type=int,
185 - default=None,
186 - help="Set the seed when initializing `vllm.LLM`.")
187 -
188 - args = parser.parse_args()
191 + args = parse_args()
192 main(args)
examples/offline_inference/mistral-small.py
+5 -2
@@ -168,7 +168,7 @@ def run_advanced_demo(args: argparse.Namespace):
168 print("-" * 50)
169
170
171 -def main():
171 +def parse_args():
172 parser = argparse.ArgumentParser(
173 description="Run a demo in simple or advanced mode.")
174
@@ -187,8 +187,11 @@ def main():
187 '--disable-mm-preprocessor-cache',
188 action='store_true',
189 help='If True, disables caching of multi-modal preprocessor/mapper.')
190 + return parser.parse_args()
191 +
192
191 - args = parser.parse_args()
193 +def main():
194 + args = parse_args()
195
196 if args.mode == "simple":
197 print("Running simple demo...")
examples/offline_inference/mlpspeculator.py
+5 -2
@@ -34,8 +34,7 @@ def time_generation(llm: LLM, prompts: list[str],
34 print("-" * 50)
35
36
37 -if __name__ == "__main__":
38 -
37 +def main():
38 template = (
39 "Below is an instruction that describes a task. Write a response "
40 "that appropriately completes the request.\n\n### Instruction:\n{}"
@@ -66,3 +65,7 @@ if __name__ == "__main__":
65 )
66
67 time_generation(llm, prompts, sampling_params, "With speculation")
68 +
69 +
70 +if __name__ == "__main__":
71 + main()
examples/offline_inference/prithvi_geospatial_mae.py
+33 -29
@@ -417,6 +417,38 @@ def run_model(input_data,
417 return pred_imgs
418
419
420 +def parse_args():
421 + parser = argparse.ArgumentParser("MAE run inference", add_help=False)
422 +
423 + parser.add_argument(
424 + "--data_file",
425 + type=str,
426 + default="./India_900498_S2Hand.tif",
427 + help="Path to the file.",
428 + )
429 + parser.add_argument(
430 + "--output_dir",
431 + type=str,
432 + default="output",
433 + help="Path to the directory where to save outputs.",
434 + )
435 + parser.add_argument(
436 + "--input_indices",
437 + default=[1, 2, 3, 8, 11, 12],
438 + type=int,
439 + nargs="+",
440 + help=
441 + "0-based indices of the six Prithvi channels to be selected from the "
442 + "input. By default selects [1,2,3,8,11,12] for S2L1C data.",
443 + )
444 + parser.add_argument(
445 + "--rgb_outputs",
446 + action="store_true",
447 + help="If present, output files will only contain RGB channels. "
448 + "Otherwise, all bands will be saved.",
449 + )
450 +
451 +
452 def main(
453 data_file: str,
454 output_dir: str,
@@ -496,35 +528,7 @@ def main(
528
529
530 if __name__ == "__main__":
499 - parser = argparse.ArgumentParser("MAE run inference", add_help=False)
531
501 - parser.add_argument(
502 - "--data_file",
503 - type=str,
504 - default="./India_900498_S2Hand.tif",
505 - help="Path to the file.",
506 - )
507 - parser.add_argument(
508 - "--output_dir",
509 - type=str,
510 - default="output",
511 - help="Path to the directory where to save outputs.",
512 - )
513 - parser.add_argument(
514 - "--input_indices",
515 - default=[1, 2, 3, 8, 11, 12],
516 - type=int,
517 - nargs="+",
518 - help=
519 - "0-based indices of the six Prithvi channels to be selected from the "
520 - "input. By default selects [1,2,3,8,11,12] for S2L1C data.",
521 - )
522 - parser.add_argument(
523 - "--rgb_outputs",
524 - action="store_true",
525 - help="If present, output files will only contain RGB channels. "
526 - "Otherwise, all bands will be saved.",
527 - )
528 - args = parser.parse_args()
532 + args = parse_args()
533
534 main(**vars(args))
examples/offline_inference/profiling.py
+10 -2
@@ -359,7 +359,7 @@ def run_profile(context: ProfileContext, csv_output: Optional[str],
359 f" in folder {context.save_chrome_traces_folder}")
360
361
362 -if __name__ == "__main__":
362 +def parse_args():
363 parser = FlexibleArgumentParser(description="""
364 Profile a model
365
@@ -449,7 +449,10 @@ Profile a model
449
450 EngineArgs.add_cli_args(parser)
451
452 - args = parser.parse_args()
452 + return parser.parse_args()
453 +
454 +
455 +def main(args):
456 context = ProfileContext(
457 engine_args=EngineArgs.from_cli_args(args),
458 **{
@@ -458,3 +461,8 @@ Profile a model
461 if k in inspect.signature(ProfileContext).parameters
462 })
463 run_profile(context, csv_output=args.csv, json_output=args.json)
464 +
465 +
466 +if __name__ == "__main__":
467 + args = parse_args()
468 + main(args)
examples/offline_inference/save_sharded_state.py
+18 -15
@@ -29,20 +29,23 @@ from pathlib import Path
29 from vllm import LLM, EngineArgs
30 from vllm.utils import FlexibleArgumentParser
31
32 -parser = FlexibleArgumentParser()
33 -EngineArgs.add_cli_args(parser)
34 -parser.add_argument("--output",
35 - "-o",
36 - required=True,
37 - type=str,
38 - help="path to output checkpoint")
39 -parser.add_argument("--file-pattern",
40 - type=str,
41 - help="string pattern of saved filenames")
42 -parser.add_argument("--max-file-size",
43 - type=str,
44 - default=5 * 1024**3,
45 - help="max size (in bytes) of each safetensors file")
32 +
33 +def parse_args():
34 + parser = FlexibleArgumentParser()
35 + EngineArgs.add_cli_args(parser)
36 + parser.add_argument("--output",
37 + "-o",
38 + required=True,
39 + type=str,
40 + help="path to output checkpoint")
41 + parser.add_argument("--file-pattern",
42 + type=str,
43 + help="string pattern of saved filenames")
44 + parser.add_argument("--max-file-size",
45 + type=str,
46 + default=5 * 1024**3,
47 + help="max size (in bytes) of each safetensors file")
48 + return parser.parse_args()
49
50
51 def main(args):
@@ -87,5 +90,5 @@ def main(args):
90
91
92 if __name__ == "__main__":
90 - args = parser.parse_args()
93 + args = parse_args()
94 main(args)
examples/offline_inference/simple_profiling.py
+5 -1
@@ -18,8 +18,8 @@ prompts = [
18 # Create a sampling params object.
19 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
20
21 -if __name__ == "__main__":
21
22 +def main():
23 # Create an LLM.
24 llm = LLM(model="facebook/opt-125m", tensor_parallel_size=1)
25
@@ -42,3 +42,7 @@ if __name__ == "__main__":
42 # Add a buffer to wait for profiler in the background process
43 # (in case MP is on) to finish writing profiling output.
44 time.sleep(10)
45 +
46 +
47 +if __name__ == "__main__":
48 + main()
examples/offline_inference/vision_language.py
+54 -51
@@ -1097,6 +1097,59 @@ def time_counter(enable: bool):
1097 yield
1098
1099
1100 +def parse_args():
1101 + parser = FlexibleArgumentParser(
1102 + description='Demo on using vLLM for offline inference with '
1103 + 'vision language models for text generation')
1104 + parser.add_argument('--model-type',
1105 + '-m',
1106 + type=str,
1107 + default="llava",
1108 + choices=model_example_map.keys(),
1109 + help='Huggingface "model_type".')
1110 + parser.add_argument('--num-prompts',
1111 + type=int,
1112 + default=4,
1113 + help='Number of prompts to run.')
1114 + parser.add_argument('--modality',
1115 + type=str,
1116 + default="image",
1117 + choices=['image', 'video'],
1118 + help='Modality of the input.')
1119 + parser.add_argument('--num-frames',
1120 + type=int,
1121 + default=16,
1122 + help='Number of frames to extract from the video.')
1123 + parser.add_argument("--seed",
1124 + type=int,
1125 + default=None,
1126 + help="Set the seed when initializing `vllm.LLM`.")
1127 +
1128 + parser.add_argument(
1129 + '--image-repeat-prob',
1130 + type=float,
1131 + default=None,
1132 + help='Simulates the hit-ratio for multi-modal preprocessor cache'
1133 + ' (if enabled)')
1134 +
1135 + parser.add_argument(
1136 + '--disable-mm-preprocessor-cache',
1137 + action='store_true',
1138 + help='If True, disables caching of multi-modal preprocessor/mapper.')
1139 +
1140 + parser.add_argument(
1141 + '--time-generate',
1142 + action='store_true',
1143 + help='If True, then print the total generate() call time')
1144 +
1145 + parser.add_argument(
1146 + '--use-different-prompt-per-request',
1147 + action='store_true',
1148 + help='If True, then use different prompt (with the same multi-modal '
1149 + 'data) for each request.')
1150 + return parser.parse_args()
1151 +
1152 +
1153 def main(args):
1154 model = args.model_type
1155 if model not in model_example_map:
@@ -1175,55 +1228,5 @@ def main(args):
1228
1229
1230 if __name__ == "__main__":
1178 - parser = FlexibleArgumentParser(
1179 - description='Demo on using vLLM for offline inference with '
1180 - 'vision language models for text generation')
1181 - parser.add_argument('--model-type',
1182 - '-m',
1183 - type=str,
1184 - default="llava",
1185 - choices=model_example_map.keys(),
1186 - help='Huggingface "model_type".')
1187 - parser.add_argument('--num-prompts',
1188 - type=int,
1189 - default=4,
1190 - help='Number of prompts to run.')
1191 - parser.add_argument('--modality',
1192 - type=str,
1193 - default="image",
1194 - choices=['image', 'video'],
1195 - help='Modality of the input.')
1196 - parser.add_argument('--num-frames',
1197 - type=int,
1198 - default=16,
1199 - help='Number of frames to extract from the video.')
1200 - parser.add_argument("--seed",
1201 - type=int,
1202 - default=None,
1203 - help="Set the seed when initializing `vllm.LLM`.")
1204 -
1205 - parser.add_argument(
1206 - '--image-repeat-prob',
1207 - type=float,
1208 - default=None,
1209 - help='Simulates the hit-ratio for multi-modal preprocessor cache'
1210 - ' (if enabled)')
1211 -
1212 - parser.add_argument(
1213 - '--disable-mm-preprocessor-cache',
1214 - action='store_true',
1215 - help='If True, disables caching of multi-modal preprocessor/mapper.')
1216 -
1217 - parser.add_argument(
1218 - '--time-generate',
1219 - action='store_true',
1220 - help='If True, then print the total generate() call time')
1221 -
1222 - parser.add_argument(
1223 - '--use-different-prompt-per-request',
1224 - action='store_true',
1225 - help='If True, then use different prompt (with the same multi-modal '
1226 - 'data) for each request.')
1227 -
1228 - args = parser.parse_args()
1231 + args = parse_args()
1232 main(args)
examples/offline_inference/vision_language_embedding.py
+10 -6
@@ -156,16 +156,13 @@ def run_encode(model: str, modality: QueryModality, seed: Optional[int]):
156 print("-" * 50)
157
158
159 -def main(args: Namespace):
160 - run_encode(args.model_name, args.modality, args.seed)
161 -
162 -
159 model_example_map = {
160 "e5_v": run_e5_v,
161 "vlm2vec": run_vlm2vec,
162 }
163
168 -if __name__ == "__main__":
164 +
165 +def parse_args():
166 parser = FlexibleArgumentParser(
167 description='Demo on using vLLM for offline inference with '
168 'vision language models for multimodal embedding')
@@ -184,6 +181,13 @@ if __name__ == "__main__":
181 type=int,
182 default=None,
183 help="Set the seed when initializing `vllm.LLM`.")
184 + return parser.parse_args()
185
188 - args = parser.parse_args()
186 +
187 +def main(args: Namespace):
188 + run_encode(args.model_name, args.modality, args.seed)
189 +
190 +
191 +if __name__ == "__main__":
192 + args = parse_args()
193 main(args)
examples/offline_inference/vision_language_multi_image.py
+20 -17
@@ -767,22 +767,7 @@ def run_chat(model: str, question: str, image_urls: list[str],
767 print("-" * 50)
768
769
770 -def main(args: Namespace):
771 - model = args.model_type
772 - method = args.method
773 - seed = args.seed
774 -
775 - image_urls = IMAGE_URLS[:args.num_images]
776 -
777 - if method == "generate":
778 - run_generate(model, QUESTION, image_urls, seed)
779 - elif method == "chat":
780 - run_chat(model, QUESTION, image_urls, seed)
781 - else:
782 - raise ValueError(f"Invalid method: {method}")
783 -
784 -
785 -if __name__ == "__main__":
770 +def parse_args():
771 parser = FlexibleArgumentParser(
772 description='Demo on using vLLM for offline inference with '
773 'vision language models that support multi-image input for text '
@@ -808,6 +793,24 @@ if __name__ == "__main__":
793 choices=list(range(1, 13)), # 12 is the max number of images
794 default=2,
795 help="Number of images to use for the demo.")
796 + return parser.parse_args()
797 +
798
812 - args = parser.parse_args()
799 +def main(args: Namespace):
800 + model = args.model_type
801 + method = args.method
802 + seed = args.seed
803 +
804 + image_urls = IMAGE_URLS[:args.num_images]
805 +
806 + if method == "generate":
807 + run_generate(model, QUESTION, image_urls, seed)
808 + elif method == "chat":
809 + run_chat(model, QUESTION, image_urls, seed)
810 + else:
811 + raise ValueError(f"Invalid method: {method}")
812 +
813 +
814 +if __name__ == "__main__":
815 + args = parse_args()
816 main(args)
examples/online_serving/api_client.py
+11 -7
@@ -58,6 +58,16 @@ def get_response(response: requests.Response) -> list[str]:
58 return output
59
60
61 +def parse_args():
62 + parser = argparse.ArgumentParser()
63 + parser.add_argument("--host", type=str, default="localhost")
64 + parser.add_argument("--port", type=int, default=8000)
65 + parser.add_argument("--n", type=int, default=1)
66 + parser.add_argument("--prompt", type=str, default="San Francisco is a")
67 + parser.add_argument("--stream", action="store_true")
68 + return parser.parse_args()
69 +
70 +
71 def main(args: Namespace):
72 prompt = args.prompt
73 api_url = f"http://{args.host}:{args.port}/generate"
@@ -82,11 +92,5 @@ def main(args: Namespace):
92
93
94 if __name__ == "__main__":
85 - parser = argparse.ArgumentParser()
86 - parser.add_argument("--host", type=str, default="localhost")
87 - parser.add_argument("--port", type=int, default=8000)
88 - parser.add_argument("--n", type=int, default=1)
89 - parser.add_argument("--prompt", type=str, default="San Francisco is a")
90 - parser.add_argument("--stream", action="store_true")
91 - args = parser.parse_args()
95 + args = parse_args()
96 main(args)
examples/online_serving/gradio_openai_chatbot_webserver.py
+102 -57
@@ -1,52 +1,36 @@
1 # SPDX-License-Identifier: Apache-2.0
2 +"""Example for starting a Gradio OpenAI Chatbot Webserver
3 +Start vLLM API server:
4 + vllm serve meta-llama/Llama-2-7b-chat-hf
5
6 +Start Gradio OpenAI Chatbot Webserver:
7 + python examples/online_serving/gradio_openai_chatbot_webserver.py \
8 + -m meta-llama/Llama-2-7b-chat-hf
9 +
10 +Note that `pip install --upgrade gradio` is needed to run this example.
11 +More details: https://github.com/gradio-app/gradio
12 +
13 +If your antivirus software blocks the download of frpc for gradio,
14 +you can install it manually by following these steps:
15 +
16 +1. Download this file: https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_amd64
17 +2. Rename the downloaded file to: frpc_linux_amd64_v0.3
18 +3. Move the file to this location: /home/user/.cache/huggingface/gradio/frpc
19 +"""
20 import argparse
21
22 import gradio as gr
23 from openai import OpenAI
24
8 -# Argument parser setup
9 -parser = argparse.ArgumentParser(
10 - description='Chatbot Interface with Customizable Parameters')
11 -parser.add_argument('--model-url',
12 - type=str,
13 - default='http://localhost:8000/v1',
14 - help='Model URL')
15 -parser.add_argument('-m',
16 - '--model',
17 - type=str,
18 - required=True,
19 - help='Model name for the chatbot')
20 -parser.add_argument('--temp',
21 - type=float,
22 - default=0.8,
23 - help='Temperature for text generation')
24 -parser.add_argument('--stop-token-ids',
25 - type=str,
26 - default='',
27 - help='Comma-separated stop token IDs')
28 -parser.add_argument("--host", type=str, default=None)
29 -parser.add_argument("--port", type=int, default=8001)
30 -
31 -# Parse the arguments
32 -args = parser.parse_args()
33 -
34 -# Set OpenAI's API key and API base to use vLLM's API server.
35 -openai_api_key = "EMPTY"
36 -openai_api_base = args.model_url
37 -
38 -# Create an OpenAI client to interact with the API server
39 -client = OpenAI(
40 - api_key=openai_api_key,
41 - base_url=openai_api_base,
42 -)
43 -
44 -
45 -def predict(message, history):
46 - # Convert chat history to OpenAI format
25 +
26 +def create_openai_client(api_key, base_url):
27 + return OpenAI(api_key=api_key, base_url=base_url)
28 +
29 +
30 +def format_history_to_openai(history):
31 history_openai_format = [{
32 "role": "system",
49 - "content": "You are a great ai assistant."
33 + "content": "You are a great AI assistant."
34 }]
35 for human, assistant in history:
36 history_openai_format.append({"role": "user", "content": human})
@@ -54,31 +38,92 @@ def predict(message, history):
38 "role": "assistant",
39 "content": assistant
40 })
41 + return history_openai_format
42 +
43 +
44 +def predict(message, history, client, model_name, temp, stop_token_ids):
45 + # Format history to OpenAI chat format
46 + history_openai_format = format_history_to_openai(history)
47 history_openai_format.append({"role": "user", "content": message})
48
59 - # Create a chat completion request and send it to the API server
49 + # Send request to OpenAI API (vLLM server)
50 stream = client.chat.completions.create(
61 - model=args.model, # Model name to use
62 - messages=history_openai_format, # Chat history
63 - temperature=args.temp, # Temperature for text generation
64 - stream=True, # Stream response
51 + model=model_name,
52 + messages=history_openai_format,
53 + temperature=temp,
54 + stream=True,
55 extra_body={
56 'repetition_penalty':
57 1,
68 - 'stop_token_ids': [
69 - int(id.strip()) for id in args.stop_token_ids.split(',')
70 - if id.strip()
71 - ] if args.stop_token_ids else []
58 + 'stop_token_ids':
59 + [int(id.strip())
60 + for id in stop_token_ids.split(',')] if stop_token_ids else []
61 })
62
74 - # Read and return generated text from response stream
75 - partial_message = ""
63 + # Collect all chunks and concatenate them into a full message
64 + full_message = ""
65 for chunk in stream:
77 - partial_message += (chunk.choices[0].delta.content or "")
78 - yield partial_message
66 + full_message += (chunk.choices[0].delta.content or "")
67 +
68 + # Return the full message as a single response
69 + return full_message
70 +
71 +
72 +def parse_args():
73 + parser = argparse.ArgumentParser(
74 + description='Chatbot Interface with Customizable Parameters')
75 + parser.add_argument('--model-url',
76 + type=str,
77 + default='http://localhost:8000/v1',
78 + help='Model URL')
79 + parser.add_argument('-m',
80 + '--model',
81 + type=str,
82 + required=True,
83 + help='Model name for the chatbot')
84 + parser.add_argument('--temp',
85 + type=float,
86 + default=0.8,
87 + help='Temperature for text generation')
88 + parser.add_argument('--stop-token-ids',
89 + type=str,
90 + default='',
91 + help='Comma-separated stop token IDs')
92 + parser.add_argument("--host", type=str, default=None)
93 + parser.add_argument("--port", type=int, default=8001)
94 + return parser.parse_args()
95 +
96 +
97 +def build_gradio_interface(client, model_name, temp, stop_token_ids):
98 +
99 + def chat_predict(message, history):
100 + return predict(message, history, client, model_name, temp,
101 + stop_token_ids)
102 +
103 + return gr.ChatInterface(fn=chat_predict,
104 + title="Chatbot Interface",
105 + description="A simple chatbot powered by vLLM")
106 +
107 +
108 +def main():
109 + # Parse the arguments
110 + args = parse_args()
111 +
112 + # Set OpenAI's API key and API base to use vLLM's API server
113 + openai_api_key = "EMPTY"
114 + openai_api_base = args.model_url
115 +
116 + # Create an OpenAI client
117 + client = OpenAI(api_key=openai_api_key, base_url=openai_api_base)
118 +
119 + # Define the Gradio chatbot interface using the predict function
120 + gradio_interface = build_gradio_interface(client, args.model, args.temp,
121 + args.stop_token_ids)
122 +
123 + gradio_interface.queue().launch(server_name=args.host,
124 + server_port=args.port,
125 + share=True)
126
127
81 -# Create and launch a chat interface with Gradio
82 -gr.ChatInterface(predict).queue().launch(server_name=args.host,
83 - server_port=args.port,
84 - share=True)
128 +if __name__ == "__main__":
129 + main()
examples/online_serving/gradio_webserver.py
+26 -2
@@ -1,5 +1,22 @@
1 # SPDX-License-Identifier: Apache-2.0
2 +"""Example for starting a Gradio Webserver
3 +Start vLLM API server:
4 + python -m vllm.entrypoints.api_server \
5 + --model meta-llama/Llama-2-7b-chat-hf
6
7 +Start Webserver:
8 + python examples/online_serving/gradio_webserver.py
9 +
10 +Note that `pip install --upgrade gradio` is needed to run this example.
11 +More details: https://github.com/gradio-app/gradio
12 +
13 +If your antivirus software blocks the download of frpc for gradio,
14 +you can install it manually by following these steps:
15 +
16 +1. Download this file: https://cdn-media.huggingface.co/frpc-gradio-0.3/frpc_linux_amd64
17 +2. Rename the downloaded file to: frpc_linux_amd64_v0.3
18 +3. Move the file to this location: /home/user/.cache/huggingface/gradio/frpc
19 +"""
20 import argparse
21 import json
22
@@ -39,16 +56,23 @@ def build_demo():
56 return demo
57
58
42 -if __name__ == "__main__":
59 +def parse_args():
60 parser = argparse.ArgumentParser()
61 parser.add_argument("--host", type=str, default=None)
62 parser.add_argument("--port", type=int, default=8001)
63 parser.add_argument("--model-url",
64 type=str,
65 default="http://localhost:8000/generate")
49 - args = parser.parse_args()
66 + return parser.parse_args()
67
68 +
69 +def main(args):
70 demo = build_demo()
71 demo.queue().launch(server_name=args.host,
72 server_port=args.port,
73 share=True)
74 +
75 +
76 +if __name__ == "__main__":
77 + args = parse_args()
78 + main(args)