@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf21a9bd5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

support trust_remote_code in benchmark (#518)

WRH committed Jul 20, 2023 at 08:02 UTC cf21a9bd5cd29b8b52a8dfceac22798ef648e6bc
1 file changed +9 -3
benchmarks/benchmark_throughput.py
+9 -3
@@ -67,12 +67,14 @@ def run_vllm(
67 seed: int,
68 n: int,
69 use_beam_search: bool,
70 + trust_remote_code: bool,
71 ) -> float:
72 llm = LLM(
73 model=model,
74 tokenizer=tokenizer,
75 tensor_parallel_size=tensor_parallel_size,
76 seed=seed,
77 + trust_remote_code=trust_remote_code
78 )
79
80 # Add the requests to the engine.
@@ -106,9 +108,10 @@ def run_hf(
108 n: int,
109 use_beam_search: bool,
110 max_batch_size: int,
111 + trust_remote_code: bool,
112 ) -> float:
113 assert not use_beam_search
111 - llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
114 + llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
115 if llm.config.model_type == "llama":
116 # To enable padding in the HF backend.
117 tokenizer.pad_token = tokenizer.eos_token
@@ -161,13 +164,13 @@ def main(args: argparse.Namespace):
164 random.seed(args.seed)
165
166 # Sample the requests.
164 - tokenizer = get_tokenizer(args.tokenizer)
167 + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
168 requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
169
170 if args.backend == "vllm":
171 elapsed_time = run_vllm(
172 requests, args.model, args.tokenizer, args.tensor_parallel_size,
170 - args.seed, args.n, args.use_beam_search)
173 + args.seed, args.n, args.use_beam_search, args.trust_remote_code)
174 elif args.backend == "hf":
175 assert args.tensor_parallel_size == 1
176 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -199,6 +202,9 @@ if __name__ == "__main__":
202 parser.add_argument("--seed", type=int, default=0)
203 parser.add_argument("--hf-max-batch-size", type=int, default=None,
204 help="Maximum batch size for HF backend.")
205 + parser.add_argument('--trust-remote-code',
206 + action='store_true',
207 + help='trust remote code from huggingface')
208 args = parser.parse_args()
209
210 if args.backend == "vllm":