A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
support trust_remote_code in benchmark (#518)
WRH committed
Jul 20, 2023 at 08:02 UTC
cf21a9bd5cd29b8b52a8dfceac22798ef648e6bc
1 file changed
+9
-3
benchmarks/benchmark_throughput.py
+9
-3
@@ -67,12 +67,14 @@ def run_vllm(
67
seed: int,
68
n: int,
69
use_beam_search: bool,
70
+ trust_remote_code: bool,
71
) -> float:
72
llm = LLM(
73
model=model,
74
tokenizer=tokenizer,
75
tensor_parallel_size=tensor_parallel_size,
76
seed=seed,
77
+ trust_remote_code=trust_remote_code
78
)
79
80
# Add the requests to the engine.
@@ -106,9 +108,10 @@ def run_hf(
108
n: int,
109
use_beam_search: bool,
110
max_batch_size: int,
111
+ trust_remote_code: bool,
112
) -> float:
113
assert not use_beam_search
111
- llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16)
114
+ llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
115
if llm.config.model_type == "llama":
116
# To enable padding in the HF backend.
117
tokenizer.pad_token = tokenizer.eos_token
@@ -161,13 +164,13 @@ def main(args: argparse.Namespace):
164
random.seed(args.seed)
165
166
# Sample the requests.
164
- tokenizer = get_tokenizer(args.tokenizer)
167
+ tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
168
requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
169
170
if args.backend == "vllm":
171
elapsed_time = run_vllm(
172
requests, args.model, args.tokenizer, args.tensor_parallel_size,
170
- args.seed, args.n, args.use_beam_search)
173
+ args.seed, args.n, args.use_beam_search, args.trust_remote_code)
174
elif args.backend == "hf":
175
assert args.tensor_parallel_size == 1
176
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -199,6 +202,9 @@ if __name__ == "__main__":
202
parser.add_argument("--seed", type=int, default=0)
203
parser.add_argument("--hf-max-batch-size", type=int, default=None,
204
help="Maximum batch size for HF backend.")
205
+ parser.add_argument('--trust-remote-code',
206
+ action='store_true',
207
+ help='trust remote code from huggingface')
208
args = parser.parse_args()
209
210
if args.backend == "vllm":