A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix: enable trust-remote-code in api server & benchmark. (#509)
Ricardo Lu committed
Jul 20, 2023 at 08:06 UTC
8c4b2592fb953d1a8f880d42ebb1b28eaa94d0a6
4 files changed
+14
-6
benchmarks/benchmark_latency.py
+3
@@ -21,6 +21,7 @@ def main(args: argparse.Namespace):
21
tensor_parallel_size=args.tensor_parallel_size,
22
max_num_seqs=args.batch_size,
23
max_num_batched_tokens=args.batch_size * args.input_len,
24
+ trust_remote_code=args.trust_remote_code,
25
)
26
27
sampling_params = SamplingParams(
@@ -74,5 +75,7 @@ if __name__ == '__main__':
75
parser.add_argument('--use-beam-search', action='store_true')
76
parser.add_argument('--num-iters', type=int, default=3,
77
help='Number of iterations to run.')
78
+ parser.add_argument('--trust-remote-code', action='store_true',
79
+ help='trust remote code from huggingface')
80
args = parser.parse_args()
81
main(args)
benchmarks/benchmark_serving.py
+3
-1
@@ -177,7 +177,7 @@ def main(args: argparse.Namespace):
177
np.random.seed(args.seed)
178
179
api_url = f"http://{args.host}:{args.port}/generate"
180
- tokenizer = get_tokenizer(args.tokenizer)
180
+ tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
181
input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
182
183
benchmark_start_time = time.time()
@@ -227,5 +227,7 @@ if __name__ == "__main__":
227
"Otherwise, we use Poisson process to synthesize "
228
"the request arrival times.")
229
parser.add_argument("--seed", type=int, default=0)
230
+ parser.add_argument('--trust-remote-code', action='store_true',
231
+ help='trust remote code from huggingface')
232
args = parser.parse_args()
233
main(args)
benchmarks/benchmark_throughput.py
+6
-4
@@ -74,7 +74,7 @@ def run_vllm(
74
tokenizer=tokenizer,
75
tensor_parallel_size=tensor_parallel_size,
76
seed=seed,
77
- trust_remote_code=trust_remote_code
77
+ trust_remote_code=trust_remote_code,
78
)
79
80
# Add the requests to the engine.
@@ -111,7 +111,8 @@ def run_hf(
111
trust_remote_code: bool,
112
) -> float:
113
assert not use_beam_search
114
- llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
114
+ llm = AutoModelForCausalLM.from_pretrained(model,
115
+ torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
116
if llm.config.model_type == "llama":
117
# To enable padding in the HF backend.
118
tokenizer.pad_token = tokenizer.eos_token
@@ -173,8 +174,9 @@ def main(args: argparse.Namespace):
174
args.seed, args.n, args.use_beam_search, args.trust_remote_code)
175
elif args.backend == "hf":
176
assert args.tensor_parallel_size == 1
176
- elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
177
- args.use_beam_search, args.hf_max_batch_size)
177
+ elapsed_time = run_hf(
178
+ requests, args.model, tokenizer, args.n, args.use_beam_search,
179
+ args.hf_max_batch_size, args.trust_remote_code)
180
else:
181
raise ValueError(f"Unknown backend: {args.backend}")
182
total_num_tokens = sum(
vllm/entrypoints/openai/api_server.py
+2
-1
@@ -585,7 +585,8 @@ if __name__ == "__main__":
585
586
# A separate tokenizer to map token IDs to strings.
587
tokenizer = get_tokenizer(engine_args.tokenizer,
588
- tokenizer_mode=engine_args.tokenizer_mode)
588
+ tokenizer_mode=engine_args.tokenizer_mode,
589
+ trust_remote_code=engine_args.trust_remote_code)
590
591
uvicorn.run(app,
592
host=args.host,