@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8c4b2592fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix: enable trust-remote-code in api server & benchmark. (#509)

Ricardo Lu committed Jul 20, 2023 at 08:06 UTC 8c4b2592fb953d1a8f880d42ebb1b28eaa94d0a6
4 files changed +14 -6
benchmarks/benchmark_latency.py
+3
@@ -21,6 +21,7 @@ def main(args: argparse.Namespace):
21 tensor_parallel_size=args.tensor_parallel_size,
22 max_num_seqs=args.batch_size,
23 max_num_batched_tokens=args.batch_size * args.input_len,
24 + trust_remote_code=args.trust_remote_code,
25 )
26
27 sampling_params = SamplingParams(
@@ -74,5 +75,7 @@ if __name__ == '__main__':
75 parser.add_argument('--use-beam-search', action='store_true')
76 parser.add_argument('--num-iters', type=int, default=3,
77 help='Number of iterations to run.')
78 + parser.add_argument('--trust-remote-code', action='store_true',
79 + help='trust remote code from huggingface')
80 args = parser.parse_args()
81 main(args)
benchmarks/benchmark_serving.py
+3 -1
@@ -177,7 +177,7 @@ def main(args: argparse.Namespace):
177 np.random.seed(args.seed)
178
179 api_url = f"http://{args.host}:{args.port}/generate"
180 - tokenizer = get_tokenizer(args.tokenizer)
180 + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
181 input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
182
183 benchmark_start_time = time.time()
@@ -227,5 +227,7 @@ if __name__ == "__main__":
227 "Otherwise, we use Poisson process to synthesize "
228 "the request arrival times.")
229 parser.add_argument("--seed", type=int, default=0)
230 + parser.add_argument('--trust-remote-code', action='store_true',
231 + help='trust remote code from huggingface')
232 args = parser.parse_args()
233 main(args)
benchmarks/benchmark_throughput.py
+6 -4
@@ -74,7 +74,7 @@ def run_vllm(
74 tokenizer=tokenizer,
75 tensor_parallel_size=tensor_parallel_size,
76 seed=seed,
77 - trust_remote_code=trust_remote_code
77 + trust_remote_code=trust_remote_code,
78 )
79
80 # Add the requests to the engine.
@@ -111,7 +111,8 @@ def run_hf(
111 trust_remote_code: bool,
112 ) -> float:
113 assert not use_beam_search
114 - llm = AutoModelForCausalLM.from_pretrained(model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
114 + llm = AutoModelForCausalLM.from_pretrained(model,
115 + torch_dtype=torch.float16, trust_remote_code=trust_remote_code)
116 if llm.config.model_type == "llama":
117 # To enable padding in the HF backend.
118 tokenizer.pad_token = tokenizer.eos_token
@@ -173,8 +174,9 @@ def main(args: argparse.Namespace):
174 args.seed, args.n, args.use_beam_search, args.trust_remote_code)
175 elif args.backend == "hf":
176 assert args.tensor_parallel_size == 1
176 - elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
177 - args.use_beam_search, args.hf_max_batch_size)
177 + elapsed_time = run_hf(
178 + requests, args.model, tokenizer, args.n, args.use_beam_search,
179 + args.hf_max_batch_size, args.trust_remote_code)
180 else:
181 raise ValueError(f"Unknown backend: {args.backend}")
182 total_num_tokens = sum(
vllm/entrypoints/openai/api_server.py
+2 -1
@@ -585,7 +585,8 @@ if __name__ == "__main__":
585
586 # A separate tokenizer to map token IDs to strings.
587 tokenizer = get_tokenizer(engine_args.tokenizer,
588 - tokenizer_mode=engine_args.tokenizer_mode)
588 + tokenizer_mode=engine_args.tokenizer_mode,
589 + trust_remote_code=engine_args.trust_remote_code)
590
591 uvicorn.run(app,
592 host=args.host,