@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 42f1042e1c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Enhance SamplingParams (#96)

Woosuk Kwon committed May 11, 2023 at 15:45 UTC 42f1042e1c71f89b7875a292d1adf3a8d01c6d49
7 files changed +36 -54
benchmark/benchmark_latency.py
+11 -11
@@ -6,7 +6,7 @@ from tqdm import tqdm
6 import numpy as np
7 import torch
8
9 -from cacheflow.master.server import (
9 +from cacheflow.core.server import (
10 add_server_arguments, process_server_arguments,
11 init_local_server_and_frontend_with_arguments)
12 from cacheflow.sampling_params import SamplingParams
@@ -15,15 +15,14 @@ from cacheflow.sampling_params import SamplingParams
15 def main(args: argparse.Namespace):
16 server, frontend = init_local_server_and_frontend_with_arguments(args)
17
18 - sampling_params_dict = {
19 - 'n': args.n,
20 - 'temperature': 0.0 if args.use_beam_search else 1.0,
21 - 'top_p': 1.0,
22 - 'use_beam_search': args.use_beam_search,
23 - 'stop_token_ids': set(),
24 - 'max_num_steps': args.output_len,
25 - }
26 - sampling_params = SamplingParams.from_dict(sampling_params_dict)
18 + sampling_params = SamplingParams(
19 + n=args.n,
20 + temperature=0.0 if args.use_beam_search else 1.0,
21 + top_p=1.0,
22 + use_beam_search=args.use_beam_search,
23 + stop_token_ids=set(),
24 + max_tokens=args.output_len,
25 + )
26 print(sampling_params)
27 input_token_ids = [0] * args.input_len
28
@@ -31,7 +30,8 @@ def main(args: argparse.Namespace):
30 if profile:
31 torch.cuda.cudart().cudaProfilerStart()
32 for _ in range(args.batch_size):
34 - frontend._add_query(input_token_ids, sampling_params)
33 + dummy_prompt = ""
34 + frontend._add_query(dummy_prompt, input_token_ids, sampling_params)
35 server.add_sequence_groups(frontend.get_inputs())
36 start_time = time.time()
37 while True:
cacheflow/core/scheduler.py
+1 -1
@@ -316,7 +316,7 @@ class Scheduler:
316 continue
317
318 # Check if the sequence has reached the maximum number of steps.
319 - max_num_steps = self.sampling_params[group_id].max_num_steps
319 + max_num_steps = self.sampling_params[group_id].max_tokens
320 if self.num_steps[group_id] == max_num_steps:
321 self._free_seq(seq)
322 continue
cacheflow/frontend/fastapi_frontend.py
+2 -2
@@ -89,8 +89,8 @@ class FastAPIServer:
89
90 async def generate(self, request_dict: Dict):
91 # Preprocess the request.
92 - prompt = request_dict["prompt"]
93 - sampling_params = SamplingParams.from_dict(request_dict)
92 + prompt = request_dict.pop("prompt")
93 + sampling_params = SamplingParams(**request_dict)
94 sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
95 token_ids = self.tokenizer.encode(prompt)
96 seqs: List[Sequence] = []
cacheflow/model_executor/layers/sampler.py
+2 -2
@@ -367,7 +367,7 @@ def _sample(
367 next_token_ids = _sample_from_prompt(prob, sampling_params)
368 # Get top-k log probabilities for the next tokens.
369 next_logprobs = _get_topk_logprobs(
370 - logprob, sampling_params.num_logprobs)
370 + logprob, sampling_params.logprobs)
371
372 # Build the output.
373 for seq_id, next_token_id in zip(seq_ids, next_token_ids):
@@ -392,7 +392,7 @@ def _sample(
392 next_logprobs: Dict[int, Dict[int, float]] = {}
393 for i, seq_id in enumerate(seq_ids):
394 next_logprobs[seq_id] = _get_topk_logprobs(
395 - logprob[i], sampling_params.num_logprobs)
395 + logprob[i], sampling_params.logprobs)
396
397 # Build the output.
398 for seq_id, parent_seq_id, next_token_id in zip(
cacheflow/sampling_params.py
+18 -36
@@ -5,16 +5,16 @@ class SamplingParams:
5
6 def __init__(
7 self,
8 - n: int,
9 - presence_penalty: float,
10 - frequency_penalty: float,
11 - temperature: float,
12 - top_p: float,
13 - top_k: int,
14 - use_beam_search: bool,
15 - stop_token_ids: Set[int],
16 - max_num_steps: int,
17 - num_logprobs: int,
8 + n: int = 1,
9 + presence_penalty: float = 0.0,
10 + frequency_penalty: float = 0.0,
11 + temperature: float = 1.0,
12 + top_p: float = 1.0,
13 + top_k: int = -1,
14 + use_beam_search: bool = False,
15 + stop_token_ids: Set[int] = set(),
16 + max_tokens: int = 16,
17 + logprobs: int = 0,
18 ) -> None:
19 if n < 1:
20 raise ValueError(f"n must be at least 1, got {n}.")
@@ -32,12 +32,12 @@ class SamplingParams:
32 if top_k < -1 or top_k == 0:
33 raise ValueError(f"top_k must be -1 (disable), or at least 1, "
34 f"got {top_k}.")
35 - if max_num_steps < 1:
35 + if max_tokens < 1:
36 raise ValueError(
37 - f"max_num_steps must be at least 1, got {max_num_steps}.")
38 - if num_logprobs < 0:
37 + f"max_tokens must be at least 1, got {max_tokens}.")
38 + if logprobs < 0:
39 raise ValueError(
40 - f"num_logprobs must be non-negative, got {num_logprobs}.")
40 + f"logprobs must be non-negative, got {logprobs}.")
41
42 if use_beam_search:
43 if n == 1:
@@ -72,8 +72,8 @@ class SamplingParams:
72 self.top_k = top_k
73 self.use_beam_search = use_beam_search
74 self.stop_token_ids = stop_token_ids
75 - self.max_num_steps = max_num_steps
76 - self.num_logprobs = num_logprobs
75 + self.max_tokens = max_tokens
76 + self.logprobs = logprobs
77
78 def __repr__(self) -> str:
79 return (f"SamplingParams(n={self.n}, "
@@ -84,23 +84,5 @@ class SamplingParams:
84 f"top_k={self.top_k},"
85 f"use_beam_search={self.use_beam_search}, "
86 f"stop_token_ids={self.stop_token_ids}, "
87 - f"max_num_steps={self.max_num_steps}, "
88 - f"num_logprobs={self.num_logprobs}")
89 -
90 - @classmethod
91 - def from_dict(cls, d: Dict) -> "SamplingParams":
92 - sampling_params = cls(
93 - n=d.pop("n", 1),
94 - presence_penalty=d.pop("presence_penalty", 0.0),
95 - frequency_penalty=d.pop("frequency_penalty", 0.0),
96 - temperature=d.pop("temperature", 1.0),
97 - top_p=d.pop("top_p", 1.0),
98 - top_k=d.pop("top_k", -1),
99 - use_beam_search=d.pop("use_beam_search", False),
100 - stop_token_ids=set(d.pop("stop_token_ids", set())),
101 - max_num_steps=d.pop("max_num_steps", 16),
102 - num_logprobs=d.pop("num_logprobs", 0),
103 - )
104 - if d:
105 - raise ValueError(f"Unrecognized keys in dict: {d.keys()}")
106 - return sampling_params
87 + f"max_tokens={self.max_tokens}, "
88 + f"logprobs={self.logprobs}")
gradio_webserver.py
+1 -1
@@ -10,7 +10,7 @@ def http_bot(prompt):
10 headers = {"User-Agent": "Cacheflow Client"}
11 pload = {
12 "prompt": prompt,
13 - "max_num_steps": 128,
13 + "max_tokens": 128,
14 }
15 response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
16
simple_server.py
+1 -1
@@ -18,7 +18,7 @@ def main(args: argparse.Namespace):
18 while True:
19 if test_inputs:
20 text, sampling_params_dict = test_inputs.pop(0)
21 - sampling_params = SamplingParams.from_dict(sampling_params_dict)
21 + sampling_params = SamplingParams(**sampling_params_dict)
22 sampling_params = frontend.add_eos_token(sampling_params)
23 frontend.query(text, sampling_params)
24 server.add_sequence_groups(frontend.get_inputs())