A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Enhance SamplingParams (#96)
Woosuk Kwon committed
May 11, 2023 at 15:45 UTC
42f1042e1c71f89b7875a292d1adf3a8d01c6d49
7 files changed
+36
-54
benchmark/benchmark_latency.py
+11
-11
@@ -6,7 +6,7 @@ from tqdm import tqdm
6
import numpy as np
7
import torch
8
9
-from cacheflow.master.server import (
9
+from cacheflow.core.server import (
10
add_server_arguments, process_server_arguments,
11
init_local_server_and_frontend_with_arguments)
12
from cacheflow.sampling_params import SamplingParams
@@ -15,15 +15,14 @@ from cacheflow.sampling_params import SamplingParams
15
def main(args: argparse.Namespace):
16
server, frontend = init_local_server_and_frontend_with_arguments(args)
17
18
- sampling_params_dict = {
19
- 'n': args.n,
20
- 'temperature': 0.0 if args.use_beam_search else 1.0,
21
- 'top_p': 1.0,
22
- 'use_beam_search': args.use_beam_search,
23
- 'stop_token_ids': set(),
24
- 'max_num_steps': args.output_len,
25
- }
26
- sampling_params = SamplingParams.from_dict(sampling_params_dict)
18
+ sampling_params = SamplingParams(
19
+ n=args.n,
20
+ temperature=0.0 if args.use_beam_search else 1.0,
21
+ top_p=1.0,
22
+ use_beam_search=args.use_beam_search,
23
+ stop_token_ids=set(),
24
+ max_tokens=args.output_len,
25
+ )
26
print(sampling_params)
27
input_token_ids = [0] * args.input_len
28
@@ -31,7 +30,8 @@ def main(args: argparse.Namespace):
30
if profile:
31
torch.cuda.cudart().cudaProfilerStart()
32
for _ in range(args.batch_size):
34
- frontend._add_query(input_token_ids, sampling_params)
33
+ dummy_prompt = ""
34
+ frontend._add_query(dummy_prompt, input_token_ids, sampling_params)
35
server.add_sequence_groups(frontend.get_inputs())
36
start_time = time.time()
37
while True:
cacheflow/core/scheduler.py
+1
-1
@@ -316,7 +316,7 @@ class Scheduler:
316
continue
317
318
# Check if the sequence has reached the maximum number of steps.
319
- max_num_steps = self.sampling_params[group_id].max_num_steps
319
+ max_num_steps = self.sampling_params[group_id].max_tokens
320
if self.num_steps[group_id] == max_num_steps:
321
self._free_seq(seq)
322
continue
cacheflow/frontend/fastapi_frontend.py
+2
-2
@@ -89,8 +89,8 @@ class FastAPIServer:
89
90
async def generate(self, request_dict: Dict):
91
# Preprocess the request.
92
- prompt = request_dict["prompt"]
93
- sampling_params = SamplingParams.from_dict(request_dict)
92
+ prompt = request_dict.pop("prompt")
93
+ sampling_params = SamplingParams(**request_dict)
94
sampling_params.stop_token_ids.add(self.tokenizer.eos_token_id)
95
token_ids = self.tokenizer.encode(prompt)
96
seqs: List[Sequence] = []
cacheflow/model_executor/layers/sampler.py
+2
-2
@@ -367,7 +367,7 @@ def _sample(
367
next_token_ids = _sample_from_prompt(prob, sampling_params)
368
# Get top-k log probabilities for the next tokens.
369
next_logprobs = _get_topk_logprobs(
370
- logprob, sampling_params.num_logprobs)
370
+ logprob, sampling_params.logprobs)
371
372
# Build the output.
373
for seq_id, next_token_id in zip(seq_ids, next_token_ids):
@@ -392,7 +392,7 @@ def _sample(
392
next_logprobs: Dict[int, Dict[int, float]] = {}
393
for i, seq_id in enumerate(seq_ids):
394
next_logprobs[seq_id] = _get_topk_logprobs(
395
- logprob[i], sampling_params.num_logprobs)
395
+ logprob[i], sampling_params.logprobs)
396
397
# Build the output.
398
for seq_id, parent_seq_id, next_token_id in zip(
cacheflow/sampling_params.py
+18
-36
@@ -5,16 +5,16 @@ class SamplingParams:
5
6
def __init__(
7
self,
8
- n: int,
9
- presence_penalty: float,
10
- frequency_penalty: float,
11
- temperature: float,
12
- top_p: float,
13
- top_k: int,
14
- use_beam_search: bool,
15
- stop_token_ids: Set[int],
16
- max_num_steps: int,
17
- num_logprobs: int,
8
+ n: int = 1,
9
+ presence_penalty: float = 0.0,
10
+ frequency_penalty: float = 0.0,
11
+ temperature: float = 1.0,
12
+ top_p: float = 1.0,
13
+ top_k: int = -1,
14
+ use_beam_search: bool = False,
15
+ stop_token_ids: Set[int] = set(),
16
+ max_tokens: int = 16,
17
+ logprobs: int = 0,
18
) -> None:
19
if n < 1:
20
raise ValueError(f"n must be at least 1, got {n}.")
@@ -32,12 +32,12 @@ class SamplingParams:
32
if top_k < -1 or top_k == 0:
33
raise ValueError(f"top_k must be -1 (disable), or at least 1, "
34
f"got {top_k}.")
35
- if max_num_steps < 1:
35
+ if max_tokens < 1:
36
raise ValueError(
37
- f"max_num_steps must be at least 1, got {max_num_steps}.")
38
- if num_logprobs < 0:
37
+ f"max_tokens must be at least 1, got {max_tokens}.")
38
+ if logprobs < 0:
39
raise ValueError(
40
- f"num_logprobs must be non-negative, got {num_logprobs}.")
40
+ f"logprobs must be non-negative, got {logprobs}.")
41
42
if use_beam_search:
43
if n == 1:
@@ -72,8 +72,8 @@ class SamplingParams:
72
self.top_k = top_k
73
self.use_beam_search = use_beam_search
74
self.stop_token_ids = stop_token_ids
75
- self.max_num_steps = max_num_steps
76
- self.num_logprobs = num_logprobs
75
+ self.max_tokens = max_tokens
76
+ self.logprobs = logprobs
77
78
def __repr__(self) -> str:
79
return (f"SamplingParams(n={self.n}, "
@@ -84,23 +84,5 @@ class SamplingParams:
84
f"top_k={self.top_k},"
85
f"use_beam_search={self.use_beam_search}, "
86
f"stop_token_ids={self.stop_token_ids}, "
87
- f"max_num_steps={self.max_num_steps}, "
88
- f"num_logprobs={self.num_logprobs}")
89
-
90
- @classmethod
91
- def from_dict(cls, d: Dict) -> "SamplingParams":
92
- sampling_params = cls(
93
- n=d.pop("n", 1),
94
- presence_penalty=d.pop("presence_penalty", 0.0),
95
- frequency_penalty=d.pop("frequency_penalty", 0.0),
96
- temperature=d.pop("temperature", 1.0),
97
- top_p=d.pop("top_p", 1.0),
98
- top_k=d.pop("top_k", -1),
99
- use_beam_search=d.pop("use_beam_search", False),
100
- stop_token_ids=set(d.pop("stop_token_ids", set())),
101
- max_num_steps=d.pop("max_num_steps", 16),
102
- num_logprobs=d.pop("num_logprobs", 0),
103
- )
104
- if d:
105
- raise ValueError(f"Unrecognized keys in dict: {d.keys()}")
106
- return sampling_params
87
+ f"max_tokens={self.max_tokens}, "
88
+ f"logprobs={self.logprobs}")
gradio_webserver.py
+1
-1
@@ -10,7 +10,7 @@ def http_bot(prompt):
10
headers = {"User-Agent": "Cacheflow Client"}
11
pload = {
12
"prompt": prompt,
13
- "max_num_steps": 128,
13
+ "max_tokens": 128,
14
}
15
response = requests.post(args.model_url, headers=headers, json=pload, stream=True)
16
simple_server.py
+1
-1
@@ -18,7 +18,7 @@ def main(args: argparse.Namespace):
18
while True:
19
if test_inputs:
20
text, sampling_params_dict = test_inputs.pop(0)
21
- sampling_params = SamplingParams.from_dict(sampling_params_dict)
21
+ sampling_params = SamplingParams(**sampling_params_dict)
22
sampling_params = frontend.add_eos_token(sampling_params)
23
frontend.query(text, sampling_params)
24
server.add_sequence_groups(frontend.get_inputs())