@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f88db35da
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support top-k sampling (#94)

Woosuk Kwon committed May 10, 2023 at 12:51 UTC 9f88db35da012544a9cd9450c68b8df2e6509b92
3 files changed +78 -43
cacheflow/model_executor/layers/sampler.py
+37 -16
@@ -46,12 +46,13 @@ class Sampler(nn.Module):
46 # Compute the log probabilities (before applying top-p).
47 logprobs = torch.log(probs)
48
49 - # Apply top-p truncation.
50 - top_ps = _get_top_ps(input_metadata)
51 - assert len(top_ps) == probs.shape[0]
52 - if any(p < 1.0 for p in top_ps):
49 + # Apply top-p and top-k truncation.
50 + top_ps, top_ks = _get_top_p_top_k(input_metadata, self.vocab_size)
51 + assert len(top_ps) == len(top_ks) == probs.shape[0]
52 + if any(p < 1.0 for p in top_ps) or any(k != -1 for k in top_ks):
53 p = torch.tensor(top_ps, dtype=probs.dtype, device=probs.device)
54 - probs = _apply_top_p(probs, p)
54 + k = torch.tensor(top_ks, dtype=torch.int, device=probs.device)
55 + probs = _apply_top_p_top_k(probs, p, k)
56
57 # Sample the next tokens.
58 return _sample(probs, logprobs, input_metadata)
@@ -94,31 +95,51 @@ def _get_temperatures(
95 return temperatures
96
97
97 -def _get_top_ps(
98 +def _get_top_p_top_k(
99 input_metadata: InputMetadata,
99 -) -> List[float]:
100 + vocab_size: int,
101 +) -> Tuple[List[float], List[int]]:
102 top_ps: List[float] = []
103 + top_ks: List[int] = []
104 for i, seq_group in enumerate(input_metadata.seq_groups):
105 seq_ids, sampling_params = seq_group
106 + top_p = sampling_params.top_p
107 + # k should not be greater than the vocab size.
108 + top_k = min(sampling_params.top_k, vocab_size)
109 + # k=-1 means no truncation.
110 + top_k = vocab_size if top_k == -1 else top_k
111 if i < input_metadata.num_prompts:
112 # A prompt input.
105 - top_ps.append(sampling_params.top_p)
113 + top_ps.append(top_p)
114 + top_ks.append(top_k)
115 else:
116 # A generation token.
108 - top_ps += [sampling_params.top_p] * len(seq_ids)
109 - return top_ps
117 + top_ps += [top_p] * len(seq_ids)
118 + top_ks += [top_k] * len(seq_ids)
119 + return top_ps, top_ks
120
121
112 -def _apply_top_p(
122 +def _apply_top_p_top_k(
123 probs: torch.Tensor,
124 p: torch.Tensor,
125 + k: torch.Tensor,
126 ) -> torch.Tensor:
127 # TODO(woosuk): Optimize.
128 probs_sort, probs_idx = probs.sort(dim=-1, descending=True)
129 +
130 + # Apply top-p.
131 probs_sum = torch.cumsum(probs_sort, dim=-1)
119 - mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
120 - probs_sort[mask] = 0.0
121 - probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True))
132 + top_p_mask = (probs_sum - probs_sort) > p.unsqueeze(dim=1)
133 + probs_sort[top_p_mask] = 0.0
134 +
135 + # Apply top-k.
136 + # Create a mask for the top-k elements.
137 + top_k_mask = torch.arange(probs_idx.shape[-1], device=probs_idx.device)
138 + top_k_mask = top_k_mask.expand(probs_idx.shape[0], -1)
139 + top_k_mask = top_k_mask >= k.unsqueeze(dim=1)
140 + probs_sort[top_k_mask] = 0.0
141 +
142 + # Re-sort the probabilities.
143 probs = torch.gather(
144 probs_sort, dim=-1, index=torch.argsort(probs_idx, dim=-1))
145 return probs
@@ -160,7 +181,7 @@ def _sample_from_prompt(
181 next_token_id = torch.argmax(prob)
182 next_token_ids = [next_token_id.item()]
183 else:
163 - # Neucleus sampling.
184 + # Random sampling.
185 # Sample n tokens for the prompt.
186 n = sampling_params.n
187 next_token_ids = torch.multinomial(
@@ -218,7 +239,7 @@ def _sample_from_generation_tokens(
239 next_token_ids = [next_token_id.item()]
240 parent_seq_ids = seq_ids
241 else:
221 - # Neucleus sampling.
242 + # Random sampling.
243 # Sample 1 token for each sequence in the group.
244 next_token_ids = torch.multinomial(
245 probs, num_samples=1, replacement=True)
cacheflow/sampling_params.py
+38 -25
@@ -8,69 +8,82 @@ class SamplingParams:
8 n: int,
9 temperature: float,
10 top_p: float,
11 + top_k: int,
12 use_beam_search: bool,
13 stop_token_ids: Set[int],
14 max_num_steps: int,
15 num_logprobs: int,
16 ) -> None:
17 if n < 1:
17 - raise ValueError(f'n must be at least 1, got {n}.')
18 + raise ValueError(f"n must be at least 1, got {n}.")
19 if temperature < 0.0:
20 raise ValueError(
20 - f'temperature must be non-negative, got {temperature}.')
21 + f"temperature must be non-negative, got {temperature}.")
22 if not 0.0 < top_p <= 1.0:
22 - raise ValueError(f'top_p must be in (0, 1], got {top_p}.')
23 + raise ValueError(f"top_p must be in (0, 1], got {top_p}.")
24 + if top_k < -1 or top_k == 0:
25 + raise ValueError(f"top_k must be -1 (disable), or at least 1, "
26 + f"got {top_k}.")
27 if max_num_steps < 1:
28 raise ValueError(
25 - f'max_num_steps must be at least 1, got {max_num_steps}.')
29 + f"max_num_steps must be at least 1, got {max_num_steps}.")
30 if num_logprobs < 0:
31 raise ValueError(
28 - f'num_logprobs must be non-negative, got {num_logprobs}.')
32 + f"num_logprobs must be non-negative, got {num_logprobs}.")
33
34 if use_beam_search:
35 if n == 1:
36 raise ValueError(
33 - 'n must be greater than 1 when using beam search.')
37 + "n must be greater than 1 when using beam search.")
38 if temperature > 0.0:
39 raise ValueError(
36 - 'temperature must be 0 when using beam search.')
40 + "temperature must be 0 when using beam search.")
41 if top_p < 1.0:
42 raise ValueError(
39 - 'top_p must be 1 when using beam search.')
43 + "top_p must be 1 when using beam search.")
44 + if top_k != -1:
45 + raise ValueError(
46 + "top_k must be -1 when using beam search.")
47 elif temperature == 0.0:
48 # Zero temperature means greedy sampling.
49 if n > 1:
50 raise ValueError(
44 - 'n must be 1 when using greedy sampling.')
51 + "n must be 1 when using greedy sampling.")
52 if top_p < 1.0:
53 raise ValueError(
47 - 'top_p must be 1 when using greedy sampling.')
54 + "top_p must be 1 when using greedy sampling.")
55 + if top_k != -1:
56 + raise ValueError(
57 + "top_k must be -1 when using greedy sampling.")
58
59 self.n = n
60 self.temperature = temperature
61 self.top_p = top_p
62 + self.top_k = top_k
63 self.use_beam_search = use_beam_search
64 self.stop_token_ids = stop_token_ids
65 self.max_num_steps = max_num_steps
66 self.num_logprobs = num_logprobs
67
68 def __repr__(self) -> str:
58 - return (f'SamplingParams(n={self.n}, '
59 - f'temperature={self.temperature}, '
60 - f'top_p={self.top_p}, '
61 - f'use_beam_search={self.use_beam_search}, '
62 - f'stop_token_ids={self.stop_token_ids}, '
63 - f'max_num_steps={self.max_num_steps}, '
64 - f'num_logprobs={self.num_logprobs}')
69 + return (f"SamplingParams(n={self.n}, "
70 + f"temperature={self.temperature}, "
71 + f"top_p={self.top_p}, "
72 + f"top_k={self.top_k},"
73 + f"use_beam_search={self.use_beam_search}, "
74 + f"stop_token_ids={self.stop_token_ids}, "
75 + f"max_num_steps={self.max_num_steps}, "
76 + f"num_logprobs={self.num_logprobs}")
77
78 @classmethod
67 - def from_dict(cls, d: Dict) -> 'SamplingParams':
79 + def from_dict(cls, d: Dict) -> "SamplingParams":
80 return cls(
69 - n=d.get('n', 1),
70 - temperature=d.get('temperature', 1.0),
71 - top_p=d.get('top_p', 1.0),
72 - use_beam_search=d.get('use_beam_search', False),
73 - stop_token_ids=set(d.get('stop_token_ids', set())),
74 - max_num_steps=d.get('max_num_steps', 16),
75 - num_logprobs=d.get('num_logprobs', 0),
81 + n=d.get("n", 1),
82 + temperature=d.get("temperature", 1.0),
83 + top_p=d.get("top_p", 1.0),
84 + top_k=d.get("top_k", -1),
85 + use_beam_search=d.get("use_beam_search", False),
86 + stop_token_ids=set(d.get("stop_token_ids", set())),
87 + max_num_steps=d.get("max_num_steps", 16),
88 + num_logprobs=d.get("num_logprobs", 0),
89 )
simple_server.py
+3 -2
@@ -11,8 +11,9 @@ def main(args: argparse.Namespace):
11 # Test the following inputs.
12 test_inputs = [
13 ("A robot may not injure a human being", {}), # Use default parameters.
14 - ("What is the meaning of life?", {"n": 3, "temperature": 0.8, "top_p": 0.99}),
15 - ("It is only with the heart that one can see rightly", {"n": 4, "use_beam_search": True, "temperature": 0.0}),
14 + ("To be or not to be,", {"temperature": 0.8, "top_k": 5}),
15 + ("What is the meaning of life?", {"n": 2, "temperature": 0.8, "top_p": 0.95}),
16 + ("It is only with the heart that one can see rightly", {"n": 3, "use_beam_search": True, "temperature": 0.0}),
17 ]
18 while True:
19 if test_inputs: