A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add greedy sampler
Woosuk Kwon committed
Feb 23, 2023 at 09:26 UTC
b56b6ca0d650c653c80ec113e27d6a8e640a4b2f
1 file changed
+45
cacheflow/models/sample.py
new
+45
@@ -0,0 +1,45 @@
1
+from typing import Dict, List, Tuple
2
+
3
+import torch
4
+import torch.nn as nn
5
+
6
+from cacheflow.models import InputMetadata
7
+
8
+
9
+class Sampler(nn.Module):
10
+
11
+ def __init__(
12
+ self,
13
+ embedding: torch.Tensor,
14
+ ) -> None:
15
+ super().__init__()
16
+ self.embedding = embedding.t() # [hidden_size, vocab_size]
17
+
18
+ def forward(
19
+ self,
20
+ hidden_states: torch.Tensor,
21
+ input_metadata: InputMetadata,
22
+ ) -> Dict[int, Tuple[int, int]]:
23
+ # Get the hidden states of the last tokens.
24
+ start_idx = 0
25
+ last_token_indicies: List[int] = []
26
+ for prompt_len in input_metadata.prompt_lens:
27
+ last_token_indicies.append(start_idx + prompt_len - 1)
28
+ start_idx += prompt_len
29
+ last_token_indicies.extend(
30
+ range(start_idx, start_idx + input_metadata.num_generation_tokens))
31
+ hidden_states = hidden_states[last_token_indicies]
32
+
33
+ # Get the logits for the next tokens.
34
+ logits = torch.matmul(hidden_states, self.embedding)
35
+
36
+ # Sample the next tokens.
37
+ # TODO(woosuk): Implement other sampling methods.
38
+ next_token_ids = torch.argmax(logits, dim=-1)
39
+ next_token_ids = next_token_ids.tolist()
40
+
41
+ # Return the next tokens.
42
+ next_tokens: Dict[int, Tuple[int, int]] = {}
43
+ for seq_id, token_id in zip(input_metadata.seq_ids, next_token_ids):
44
+ next_tokens[seq_id] = (seq_id, token_id)
45
+ return next_tokens