@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b56b6ca0d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add greedy sampler

Woosuk Kwon committed Feb 23, 2023 at 09:26 UTC b56b6ca0d650c653c80ec113e27d6a8e640a4b2f
1 file changed +45
cacheflow/models/sample.py new
+45
@@ -0,0 +1,45 @@
1 +from typing import Dict, List, Tuple
2 +
3 +import torch
4 +import torch.nn as nn
5 +
6 +from cacheflow.models import InputMetadata
7 +
8 +
9 +class Sampler(nn.Module):
10 +
11 + def __init__(
12 + self,
13 + embedding: torch.Tensor,
14 + ) -> None:
15 + super().__init__()
16 + self.embedding = embedding.t() # [hidden_size, vocab_size]
17 +
18 + def forward(
19 + self,
20 + hidden_states: torch.Tensor,
21 + input_metadata: InputMetadata,
22 + ) -> Dict[int, Tuple[int, int]]:
23 + # Get the hidden states of the last tokens.
24 + start_idx = 0
25 + last_token_indicies: List[int] = []
26 + for prompt_len in input_metadata.prompt_lens:
27 + last_token_indicies.append(start_idx + prompt_len - 1)
28 + start_idx += prompt_len
29 + last_token_indicies.extend(
30 + range(start_idx, start_idx + input_metadata.num_generation_tokens))
31 + hidden_states = hidden_states[last_token_indicies]
32 +
33 + # Get the logits for the next tokens.
34 + logits = torch.matmul(hidden_states, self.embedding)
35 +
36 + # Sample the next tokens.
37 + # TODO(woosuk): Implement other sampling methods.
38 + next_token_ids = torch.argmax(logits, dim=-1)
39 + next_token_ids = next_token_ids.tolist()
40 +
41 + # Return the next tokens.
42 + next_tokens: Dict[int, Tuple[int, int]] = {}
43 + for seq_id, token_id in zip(input_metadata.seq_ids, next_token_ids):
44 + next_tokens[seq_id] = (seq_id, token_id)
45 + return next_tokens