@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de0fabbc5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix sampler

Woosuk Kwon committed Feb 23, 2023 at 20:30 UTC de0fabbc5c84e6771d70b92014ae06fe82654ff0
2 files changed +6 -8
cacheflow/models/opt.py
+3 -2
@@ -227,7 +227,7 @@ class OPTForCausalLM(OPTPreTrainedModel):
227 self.model = OPTModel(config)
228 # the lm_head weight is automatically tied to the embed tokens weight
229 self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
230 - self.sampler = Sampler(embedding=self.lm_head.weight)
230 + self.sampler = Sampler()
231
232 # Initialize weights and apply final processing
233 self.post_init()
@@ -242,5 +242,6 @@ class OPTForCausalLM(OPTPreTrainedModel):
242 ) -> Dict[int, Tuple[int, int]]:
243 hidden_states = self.model(
244 input_ids, positions, kv_caches, input_metadata, cache_events)
245 - next_tokens = self.sampler(hidden_states, input_metadata)
245 + next_tokens = self.sampler(
246 + self.lm_head.weight, hidden_states, input_metadata)
247 return next_tokens
cacheflow/models/sample.py
+3 -6
@@ -8,15 +8,12 @@ from cacheflow.models import InputMetadata
8
9 class Sampler(nn.Module):
10
11 - def __init__(
12 - self,
13 - embedding: torch.Tensor,
14 - ) -> None:
11 + def __init__(self) -> None:
12 super().__init__()
16 - self.embedding = embedding # [vocab_size, hidden_size]
13
14 def forward(
15 self,
16 + embedding: torch.Tensor,
17 hidden_states: torch.Tensor,
18 input_metadata: InputMetadata,
19 ) -> Dict[int, Tuple[int, int]]:
@@ -31,7 +28,7 @@ class Sampler(nn.Module):
28 hidden_states = hidden_states[last_token_indicies]
29
30 # Get the logits for the next tokens.
34 - logits = torch.matmul(hidden_states, self.embedding.t())
31 + logits = torch.matmul(hidden_states, embedding.t())
32
33 # Sample the next tokens.
34 # TODO(woosuk): Implement other sampling methods.