A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix sampler
Woosuk Kwon committed
Feb 23, 2023 at 20:30 UTC
de0fabbc5c84e6771d70b92014ae06fe82654ff0
2 files changed
+6
-8
cacheflow/models/opt.py
+3
-2
@@ -227,7 +227,7 @@ class OPTForCausalLM(OPTPreTrainedModel):
227
self.model = OPTModel(config)
228
# the lm_head weight is automatically tied to the embed tokens weight
229
self.lm_head = nn.Linear(config.word_embed_proj_dim, config.vocab_size, bias=False)
230
- self.sampler = Sampler(embedding=self.lm_head.weight)
230
+ self.sampler = Sampler()
231
232
# Initialize weights and apply final processing
233
self.post_init()
@@ -242,5 +242,6 @@ class OPTForCausalLM(OPTPreTrainedModel):
242
) -> Dict[int, Tuple[int, int]]:
243
hidden_states = self.model(
244
input_ids, positions, kv_caches, input_metadata, cache_events)
245
- next_tokens = self.sampler(hidden_states, input_metadata)
245
+ next_tokens = self.sampler(
246
+ self.lm_head.weight, hidden_states, input_metadata)
247
return next_tokens
cacheflow/models/sample.py
+3
-6
@@ -8,15 +8,12 @@ from cacheflow.models import InputMetadata
8
9
class Sampler(nn.Module):
10
11
- def __init__(
12
- self,
13
- embedding: torch.Tensor,
14
- ) -> None:
11
+ def __init__(self) -> None:
12
super().__init__()
16
- self.embedding = embedding # [vocab_size, hidden_size]
13
14
def forward(
15
self,
16
+ embedding: torch.Tensor,
17
hidden_states: torch.Tensor,
18
input_metadata: InputMetadata,
19
) -> Dict[int, Tuple[int, int]]:
@@ -31,7 +28,7 @@ class Sampler(nn.Module):
28
hidden_states = hidden_states[last_token_indicies]
29
30
# Get the logits for the next tokens.
34
- logits = torch.matmul(hidden_states, self.embedding.t())
31
+ logits = torch.matmul(hidden_states, embedding.t())
32
33
# Sample the next tokens.
34
# TODO(woosuk): Implement other sampling methods.