A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Minor
Woosuk Kwon committed
Feb 23, 2023 at 20:23 UTC
fdd0f2f4723afd8c45edacf9357c5a8973767da6
3 files changed
+4
-3
cacheflow/master/scheduler.py
+1
-1
@@ -216,7 +216,7 @@ class Scheduler:
216
self.block_manager.fork(parent_seq, seq)
217
218
# Append a new token to the sequence.
219
- seq.append(next_token)
219
+ seq.append([next_token])
220
221
# Check if the sequence has generated a stop token.
222
if next_token in stop_token_ids:
cacheflow/models/sample.py
+2
-2
@@ -13,7 +13,7 @@ class Sampler(nn.Module):
13
embedding: torch.Tensor,
14
) -> None:
15
super().__init__()
16
- self.embedding = embedding.t() # [hidden_size, vocab_size]
16
+ self.embedding = embedding # [vocab_size, hidden_size]
17
18
def forward(
19
self,
@@ -31,7 +31,7 @@ class Sampler(nn.Module):
31
hidden_states = hidden_states[last_token_indicies]
32
33
# Get the logits for the next tokens.
34
- logits = torch.matmul(hidden_states, self.embedding)
34
+ logits = torch.matmul(hidden_states, self.embedding.t())
35
36
# Sample the next tokens.
37
# TODO(woosuk): Implement other sampling methods.
cacheflow/worker/worker.py
+1
@@ -165,6 +165,7 @@ class Worker:
165
output = self.model(
166
input_ids=input_tokens,
167
positions=input_positions,
168
+ kv_caches=self.gpu_cache,
169
input_metadata=input_metadata,
170
cache_events=cache_events,
171
)