@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fdd0f2f472
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Minor

Woosuk Kwon committed Feb 23, 2023 at 20:23 UTC fdd0f2f4723afd8c45edacf9357c5a8973767da6
3 files changed +4 -3
cacheflow/master/scheduler.py
+1 -1
@@ -216,7 +216,7 @@ class Scheduler:
216 self.block_manager.fork(parent_seq, seq)
217
218 # Append a new token to the sequence.
219 - seq.append(next_token)
219 + seq.append([next_token])
220
221 # Check if the sequence has generated a stop token.
222 if next_token in stop_token_ids:
cacheflow/models/sample.py
+2 -2
@@ -13,7 +13,7 @@ class Sampler(nn.Module):
13 embedding: torch.Tensor,
14 ) -> None:
15 super().__init__()
16 - self.embedding = embedding.t() # [hidden_size, vocab_size]
16 + self.embedding = embedding # [vocab_size, hidden_size]
17
18 def forward(
19 self,
@@ -31,7 +31,7 @@ class Sampler(nn.Module):
31 hidden_states = hidden_states[last_token_indicies]
32
33 # Get the logits for the next tokens.
34 - logits = torch.matmul(hidden_states, self.embedding)
34 + logits = torch.matmul(hidden_states, self.embedding.t())
35
36 # Sample the next tokens.
37 # TODO(woosuk): Implement other sampling methods.
cacheflow/worker/worker.py
+1
@@ -165,6 +165,7 @@ class Worker:
165 output = self.model(
166 input_ids=input_tokens,
167 positions=input_positions,
168 + kv_caches=self.gpu_cache,
169 input_metadata=input_metadata,
170 cache_events=cache_events,
171 )