@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04e5acc08e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix a bug in 1D input shape (#5)

Woosuk Kwon committed Mar 6, 2023 at 10:05 UTC 04e5acc08ed5b878225491bf62540ea10274fb29
3 files changed +11 -6
cacheflow/models/attention.py
+8 -3
@@ -47,9 +47,8 @@ class OPTCacheFlowAttention(nn.Module):
47 max_s=max_prompt_len,
48 causal=True,
49 )[0]
50 - num_tokens = prefix_sum[-1]
50 # FIXME(woosuk): Unnecessary copy. Optimize this.
52 - output[:num_tokens].copy_(out, non_blocking=True)
51 + output.copy_(out, non_blocking=True)
52
53 def single_query_cached_kv_attention(
54 self,
@@ -108,8 +107,14 @@ class OPTCacheFlowAttention(nn.Module):
107
108 # Compute the attention op for prompts.
109 if input_metadata.num_prompts > 0:
110 + num_prompt_tokens = sum(input_metadata.prompt_lens)
111 self.multi_query_kv_attention(
112 - output, query, key, value, input_metadata.prompt_lens)
112 + output[:num_prompt_tokens],
113 + query[:num_prompt_tokens],
114 + key[:num_prompt_tokens],
115 + value[:num_prompt_tokens],
116 + input_metadata.prompt_lens,
117 + )
118
119 # Wait until the cache op is done.
120 if cache_event is not None:
cacheflow/models/input_metadata.py
+1 -1
@@ -24,7 +24,7 @@ class InputMetadata:
24
25 self.num_prompts = len(prompt_lens)
26 self.num_generation_tokens = context_lens.shape[0]
27 - self.num_valid_tokens = len(slot_mapping)
27 + self.num_valid_tokens = slot_mapping.shape[0]
28 if block_tables.numel() > 0:
29 self.max_num_blocks_per_seq = block_tables.shape[1]
30 else:
server.py
+2 -2
@@ -57,11 +57,11 @@ def main():
57 'UC Berkeley is',
58 'The future of cloud computing is',
59 ]
60 - for prompt in test_inputs:
61 - frontend.query(prompt)
60
61 # FIXME
62 while True:
63 + if test_inputs:
64 + frontend.query(test_inputs.pop())
65 scheduler.step()
66 if not scheduler.pending and not scheduler.running:
67 break