A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix a bug in 1D input shape (#5)
Woosuk Kwon committed
Mar 6, 2023 at 10:05 UTC
04e5acc08ed5b878225491bf62540ea10274fb29
3 files changed
+11
-6
cacheflow/models/attention.py
+8
-3
@@ -47,9 +47,8 @@ class OPTCacheFlowAttention(nn.Module):
47
max_s=max_prompt_len,
48
causal=True,
49
)[0]
50
- num_tokens = prefix_sum[-1]
50
# FIXME(woosuk): Unnecessary copy. Optimize this.
52
- output[:num_tokens].copy_(out, non_blocking=True)
51
+ output.copy_(out, non_blocking=True)
52
53
def single_query_cached_kv_attention(
54
self,
@@ -108,8 +107,14 @@ class OPTCacheFlowAttention(nn.Module):
107
108
# Compute the attention op for prompts.
109
if input_metadata.num_prompts > 0:
110
+ num_prompt_tokens = sum(input_metadata.prompt_lens)
111
self.multi_query_kv_attention(
112
- output, query, key, value, input_metadata.prompt_lens)
112
+ output[:num_prompt_tokens],
113
+ query[:num_prompt_tokens],
114
+ key[:num_prompt_tokens],
115
+ value[:num_prompt_tokens],
116
+ input_metadata.prompt_lens,
117
+ )
118
119
# Wait until the cache op is done.
120
if cache_event is not None:
cacheflow/models/input_metadata.py
+1
-1
@@ -24,7 +24,7 @@ class InputMetadata:
24
25
self.num_prompts = len(prompt_lens)
26
self.num_generation_tokens = context_lens.shape[0]
27
- self.num_valid_tokens = len(slot_mapping)
27
+ self.num_valid_tokens = slot_mapping.shape[0]
28
if block_tables.numel() > 0:
29
self.max_num_blocks_per_seq = block_tables.shape[1]
30
else:
server.py
+2
-2
@@ -57,11 +57,11 @@ def main():
57
'UC Berkeley is',
58
'The future of cloud computing is',
59
]
60
- for prompt in test_inputs:
61
- frontend.query(prompt)
60
61
# FIXME
62
while True:
63
+ if test_inputs:
64
+ frontend.query(test_inputs.pop())
65
scheduler.step()
66
if not scheduler.pending and not scheduler.running:
67
break