A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix input_metadata.selected_token_indices in worker prepare_inputs (#1546)
Yanming W committed
Nov 9, 2023 at 06:19 UTC
8efe23f15087222540ec076ed00785544442c02f
2 files changed
+47
-1
tests/worker/test_worker.py
new
+44
@@ -0,0 +1,44 @@
1
+# pylint: disable=protected-access
2
+import random
3
+import torch
4
+
5
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
6
+from vllm.worker.worker import Worker
7
+
8
+
9
+def test_worker_prepare_inputs_for_prompt():
10
+ worker = Worker(None, None, None)
11
+ worker.block_size = 16
12
+ batch_size = random.randint(1, 256)
13
+ prompt_lens = []
14
+ seq_group_metadata_list = []
15
+ for i in range(batch_size):
16
+ # make sure all tokens fit into one block
17
+ prompt_len = i % (worker.block_size - 1) + 1
18
+ prompt_lens.append(prompt_len)
19
+ seq_data = list(range(prompt_len))
20
+ seq_group_metadata_list.append(
21
+ SequenceGroupMetadata(
22
+ request_id=f"test_{i}",
23
+ is_prompt=True,
24
+ seq_data={0: SequenceData(seq_data)},
25
+ sampling_params=SamplingParams(temperature=0),
26
+ block_tables={0: [1]},
27
+ ))
28
+ expected_selected_token_indices = []
29
+ selected_token_start_idx = 0
30
+ max_seq_len = max(prompt_lens)
31
+ for prompt_len in prompt_lens:
32
+ expected_selected_token_indices.append(selected_token_start_idx +
33
+ prompt_len - 1)
34
+ selected_token_start_idx += max_seq_len
35
+ input_tokens, input_positions, input_metadata = worker._prepare_inputs(
36
+ seq_group_metadata_list)
37
+ assert input_tokens.shape == input_positions.shape == (batch_size,
38
+ max_seq_len)
39
+ torch.testing.assert_close(input_tokens, input_positions)
40
+ actual = input_metadata.selected_token_indices
41
+ expected = torch.tensor(expected_selected_token_indices,
42
+ device=actual.device,
43
+ dtype=actual.dtype)
44
+ torch.testing.assert_close(actual, expected)
vllm/worker/worker.py
+3
-1
@@ -211,12 +211,14 @@ class Worker:
211
context_lens: List[int] = []
212
generation_block_tables: List[List[int]] = []
213
max_seq_len = max(prompt_lens) if prompt_lens else 1
214
- for seq_group_metadata in seq_group_metadata_list:
214
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
215
if seq_group_metadata.is_prompt:
216
# We need to do this in this loop as we need to know max_seq_len
217
assert len(
218
seq_ids) == 1, "Prompt input should have only one seq."
219
sampling_params = seq_group_metadata.sampling_params
220
+ assert len(prompt_lens) == len(seq_group_metadata_list)
221
+ prompt_len = prompt_lens[i]
222
if sampling_params.prompt_logprobs is not None:
223
selected_token_indices.extend(
224
range(selected_token_start_idx,