A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Factor out common code in `SequenceData` and `Sequence` (#8675)
Cyrus Leung committed
Sep 21, 2024 at 10:30 UTC
0455c46ed434d70f0a6219204e89ee04f1d01336
8 files changed
+64
-97
tests/samplers/test_sampler.py
+7
-20
@@ -1,6 +1,5 @@
1
import itertools
2
import random
3
-from array import array
3
from typing import Dict, List, Optional, Tuple
4
from unittest.mock import Mock, patch
5
@@ -12,8 +11,7 @@ import vllm.envs as envs
11
from vllm.model_executor.layers.sampler import Sampler
12
from vllm.model_executor.sampling_metadata import SamplingMetadata
13
from vllm.model_executor.utils import set_random_seed
15
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
16
- SequenceData, SequenceGroupMetadata)
14
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
15
from vllm.utils import Counter, is_pin_memory_available
16
17
@@ -59,9 +57,7 @@ def _do_sample(
57
SequenceGroupMetadata(
58
request_id=f"test_{i}",
59
is_prompt=True,
62
- seq_data={
63
- 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
64
- },
60
+ seq_data={0: SequenceData.from_seqs([1, 2, 3])},
61
sampling_params=sampling_params,
62
block_tables={0: [1]},
63
))
@@ -205,9 +201,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
201
return sampling_params
202
203
def create_sequence_data(num_input=3, num_generated=0):
208
- seq_data = SequenceData(
209
- array(VLLM_TOKEN_ID_ARRAY_TYPE,
210
- random.choices(range(0, VOCAB_SIZE), k=num_input)))
204
+ seq_data = SequenceData.from_seqs(
205
+ random.choices(range(0, VOCAB_SIZE), k=num_input))
206
if num_generated > 0:
207
seq_data.output_token_ids = random.choices(range(0, VOCAB_SIZE),
208
k=num_generated)
@@ -511,9 +506,7 @@ def test_sampler_mixed(seed: int, device: str):
506
SequenceGroupMetadata(
507
request_id=f"test_{i}",
508
is_prompt=True,
514
- seq_data={
515
- 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
516
- },
509
+ seq_data={0: SequenceData.from_seqs([1, 2, 3])},
510
sampling_params=sampling_params,
511
block_tables={0: [1]},
512
))
@@ -613,9 +606,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
606
SequenceGroupMetadata(
607
request_id=f"test_{i}",
608
is_prompt=True,
616
- seq_data={
617
- 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
618
- },
609
+ seq_data={0: SequenceData.from_seqs([1, 2, 3])},
610
sampling_params=SamplingParams(
611
temperature=1,
612
top_k=top_k,
@@ -699,11 +690,7 @@ def test_sampler_repetition_penalty_mixed(device: str):
690
SequenceGroupMetadata(
691
request_id=f"test_{i}",
692
is_prompt=True,
702
- seq_data={
703
- 0:
704
- SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
705
- [1, 2, 3]))
706
- },
693
+ seq_data={0: SequenceData.from_seqs([1, 2, 3])},
694
sampling_params=sampling_params[i],
695
block_tables={0: [1]},
696
))
tests/spec_decode/utils.py
+3
-9
@@ -1,4 +1,3 @@
1
-from array import array
1
from itertools import count
2
from typing import Callable, Dict, List, Optional
3
from typing import Sequence as GenericSequence
@@ -11,8 +10,7 @@ from vllm.engine.arg_utils import EngineArgs
10
from vllm.model_executor.layers.sampler import SamplerOutput
11
from vllm.model_executor.utils import set_random_seed
12
from vllm.sampling_params import SamplingParams
14
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
15
- CompletionSequenceGroupOutput, Logprob,
13
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
14
SequenceData, SequenceGroupMetadata, SequenceOutput)
15
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
16
from vllm.worker.cache_engine import CacheEngine
@@ -138,12 +136,8 @@ def create_seq_group_metadata_from_prompts(
136
request_id=str(i),
137
is_prompt=len(cont_token_ids) == 0,
138
seq_data={
141
- i:
142
- SequenceData(
143
- array(VLLM_TOKEN_ID_ARRAY_TYPE, prompt_token_ids[:]),
144
- _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE,
145
- cont_token_ids[:]),
146
- ),
139
+ i: SequenceData.from_seqs(prompt_token_ids[:],
140
+ cont_token_ids[:]),
141
},
142
sampling_params=SamplingParams(temperature=0.0, ),
143
block_tables={i: block_allocations[i][:]},
tests/test_logits_processor.py
+2
-6
@@ -1,5 +1,4 @@
1
import random
2
-from array import array
2
from typing import Tuple
3
from unittest.mock import patch
4
@@ -9,8 +8,7 @@ import torch
8
from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.model_executor.utils import set_random_seed
12
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
13
- SequenceData, SequenceGroupMetadata)
11
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12
from vllm.utils import is_pin_memory_available
13
14
@@ -71,9 +69,7 @@ def test_logits_processors(seed: int, device: str):
69
SequenceGroupMetadata(
70
request_id=f"test_{i}",
71
is_prompt=True,
74
- seq_data={
75
- 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
76
- },
72
+ seq_data={0: SequenceData.from_seqs([1, 2, 3])},
73
sampling_params=SamplingParams(temperature=0,
74
logits_processors=[pick_ith]),
75
block_tables={0: [1]},
tests/test_sequence.py
+2
-5
@@ -1,10 +1,7 @@
1
-from array import array
2
-
1
import pytest
2
3
from vllm.model_executor.layers.sampler import SamplerOutput
6
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
7
- CompletionSequenceGroupOutput, SequenceData,
4
+from vllm.sequence import (CompletionSequenceGroupOutput, SequenceData,
5
SequenceOutput)
6
7
from .core.utils import create_dummy_prompt
@@ -58,7 +55,7 @@ def test_sampler_output_eq(sample_outputs):
55
56
57
def test_sequence_data_prefill():
61
- seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3, 4]))
58
+ seq_data = SequenceData.from_seqs([1, 2, 3, 4])
59
assert seq_data.get_num_uncomputed_tokens() == 4
60
assert seq_data.get_num_computed_tokens() == 0
61
# advance by 2
tests/worker/test_encoder_decoder_model_runner.py
+7
-15
@@ -1,13 +1,11 @@
1
import itertools
2
-from array import array
2
from typing import List
3
4
import pytest
5
import torch
6
7
from vllm.engine.arg_utils import EngineArgs
9
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
10
- SequenceData, SequenceGroupMetadata)
8
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
9
from vllm.utils import is_cpu, make_tensor_with_pad
10
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
11
from vllm.worker.model_runner import _get_graph_batch_size
@@ -119,12 +117,10 @@ def test_prepare_prompt(batch_size):
117
# make sure all tokens fit into one block
118
seq_len = i % (model_runner.block_size - 1) + 1
119
seq_lens.append(seq_len)
122
- seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
123
- range(seq_len)))
120
+ seq_data = SequenceData.from_seqs(range(seq_len))
121
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
122
encoder_seq_lens.append(encoder_seq_len)
126
- encoder_seq_data = SequenceData(
127
- array(VLLM_TOKEN_ID_ARRAY_TYPE, range(encoder_seq_len)))
123
+ encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
124
seq_group_metadata = SequenceGroupMetadata(
125
request_id=f"test_{i}",
126
is_prompt=True,
@@ -317,11 +313,9 @@ def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
313
for i in range(batch_size):
314
# make sure all tokens fit into one block
315
seq_len = i % (model_runner.block_size - 1) + 1
320
- seq_data = SequenceData(
321
- array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
316
+ seq_data = SequenceData.from_seqs(range(seq_len))
317
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
323
- encoder_seq_data = SequenceData(
324
- array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
318
+ encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
319
320
seq_group_metadata = SequenceGroupMetadata(
321
request_id=f"test_{i}",
@@ -523,11 +517,9 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
517
for i in range(batch_size):
518
# make sure all tokens fit into one block
519
seq_len = i % (model_runner.block_size - 1) + 1
526
- seq_data = SequenceData(
527
- array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
520
+ seq_data = SequenceData.from_seqs(range(seq_len))
521
encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
529
- encoder_seq_data = SequenceData(
530
- array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
522
+ encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
523
seq_group_metadata = SequenceGroupMetadata(
524
request_id=f"test_{i}",
525
is_prompt=False,
tests/worker/test_model_runner.py
+5
-11
@@ -1,4 +1,3 @@
1
-from array import array
1
from typing import List
2
3
import pytest
@@ -8,8 +7,7 @@ from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
7
init_distributed_environment)
8
from vllm.engine.arg_utils import EngineArgs
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
11
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
12
- SequenceData, SequenceGroupMetadata)
10
+from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11
from vllm.utils import get_open_port
12
from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
13
@@ -48,8 +46,7 @@ def test_prepare_prompt(batch_size):
46
# make sure all tokens fit into one block
47
seq_len = i % (model_runner.block_size - 1) + 1
48
seq_lens.append(seq_len)
51
- seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
52
- range(seq_len)))
49
+ seq_data = SequenceData.from_seqs(range(seq_len))
50
seq_group_metadata = SequenceGroupMetadata(
51
request_id=f"test_{i}",
52
is_prompt=True,
@@ -166,8 +163,7 @@ def test_prepare_decode_cuda_graph(batch_size):
163
# make sure all tokens fit into one block
164
context_len = i % (model_runner.block_size - 1) + 1
165
context_lens.append(context_len)
169
- seq_data = SequenceData(
170
- array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len)))
166
+ seq_data = SequenceData.from_seqs(range(context_len))
167
seq_data.update_num_computed_tokens(context_len)
168
# Append one token ID since prefill is finished.
169
seq_data.append_token_id(1, 0)
@@ -326,8 +322,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
322
# make sure all tokens fit into one block
323
seq_len = i % (model_runner.block_size - 1) + 1
324
seq_lens.append(seq_len)
329
- seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
330
- range(seq_len)))
325
+ seq_data = SequenceData.from_seqs(range(seq_len))
326
seq_group_metadata = SequenceGroupMetadata(
327
request_id=f"test_{i}",
328
is_prompt=True,
@@ -343,8 +338,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
338
for i in range(prefill_batch_size, batch_size):
339
# make sure all tokens fit into one block
340
context_len = i % (model_runner.block_size - 1) + 1
346
- prompt_toks = array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len))
347
- seq_data = SequenceData(prompt_toks)
341
+ seq_data = SequenceData.from_seqs(range(context_len))
342
seq_data.append_token_id(1, 0)
343
seq_data.update_num_computed_tokens(context_len)
344
seq_group_metadata = SequenceGroupMetadata(
vllm/inputs/registry.py
+1
-7
@@ -1,5 +1,4 @@
1
import functools
2
-from array import array
2
from collections import UserDict
3
from dataclasses import dataclass
4
from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
@@ -22,10 +21,6 @@ logger = init_logger(__name__)
21
22
C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
23
25
-# NOTE: This has to match with sequence.py's VLLM_TOKEN_ID_ARRAY_TYPE.
26
-# We cannot import it here because of circular dependencies.
27
-VLLM_TOKEN_ID_ARRAY_TYPE = "l"
28
-
24
25
@dataclass(frozen=True)
26
class InputContext:
@@ -130,8 +125,7 @@ class InputRegistry:
125
# Avoid circular import
126
from vllm.sequence import SequenceData
127
133
- dummy_seq_data = SequenceData(
134
- array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len)
128
+ dummy_seq_data = SequenceData.from_counts({0: seq_len})
129
dummy_multi_modal_data = None
130
131
return dummy_seq_data, dummy_multi_modal_data
vllm/sequence.py
+37
-24
@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
7
from dataclasses import dataclass
8
+from functools import cached_property, reduce
9
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
10
from typing import Sequence as GenericSequence
11
from typing import Set, Tuple, Union, cast
@@ -169,6 +170,35 @@ class SequenceData(msgspec.Struct,
170
# It is used to compute mrope_position_ids.
171
_mrope_position_delta: Optional[int] = None
172
173
+ @staticmethod
174
+ def from_counts(counts_by_token: Mapping[int, int]) -> "SequenceData":
175
+ if len(counts_by_token) == 0:
176
+ return SequenceData.from_seqs([])
177
+
178
+ arrs = [
179
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
180
+ for token_id, count in counts_by_token.items()
181
+ ]
182
+
183
+ return SequenceData(reduce(array.__add__, arrs))
184
+
185
+ @staticmethod
186
+ def from_seqs(
187
+ prompt_token_ids: GenericSequence[int],
188
+ output_token_ids: Optional[GenericSequence[int]] = None,
189
+ ) -> "SequenceData":
190
+ prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
191
+ prompt_token_ids)
192
+
193
+ if output_token_ids is None:
194
+ return SequenceData(prompt_token_ids_arr)
195
+
196
+ output_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
197
+ output_token_ids)
198
+
199
+ return SequenceData(prompt_token_ids_arr,
200
+ _output_token_ids=output_token_ids_arr)
201
+
202
def __post_init__(self) -> None:
203
assert self._prompt_token_ids.typecode == "l"
204
assert self._output_token_ids.typecode == "l"
@@ -370,8 +400,6 @@ class Sequence:
400
self.lora_request = lora_request
401
self.prompt_adapter_request = prompt_adapter_request
402
self.from_decoder_prompt = from_decoder_prompt
373
- self._prompt: Optional[str] = None
374
- self._prompt_token_ids: Optional[List[int]] = None
403
404
# For decoder-only models, a Sequence is constructed
405
# from an LLMInputs instance (the `inputs` arg.)
@@ -400,8 +428,7 @@ class Sequence:
428
f"invalid input {inputs}; did you forget the "
429
"encoder input prompt fields?")
430
403
- self.data = SequenceData(
404
- array(VLLM_TOKEN_ID_ARRAY_TYPE, self.prompt_token_ids))
431
+ self.data = SequenceData.from_seqs(self.prompt_token_ids)
432
self.output_logprobs: SampleLogprobs = []
433
self.output_text = ""
434
@@ -422,37 +449,23 @@ class Sequence:
449
def n_blocks(self) -> int:
450
return (self.get_len() + self.block_size - 1) // self.block_size
451
425
- @property
452
+ @cached_property
453
def prompt(self) -> Optional[str]:
427
- if self._prompt is not None:
428
- # Reuse precomputed prompt string
429
- return self._prompt
430
-
431
- # Select decoder or encoder input prompt str,
432
- # as appropriate
454
+ # Select decoder or encoder input prompt str, as appropriate
455
prompt_key: str = ("prompt"
456
if self.from_decoder_prompt else "encoder_prompt")
457
436
- # Cache prompt
437
- self._prompt = cast(Optional[str], self.inputs.get(prompt_key))
438
- return self._prompt
458
+ return cast(Optional[str], self.inputs.get(prompt_key))
459
440
- @property
460
+ @cached_property
461
def prompt_token_ids(self) -> List[int]:
442
- if self._prompt_token_ids is not None:
443
- # Reuse precomputed prompt token ids
444
- return self._prompt_token_ids
445
-
446
- # Select decoder or encoder input prompt
447
- # token ids, as appropriate
462
+ # Select decoder or encoder input prompt token ids, as appropriate
463
prompt_token_ids_key: str = ("prompt_token_ids"
464
if self.from_decoder_prompt else
465
"encoder_prompt_token_ids")
466
467
# Cache computed prompt token ids
453
- self._prompt_token_ids = cast(List[int],
454
- self.inputs.get(prompt_token_ids_key))
455
- return self._prompt_token_ids
468
+ return cast(List[int], self.inputs.get(prompt_token_ids_key))
469
470
@property
471
def multi_modal_data(self) -> "MultiModalDataDict":