@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0455c46ed4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Factor out common code in `SequenceData` and `Sequence` (#8675)

Cyrus Leung committed Sep 21, 2024 at 10:30 UTC 0455c46ed434d70f0a6219204e89ee04f1d01336
8 files changed +64 -97
tests/samplers/test_sampler.py
+7 -20
@@ -1,6 +1,5 @@
1 import itertools
2 import random
3 -from array import array
3 from typing import Dict, List, Optional, Tuple
4 from unittest.mock import Mock, patch
5
@@ -12,8 +11,7 @@ import vllm.envs as envs
11 from vllm.model_executor.layers.sampler import Sampler
12 from vllm.model_executor.sampling_metadata import SamplingMetadata
13 from vllm.model_executor.utils import set_random_seed
15 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
16 - SequenceData, SequenceGroupMetadata)
14 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
15 from vllm.utils import Counter, is_pin_memory_available
16
17
@@ -59,9 +57,7 @@ def _do_sample(
57 SequenceGroupMetadata(
58 request_id=f"test_{i}",
59 is_prompt=True,
62 - seq_data={
63 - 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
64 - },
60 + seq_data={0: SequenceData.from_seqs([1, 2, 3])},
61 sampling_params=sampling_params,
62 block_tables={0: [1]},
63 ))
@@ -205,9 +201,8 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
201 return sampling_params
202
203 def create_sequence_data(num_input=3, num_generated=0):
208 - seq_data = SequenceData(
209 - array(VLLM_TOKEN_ID_ARRAY_TYPE,
210 - random.choices(range(0, VOCAB_SIZE), k=num_input)))
204 + seq_data = SequenceData.from_seqs(
205 + random.choices(range(0, VOCAB_SIZE), k=num_input))
206 if num_generated > 0:
207 seq_data.output_token_ids = random.choices(range(0, VOCAB_SIZE),
208 k=num_generated)
@@ -511,9 +506,7 @@ def test_sampler_mixed(seed: int, device: str):
506 SequenceGroupMetadata(
507 request_id=f"test_{i}",
508 is_prompt=True,
514 - seq_data={
515 - 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
516 - },
509 + seq_data={0: SequenceData.from_seqs([1, 2, 3])},
510 sampling_params=sampling_params,
511 block_tables={0: [1]},
512 ))
@@ -613,9 +606,7 @@ def test_sampler_top_k_top_p(seed: int, device: str):
606 SequenceGroupMetadata(
607 request_id=f"test_{i}",
608 is_prompt=True,
616 - seq_data={
617 - 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
618 - },
609 + seq_data={0: SequenceData.from_seqs([1, 2, 3])},
610 sampling_params=SamplingParams(
611 temperature=1,
612 top_k=top_k,
@@ -699,11 +690,7 @@ def test_sampler_repetition_penalty_mixed(device: str):
690 SequenceGroupMetadata(
691 request_id=f"test_{i}",
692 is_prompt=True,
702 - seq_data={
703 - 0:
704 - SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
705 - [1, 2, 3]))
706 - },
693 + seq_data={0: SequenceData.from_seqs([1, 2, 3])},
694 sampling_params=sampling_params[i],
695 block_tables={0: [1]},
696 ))
tests/spec_decode/utils.py
+3 -9
@@ -1,4 +1,3 @@
1 -from array import array
1 from itertools import count
2 from typing import Callable, Dict, List, Optional
3 from typing import Sequence as GenericSequence
@@ -11,8 +10,7 @@ from vllm.engine.arg_utils import EngineArgs
10 from vllm.model_executor.layers.sampler import SamplerOutput
11 from vllm.model_executor.utils import set_random_seed
12 from vllm.sampling_params import SamplingParams
14 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
15 - CompletionSequenceGroupOutput, Logprob,
13 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
14 SequenceData, SequenceGroupMetadata, SequenceOutput)
15 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
16 from vllm.worker.cache_engine import CacheEngine
@@ -138,12 +136,8 @@ def create_seq_group_metadata_from_prompts(
136 request_id=str(i),
137 is_prompt=len(cont_token_ids) == 0,
138 seq_data={
141 - i:
142 - SequenceData(
143 - array(VLLM_TOKEN_ID_ARRAY_TYPE, prompt_token_ids[:]),
144 - _output_token_ids=array(VLLM_TOKEN_ID_ARRAY_TYPE,
145 - cont_token_ids[:]),
146 - ),
139 + i: SequenceData.from_seqs(prompt_token_ids[:],
140 + cont_token_ids[:]),
141 },
142 sampling_params=SamplingParams(temperature=0.0, ),
143 block_tables={i: block_allocations[i][:]},
tests/test_logits_processor.py
+2 -6
@@ -1,5 +1,4 @@
1 import random
2 -from array import array
2 from typing import Tuple
3 from unittest.mock import patch
4
@@ -9,8 +8,7 @@ import torch
8 from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.model_executor.utils import set_random_seed
12 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
13 - SequenceData, SequenceGroupMetadata)
11 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
12 from vllm.utils import is_pin_memory_available
13
14
@@ -71,9 +69,7 @@ def test_logits_processors(seed: int, device: str):
69 SequenceGroupMetadata(
70 request_id=f"test_{i}",
71 is_prompt=True,
74 - seq_data={
75 - 0: SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3]))
76 - },
72 + seq_data={0: SequenceData.from_seqs([1, 2, 3])},
73 sampling_params=SamplingParams(temperature=0,
74 logits_processors=[pick_ith]),
75 block_tables={0: [1]},
tests/test_sequence.py
+2 -5
@@ -1,10 +1,7 @@
1 -from array import array
2 -
1 import pytest
2
3 from vllm.model_executor.layers.sampler import SamplerOutput
6 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
7 - CompletionSequenceGroupOutput, SequenceData,
4 +from vllm.sequence import (CompletionSequenceGroupOutput, SequenceData,
5 SequenceOutput)
6
7 from .core.utils import create_dummy_prompt
@@ -58,7 +55,7 @@ def test_sampler_output_eq(sample_outputs):
55
56
57 def test_sequence_data_prefill():
61 - seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [1, 2, 3, 4]))
58 + seq_data = SequenceData.from_seqs([1, 2, 3, 4])
59 assert seq_data.get_num_uncomputed_tokens() == 4
60 assert seq_data.get_num_computed_tokens() == 0
61 # advance by 2
tests/worker/test_encoder_decoder_model_runner.py
+7 -15
@@ -1,13 +1,11 @@
1 import itertools
2 -from array import array
2 from typing import List
3
4 import pytest
5 import torch
6
7 from vllm.engine.arg_utils import EngineArgs
9 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
10 - SequenceData, SequenceGroupMetadata)
8 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
9 from vllm.utils import is_cpu, make_tensor_with_pad
10 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
11 from vllm.worker.model_runner import _get_graph_batch_size
@@ -119,12 +117,10 @@ def test_prepare_prompt(batch_size):
117 # make sure all tokens fit into one block
118 seq_len = i % (model_runner.block_size - 1) + 1
119 seq_lens.append(seq_len)
122 - seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
123 - range(seq_len)))
120 + seq_data = SequenceData.from_seqs(range(seq_len))
121 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
122 encoder_seq_lens.append(encoder_seq_len)
126 - encoder_seq_data = SequenceData(
127 - array(VLLM_TOKEN_ID_ARRAY_TYPE, range(encoder_seq_len)))
123 + encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
124 seq_group_metadata = SequenceGroupMetadata(
125 request_id=f"test_{i}",
126 is_prompt=True,
@@ -317,11 +313,9 @@ def test_prepare_decode(batch_size, multiple_seqs_per_seq_group):
313 for i in range(batch_size):
314 # make sure all tokens fit into one block
315 seq_len = i % (model_runner.block_size - 1) + 1
320 - seq_data = SequenceData(
321 - array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
316 + seq_data = SequenceData.from_seqs(range(seq_len))
317 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
323 - encoder_seq_data = SequenceData(
324 - array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
318 + encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
319
320 seq_group_metadata = SequenceGroupMetadata(
321 request_id=f"test_{i}",
@@ -523,11 +517,9 @@ def test_prepare_decode_cuda_graph(batch_size, multiple_seqs_per_seq_group):
517 for i in range(batch_size):
518 # make sure all tokens fit into one block
519 seq_len = i % (model_runner.block_size - 1) + 1
526 - seq_data = SequenceData(
527 - array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(seq_len))))
520 + seq_data = SequenceData.from_seqs(range(seq_len))
521 encoder_seq_len = (i + 1) % (model_runner.block_size - 1) + 1
529 - encoder_seq_data = SequenceData(
530 - array(VLLM_TOKEN_ID_ARRAY_TYPE, (range(encoder_seq_len))))
522 + encoder_seq_data = SequenceData.from_seqs(range(encoder_seq_len))
523 seq_group_metadata = SequenceGroupMetadata(
524 request_id=f"test_{i}",
525 is_prompt=False,
tests/worker/test_model_runner.py
+5 -11
@@ -1,4 +1,3 @@
1 -from array import array
1 from typing import List
2
3 import pytest
@@ -8,8 +7,7 @@ from vllm.distributed.parallel_state import (ensure_model_parallel_initialized,
7 init_distributed_environment)
8 from vllm.engine.arg_utils import EngineArgs
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
11 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, SamplingParams,
12 - SequenceData, SequenceGroupMetadata)
10 +from vllm.sequence import SamplingParams, SequenceData, SequenceGroupMetadata
11 from vllm.utils import get_open_port
12 from vllm.worker.model_runner import ModelRunner, _get_graph_batch_size
13
@@ -48,8 +46,7 @@ def test_prepare_prompt(batch_size):
46 # make sure all tokens fit into one block
47 seq_len = i % (model_runner.block_size - 1) + 1
48 seq_lens.append(seq_len)
51 - seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
52 - range(seq_len)))
49 + seq_data = SequenceData.from_seqs(range(seq_len))
50 seq_group_metadata = SequenceGroupMetadata(
51 request_id=f"test_{i}",
52 is_prompt=True,
@@ -166,8 +163,7 @@ def test_prepare_decode_cuda_graph(batch_size):
163 # make sure all tokens fit into one block
164 context_len = i % (model_runner.block_size - 1) + 1
165 context_lens.append(context_len)
169 - seq_data = SequenceData(
170 - array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len)))
166 + seq_data = SequenceData.from_seqs(range(context_len))
167 seq_data.update_num_computed_tokens(context_len)
168 # Append one token ID since prefill is finished.
169 seq_data.append_token_id(1, 0)
@@ -326,8 +322,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
322 # make sure all tokens fit into one block
323 seq_len = i % (model_runner.block_size - 1) + 1
324 seq_lens.append(seq_len)
329 - seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE,
330 - range(seq_len)))
325 + seq_data = SequenceData.from_seqs(range(seq_len))
326 seq_group_metadata = SequenceGroupMetadata(
327 request_id=f"test_{i}",
328 is_prompt=True,
@@ -343,8 +338,7 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
338 for i in range(prefill_batch_size, batch_size):
339 # make sure all tokens fit into one block
340 context_len = i % (model_runner.block_size - 1) + 1
346 - prompt_toks = array(VLLM_TOKEN_ID_ARRAY_TYPE, range(context_len))
347 - seq_data = SequenceData(prompt_toks)
341 + seq_data = SequenceData.from_seqs(range(context_len))
342 seq_data.append_token_id(1, 0)
343 seq_data.update_num_computed_tokens(context_len)
344 seq_group_metadata = SequenceGroupMetadata(
vllm/inputs/registry.py
+1 -7
@@ -1,5 +1,4 @@
1 import functools
2 -from array import array
2 from collections import UserDict
3 from dataclasses import dataclass
4 from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
@@ -22,10 +21,6 @@ logger = init_logger(__name__)
21
22 C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
23
25 -# NOTE: This has to match with sequence.py's VLLM_TOKEN_ID_ARRAY_TYPE.
26 -# We cannot import it here because of circular dependencies.
27 -VLLM_TOKEN_ID_ARRAY_TYPE = "l"
28 -
24
25 @dataclass(frozen=True)
26 class InputContext:
@@ -130,8 +125,7 @@ class InputRegistry:
125 # Avoid circular import
126 from vllm.sequence import SequenceData
127
133 - dummy_seq_data = SequenceData(
134 - array(VLLM_TOKEN_ID_ARRAY_TYPE, [0]) * seq_len)
128 + dummy_seq_data = SequenceData.from_counts({0: seq_len})
129 dummy_multi_modal_data = None
130
131 return dummy_seq_data, dummy_multi_modal_data
vllm/sequence.py
+37 -24
@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
7 from dataclasses import dataclass
8 +from functools import cached_property, reduce
9 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
10 from typing import Sequence as GenericSequence
11 from typing import Set, Tuple, Union, cast
@@ -169,6 +170,35 @@ class SequenceData(msgspec.Struct,
170 # It is used to compute mrope_position_ids.
171 _mrope_position_delta: Optional[int] = None
172
173 + @staticmethod
174 + def from_counts(counts_by_token: Mapping[int, int]) -> "SequenceData":
175 + if len(counts_by_token) == 0:
176 + return SequenceData.from_seqs([])
177 +
178 + arrs = [
179 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
180 + for token_id, count in counts_by_token.items()
181 + ]
182 +
183 + return SequenceData(reduce(array.__add__, arrs))
184 +
185 + @staticmethod
186 + def from_seqs(
187 + prompt_token_ids: GenericSequence[int],
188 + output_token_ids: Optional[GenericSequence[int]] = None,
189 + ) -> "SequenceData":
190 + prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
191 + prompt_token_ids)
192 +
193 + if output_token_ids is None:
194 + return SequenceData(prompt_token_ids_arr)
195 +
196 + output_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
197 + output_token_ids)
198 +
199 + return SequenceData(prompt_token_ids_arr,
200 + _output_token_ids=output_token_ids_arr)
201 +
202 def __post_init__(self) -> None:
203 assert self._prompt_token_ids.typecode == "l"
204 assert self._output_token_ids.typecode == "l"
@@ -370,8 +400,6 @@ class Sequence:
400 self.lora_request = lora_request
401 self.prompt_adapter_request = prompt_adapter_request
402 self.from_decoder_prompt = from_decoder_prompt
373 - self._prompt: Optional[str] = None
374 - self._prompt_token_ids: Optional[List[int]] = None
403
404 # For decoder-only models, a Sequence is constructed
405 # from an LLMInputs instance (the `inputs` arg.)
@@ -400,8 +428,7 @@ class Sequence:
428 f"invalid input {inputs}; did you forget the "
429 "encoder input prompt fields?")
430
403 - self.data = SequenceData(
404 - array(VLLM_TOKEN_ID_ARRAY_TYPE, self.prompt_token_ids))
431 + self.data = SequenceData.from_seqs(self.prompt_token_ids)
432 self.output_logprobs: SampleLogprobs = []
433 self.output_text = ""
434
@@ -422,37 +449,23 @@ class Sequence:
449 def n_blocks(self) -> int:
450 return (self.get_len() + self.block_size - 1) // self.block_size
451
425 - @property
452 + @cached_property
453 def prompt(self) -> Optional[str]:
427 - if self._prompt is not None:
428 - # Reuse precomputed prompt string
429 - return self._prompt
430 -
431 - # Select decoder or encoder input prompt str,
432 - # as appropriate
454 + # Select decoder or encoder input prompt str, as appropriate
455 prompt_key: str = ("prompt"
456 if self.from_decoder_prompt else "encoder_prompt")
457
436 - # Cache prompt
437 - self._prompt = cast(Optional[str], self.inputs.get(prompt_key))
438 - return self._prompt
458 + return cast(Optional[str], self.inputs.get(prompt_key))
459
440 - @property
460 + @cached_property
461 def prompt_token_ids(self) -> List[int]:
442 - if self._prompt_token_ids is not None:
443 - # Reuse precomputed prompt token ids
444 - return self._prompt_token_ids
445 -
446 - # Select decoder or encoder input prompt
447 - # token ids, as appropriate
462 + # Select decoder or encoder input prompt token ids, as appropriate
463 prompt_token_ids_key: str = ("prompt_token_ids"
464 if self.from_decoder_prompt else
465 "encoder_prompt_token_ids")
466
467 # Cache computed prompt token ids
453 - self._prompt_token_ids = cast(List[int],
454 - self.inputs.get(prompt_token_ids_key))
455 - return self._prompt_token_ids
468 + return cast(List[int], self.inputs.get(prompt_token_ids_key))
469
470 @property
471 def multi_modal_data(self) -> "MultiModalDataDict":