@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b1c255630d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Avoid the need to pass `None` values to `Sequence.inputs` (#5099)

Cyrus Leung committed May 30, 2024 at 07:05 UTC b1c255630db60e08c394964b8ed6c0154d31a29f
7 files changed +6 -19
tests/core/test_block_manager.py
-2
@@ -234,7 +234,6 @@ def test_append_slot_cow():
234 inputs={
235 "prompt": "one two three",
236 "prompt_token_ids": [1, 2, 3],
237 - "multi_modal_data": None
237 },
238 block_size=block_size)
239
@@ -525,7 +524,6 @@ def test_sliding_window_multi_seq():
524 inputs={
525 "prompt": "one two three",
526 "prompt_token_ids": [0, 1, 2],
528 - "multi_modal_data": None
527 },
528 block_size=block_size)
529 seq_group = SequenceGroup(request_id="1",
tests/core/utils.py
+1 -6
@@ -25,7 +25,6 @@ def create_dummy_prompt(
25 inputs={
26 "prompt": prompt_str,
27 "prompt_token_ids": prompt_tokens,
28 - "multi_modal_data": None,
28 },
29 block_size=block_size)
30 seq_group = SequenceGroup(request_id=request_id,
@@ -103,11 +102,7 @@ def create_seq_group(
102 for seq_id_offset, output_len in enumerate(seq_output_lens):
103 seq = Sequence(
104 seq_id=seq_id_start + seq_id_offset,
106 - inputs={
107 - "prompt": "",
108 - "prompt_token_ids": prompt_token_ids,
109 - "multi_modal_data": None,
110 - },
105 + inputs={"prompt_token_ids": prompt_token_ids},
106 block_size=16,
107 )
108
tests/engine/output_processor/test_stop_checker.py
+1 -5
@@ -15,11 +15,7 @@ def sequence_with_eos(text: str, eos_token: str,
15 """
16 seq = Sequence(
17 seq_id=0,
18 - inputs={
19 - "prompt": "",
20 - "prompt_token_ids": [],
21 - "multi_modal_data": None,
22 - },
18 + inputs={"prompt_token_ids": []},
19 block_size=16,
20 eos_token_id=eos_token_id,
21 )
tests/test_cache_block_hashing.py
-1
@@ -74,7 +74,6 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
74 inputs={
75 "prompt": prompt,
76 "prompt_token_ids": prompt_token_ids,
77 - "multi_modal_data": None,
77 },
78 block_size=block_size,
79 eos_token_id=tokenizer.tokenizer.eos_token_id,
tests/tokenization/test_detokenize.py
-1
@@ -126,7 +126,6 @@ def create_sequence(prompt_token_ids=None):
126 inputs={
127 "prompt": "<s>",
128 "prompt_token_ids": prompt_token_ids,
129 - "multi_modal_data": None,
129 },
130 block_size=16,
131 )
vllm/inputs.py
+2 -2
@@ -126,5 +126,5 @@ PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
126
127 class LLMInputs(TypedDict):
128 prompt_token_ids: List[int]
129 - prompt: Optional[str]
130 - multi_modal_data: Optional["MultiModalData"]
129 + prompt: NotRequired[Optional[str]]
130 + multi_modal_data: NotRequired[Optional["MultiModalData"]]
vllm/sequence.py
+2 -2
@@ -249,7 +249,7 @@ class Sequence:
249
250 @property
251 def prompt(self) -> Optional[str]:
252 - return self.inputs["prompt"]
252 + return self.inputs.get("prompt")
253
254 @property
255 def prompt_token_ids(self) -> List[int]:
@@ -257,7 +257,7 @@ class Sequence:
257
258 @property
259 def multi_modal_data(self) -> Optional["MultiModalData"]:
260 - return self.inputs["multi_modal_data"]
260 + return self.inputs.get("multi_modal_data")
261
262 @property
263 def lora_int_id(self) -> int: