A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Avoid the need to pass `None` values to `Sequence.inputs` (#5099)
Cyrus Leung committed
May 30, 2024 at 07:05 UTC
b1c255630db60e08c394964b8ed6c0154d31a29f
7 files changed
+6
-19
tests/core/test_block_manager.py
-2
@@ -234,7 +234,6 @@ def test_append_slot_cow():
234
inputs={
235
"prompt": "one two three",
236
"prompt_token_ids": [1, 2, 3],
237
- "multi_modal_data": None
237
},
238
block_size=block_size)
239
@@ -525,7 +524,6 @@ def test_sliding_window_multi_seq():
524
inputs={
525
"prompt": "one two three",
526
"prompt_token_ids": [0, 1, 2],
528
- "multi_modal_data": None
527
},
528
block_size=block_size)
529
seq_group = SequenceGroup(request_id="1",
tests/core/utils.py
+1
-6
@@ -25,7 +25,6 @@ def create_dummy_prompt(
25
inputs={
26
"prompt": prompt_str,
27
"prompt_token_ids": prompt_tokens,
28
- "multi_modal_data": None,
28
},
29
block_size=block_size)
30
seq_group = SequenceGroup(request_id=request_id,
@@ -103,11 +102,7 @@ def create_seq_group(
102
for seq_id_offset, output_len in enumerate(seq_output_lens):
103
seq = Sequence(
104
seq_id=seq_id_start + seq_id_offset,
106
- inputs={
107
- "prompt": "",
108
- "prompt_token_ids": prompt_token_ids,
109
- "multi_modal_data": None,
110
- },
105
+ inputs={"prompt_token_ids": prompt_token_ids},
106
block_size=16,
107
)
108
tests/engine/output_processor/test_stop_checker.py
+1
-5
@@ -15,11 +15,7 @@ def sequence_with_eos(text: str, eos_token: str,
15
"""
16
seq = Sequence(
17
seq_id=0,
18
- inputs={
19
- "prompt": "",
20
- "prompt_token_ids": [],
21
- "multi_modal_data": None,
22
- },
18
+ inputs={"prompt_token_ids": []},
19
block_size=16,
20
eos_token_id=eos_token_id,
21
)
tests/test_cache_block_hashing.py
-1
@@ -74,7 +74,6 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,
74
inputs={
75
"prompt": prompt,
76
"prompt_token_ids": prompt_token_ids,
77
- "multi_modal_data": None,
77
},
78
block_size=block_size,
79
eos_token_id=tokenizer.tokenizer.eos_token_id,
tests/tokenization/test_detokenize.py
-1
@@ -126,7 +126,6 @@ def create_sequence(prompt_token_ids=None):
126
inputs={
127
"prompt": "<s>",
128
"prompt_token_ids": prompt_token_ids,
129
- "multi_modal_data": None,
129
},
130
block_size=16,
131
)
vllm/inputs.py
+2
-2
@@ -126,5 +126,5 @@ PromptInputs = Union[str, TextPrompt, TokensPrompt, TextTokensPrompt]
126
127
class LLMInputs(TypedDict):
128
prompt_token_ids: List[int]
129
- prompt: Optional[str]
130
- multi_modal_data: Optional["MultiModalData"]
129
+ prompt: NotRequired[Optional[str]]
130
+ multi_modal_data: NotRequired[Optional["MultiModalData"]]
vllm/sequence.py
+2
-2
@@ -249,7 +249,7 @@ class Sequence:
249
250
@property
251
def prompt(self) -> Optional[str]:
252
- return self.inputs["prompt"]
252
+ return self.inputs.get("prompt")
253
254
@property
255
def prompt_token_ids(self) -> List[int]:
@@ -257,7 +257,7 @@ class Sequence:
257
258
@property
259
def multi_modal_data(self) -> Optional["MultiModalData"]:
260
- return self.inputs["multi_modal_data"]
260
+ return self.inputs.get("multi_modal_data")
261
262
@property
263
def lora_int_id(self) -> int: