@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b67feb1274
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix]Using the correct type hints (#9885)

Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>

Gregory Shtrasberg committed Nov 4, 2024 at 01:19 UTC b67feb12749ef8c01ef77142c3cd534bb3d87eda
1 file changed +5 -3
vllm/sequence.py
+5 -3
@@ -6,7 +6,8 @@ from array import array
6 from collections import defaultdict
7 from dataclasses import dataclass, field
8 from functools import cached_property, reduce
9 -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
9 +from typing import (TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List,
10 + Mapping, Optional)
11 from typing import Sequence as GenericSequence
12 from typing import Set, Tuple, Union, cast
13
@@ -256,7 +257,8 @@ class SequenceData(msgspec.Struct,
257 return tuple(self._output_token_ids)
258
259 @output_token_ids.setter
259 - def output_token_ids(self, new_output_token_ids: List[int]) -> None:
260 + def output_token_ids(self,
261 + new_output_token_ids: GenericSequence[int]) -> None:
262 self._output_token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
263 new_output_token_ids)
264 self._update_cached_all_tokens()
@@ -1173,7 +1175,7 @@ def get_all_seq_ids_and_request_ids(
1175 sequence ids.
1176 """
1177 seq_ids: List[int] = []
1176 - request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set)
1178 + request_id_seq_ids_mapping: DefaultDict[str, Set[int]] = defaultdict(set)
1179 for sg in seq_group_metadata_list:
1180 for seq_id in sg.seq_data:
1181 seq_ids.append(seq_id)