@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 37e84a403d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Typing] Fix Sequence type GenericAlias only available after Python 3.9. (#4092)

SangBin Cho committed Apr 16, 2024 at 06:47 UTC 37e84a403d6d11b670a42e84153204cd8b76b849
4 files changed +9 -7
vllm/core/block_manager_v1.py
+3 -2
@@ -1,9 +1,10 @@
1 """A block manager that manages token blocks."""
2 from abc import ABC, abstractmethod
3 -from collections.abc import Sequence as GenericSequence
3 from itertools import count, takewhile
4 from os.path import commonprefix
6 -from typing import Dict, List, Optional, Set
5 +from typing import Dict, List, Optional
6 +from typing import Sequence as GenericSequence
7 +from typing import Set
8
9 from vllm.block import BlockTable, PhysicalTokenBlock
10 from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
vllm/core/block_manager_v2.py
+1 -1
@@ -1,6 +1,6 @@
1 """A block manager that manages token blocks."""
2 -from collections.abc import Sequence as GenericSequence
2 from typing import Dict, List, Optional
3 +from typing import Sequence as GenericSequence
4
5 from vllm.core.block.block_table import BlockTable
6 from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
vllm/core/interfaces.py
+1 -1
@@ -1,7 +1,7 @@
1 import enum
2 from abc import ABC, abstractmethod
3 -from collections.abc import Sequence as GenericSequence
3 from typing import Dict, List
4 +from typing import Sequence as GenericSequence
5
6 from vllm.sequence import Sequence, SequenceGroup
7
vllm/utils.py
+4 -3
@@ -6,11 +6,12 @@ import socket
6 import subprocess
7 import uuid
8 import warnings
9 -from collections import OrderedDict, defaultdict
9 +from collections import defaultdict
10 from functools import lru_cache, partial
11 from platform import uname
12 from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
13 - Hashable, List, Optional, Tuple, TypeVar, Union)
13 + Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
14 + Union)
15
16 import psutil
17 import torch
@@ -51,7 +52,7 @@ class Counter:
52 class LRUCache(Generic[T]):
53
54 def __init__(self, capacity: int):
54 - self.cache = OrderedDict[Hashable, T]()
55 + self.cache: OrderedDict[Hashable, T] = OrderedDict()
56 self.capacity = capacity
57
58 def __contains__(self, key: Hashable) -> bool: