A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Typing] Fix Sequence type GenericAlias only available after Python 3.9. (#4092)
SangBin Cho committed
Apr 16, 2024 at 06:47 UTC
37e84a403d6d11b670a42e84153204cd8b76b849
4 files changed
+9
-7
vllm/core/block_manager_v1.py
+3
-2
@@ -1,9 +1,10 @@
1
"""A block manager that manages token blocks."""
2
from abc import ABC, abstractmethod
3
-from collections.abc import Sequence as GenericSequence
3
from itertools import count, takewhile
4
from os.path import commonprefix
6
-from typing import Dict, List, Optional, Set
5
+from typing import Dict, List, Optional
6
+from typing import Sequence as GenericSequence
7
+from typing import Set
8
9
from vllm.block import BlockTable, PhysicalTokenBlock
10
from vllm.core.evictor import EvictionPolicy, Evictor, make_evictor
vllm/core/block_manager_v2.py
+1
-1
@@ -1,6 +1,6 @@
1
"""A block manager that manages token blocks."""
2
-from collections.abc import Sequence as GenericSequence
2
from typing import Dict, List, Optional
3
+from typing import Sequence as GenericSequence
4
5
from vllm.core.block.block_table import BlockTable
6
from vllm.core.block.cpu_gpu_block_allocator import CpuGpuBlockAllocator
vllm/core/interfaces.py
+1
-1
@@ -1,7 +1,7 @@
1
import enum
2
from abc import ABC, abstractmethod
3
-from collections.abc import Sequence as GenericSequence
3
from typing import Dict, List
4
+from typing import Sequence as GenericSequence
5
6
from vllm.sequence import Sequence, SequenceGroup
7
vllm/utils.py
+4
-3
@@ -6,11 +6,12 @@ import socket
6
import subprocess
7
import uuid
8
import warnings
9
-from collections import OrderedDict, defaultdict
9
+from collections import defaultdict
10
from functools import lru_cache, partial
11
from platform import uname
12
from typing import (Any, AsyncIterator, Awaitable, Callable, Dict, Generic,
13
- Hashable, List, Optional, Tuple, TypeVar, Union)
13
+ Hashable, List, Optional, OrderedDict, Tuple, TypeVar,
14
+ Union)
15
16
import psutil
17
import torch
@@ -51,7 +52,7 @@ class Counter:
52
class LRUCache(Generic[T]):
53
54
def __init__(self, capacity: int):
54
- self.cache = OrderedDict[Hashable, T]()
55
+ self.cache: OrderedDict[Hashable, T] = OrderedDict()
56
self.capacity = capacity
57
58
def __contains__(self, key: Hashable) -> bool: