A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1] Add all_token_ids attribute to Request (#10135)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 7, 2024 at 17:08 UTC
42b4f46b71572e21582fd12c498ec3b0b78ada7b
4 files changed
+92
-5
vllm/v1/core/scheduler.py
+1
-1
@@ -246,7 +246,7 @@ class Scheduler:
246
# NOTE(woosuk): Currently, we assume that each request
247
# generates at most one token at each step.
248
token_id = sampled_token_ids[req_index]
249
- request.output_token_ids.append(token_id)
249
+ request.append_output_token_ids(token_id)
250
sampled.append((request, 1))
251
# TODO: Update the KV cache manager for prefix caching.
252
vllm/v1/engine/llm_engine.py
+1
-1
@@ -324,7 +324,7 @@ class LLMEngine:
324
)
325
for req, num_tokens in sampled:
326
inputs.req_ids.append(req.request_id)
327
- if len(req.output_token_ids) == num_tokens:
327
+ if req.num_output_tokens == num_tokens:
328
# The request is first detokenized.
329
inputs.prompt_token_ids.append(req.prompt_token_ids)
330
else:
vllm/v1/request.py
+26
-3
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
4
from vllm.lora.request import LoRARequest
5
from vllm.sampling_params import SamplingParams
6
from vllm.sequence import RequestMetrics
7
+from vllm.v1.utils import ConstantList
8
9
if TYPE_CHECKING:
10
from vllm.inputs import DecoderOnlyInputs
@@ -40,17 +41,39 @@ class Request:
41
self.prompt = inputs.get("prompt")
42
self.prompt_token_ids = inputs["prompt_token_ids"]
43
self.num_prompt_tokens = len(self.prompt_token_ids)
43
- self.output_token_ids: List[int] = []
44
+ self._output_token_ids: List[int] = []
45
+ self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46
self.output_text = ""
47
self.num_computed_tokens = 0
48
49
+ @property
50
+ def output_token_ids(self) -> ConstantList[int]:
51
+ # Prevent directly appending to the output_token_ids since
52
+ # all_token_ids should also be updated simultaneously.
53
+ return ConstantList(self._output_token_ids)
54
+
55
+ @property
56
+ def all_token_ids(self) -> ConstantList[int]:
57
+ # Prevent directly appending to the all_token_ids since
58
+ # output_token_ids should also be updated simultaneously
59
+ return ConstantList(self._all_token_ids)
60
+
61
+ def append_output_token_ids(
62
+ self,
63
+ token_ids: Union[int, List[int]],
64
+ ) -> None:
65
+ if isinstance(token_ids, int):
66
+ token_ids = [token_ids]
67
+ self._output_token_ids.extend(token_ids)
68
+ self._all_token_ids.extend(token_ids)
69
+
70
@property
71
def num_tokens(self) -> int:
49
- return self.num_prompt_tokens + len(self.output_token_ids)
72
+ return len(self._all_token_ids)
73
74
@property
75
def num_output_tokens(self) -> int:
53
- return len(self.output_token_ids)
76
+ return len(self._output_token_ids)
77
78
def is_finished(self) -> bool:
79
return RequestStatus.is_finished(self.status)
vllm/v1/utils.py
new
+64
@@ -0,0 +1,64 @@
1
+from typing import Generic, List, TypeVar, overload
2
+
3
+T = TypeVar("T")
4
+
5
+
6
+class ConstantList(Generic[T]):
7
+
8
+ def __init__(self, x: List[T]) -> None:
9
+ self._x = x
10
+
11
+ def append(self, item):
12
+ raise Exception("Cannot append to a constant list")
13
+
14
+ def extend(self, item):
15
+ raise Exception("Cannot extend a constant list")
16
+
17
+ def insert(self, item):
18
+ raise Exception("Cannot insert into a constant list")
19
+
20
+ def pop(self, item):
21
+ raise Exception("Cannot pop from a constant list")
22
+
23
+ def remove(self, item):
24
+ raise Exception("Cannot remove from a constant list")
25
+
26
+ def clear(self):
27
+ raise Exception("Cannot clear a constant list")
28
+
29
+ def index(self, item):
30
+ return self._x.index(item)
31
+
32
+ @overload
33
+ def __getitem__(self, item) -> T:
34
+ ...
35
+
36
+ @overload
37
+ def __getitem__(self, s: slice, /) -> List[T]:
38
+ ...
39
+
40
+ def __getitem__(self, item):
41
+ return self._x[item]
42
+
43
+ @overload
44
+ def __setitem__(self, item, value):
45
+ ...
46
+
47
+ @overload
48
+ def __setitem__(self, s: slice, value, /):
49
+ ...
50
+
51
+ def __setitem__(self, item, value):
52
+ raise Exception("Cannot set item in a constant list")
53
+
54
+ def __delitem__(self, item):
55
+ raise Exception("Cannot delete item from a constant list")
56
+
57
+ def __iter__(self):
58
+ return iter(self._x)
59
+
60
+ def __contains__(self, item):
61
+ return item in self._x
62
+
63
+ def __len__(self):
64
+ return len(self._x)