@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 42b4f46b71
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1] Add all_token_ids attribute to Request (#10135)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Nov 7, 2024 at 17:08 UTC 42b4f46b71572e21582fd12c498ec3b0b78ada7b
4 files changed +92 -5
vllm/v1/core/scheduler.py
+1 -1
@@ -246,7 +246,7 @@ class Scheduler:
246 # NOTE(woosuk): Currently, we assume that each request
247 # generates at most one token at each step.
248 token_id = sampled_token_ids[req_index]
249 - request.output_token_ids.append(token_id)
249 + request.append_output_token_ids(token_id)
250 sampled.append((request, 1))
251 # TODO: Update the KV cache manager for prefix caching.
252
vllm/v1/engine/llm_engine.py
+1 -1
@@ -324,7 +324,7 @@ class LLMEngine:
324 )
325 for req, num_tokens in sampled:
326 inputs.req_ids.append(req.request_id)
327 - if len(req.output_token_ids) == num_tokens:
327 + if req.num_output_tokens == num_tokens:
328 # The request is first detokenized.
329 inputs.prompt_token_ids.append(req.prompt_token_ids)
330 else:
vllm/v1/request.py
+26 -3
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
4 from vllm.lora.request import LoRARequest
5 from vllm.sampling_params import SamplingParams
6 from vllm.sequence import RequestMetrics
7 +from vllm.v1.utils import ConstantList
8
9 if TYPE_CHECKING:
10 from vllm.inputs import DecoderOnlyInputs
@@ -40,17 +41,39 @@ class Request:
41 self.prompt = inputs.get("prompt")
42 self.prompt_token_ids = inputs["prompt_token_ids"]
43 self.num_prompt_tokens = len(self.prompt_token_ids)
43 - self.output_token_ids: List[int] = []
44 + self._output_token_ids: List[int] = []
45 + self._all_token_ids: List[int] = self.prompt_token_ids.copy()
46 self.output_text = ""
47 self.num_computed_tokens = 0
48
49 + @property
50 + def output_token_ids(self) -> ConstantList[int]:
51 + # Prevent directly appending to the output_token_ids since
52 + # all_token_ids should also be updated simultaneously.
53 + return ConstantList(self._output_token_ids)
54 +
55 + @property
56 + def all_token_ids(self) -> ConstantList[int]:
57 + # Prevent directly appending to the all_token_ids since
58 + # output_token_ids should also be updated simultaneously
59 + return ConstantList(self._all_token_ids)
60 +
61 + def append_output_token_ids(
62 + self,
63 + token_ids: Union[int, List[int]],
64 + ) -> None:
65 + if isinstance(token_ids, int):
66 + token_ids = [token_ids]
67 + self._output_token_ids.extend(token_ids)
68 + self._all_token_ids.extend(token_ids)
69 +
70 @property
71 def num_tokens(self) -> int:
49 - return self.num_prompt_tokens + len(self.output_token_ids)
72 + return len(self._all_token_ids)
73
74 @property
75 def num_output_tokens(self) -> int:
53 - return len(self.output_token_ids)
76 + return len(self._output_token_ids)
77
78 def is_finished(self) -> bool:
79 return RequestStatus.is_finished(self.status)
vllm/v1/utils.py new
+64
@@ -0,0 +1,64 @@
1 +from typing import Generic, List, TypeVar, overload
2 +
3 +T = TypeVar("T")
4 +
5 +
6 +class ConstantList(Generic[T]):
7 +
8 + def __init__(self, x: List[T]) -> None:
9 + self._x = x
10 +
11 + def append(self, item):
12 + raise Exception("Cannot append to a constant list")
13 +
14 + def extend(self, item):
15 + raise Exception("Cannot extend a constant list")
16 +
17 + def insert(self, item):
18 + raise Exception("Cannot insert into a constant list")
19 +
20 + def pop(self, item):
21 + raise Exception("Cannot pop from a constant list")
22 +
23 + def remove(self, item):
24 + raise Exception("Cannot remove from a constant list")
25 +
26 + def clear(self):
27 + raise Exception("Cannot clear a constant list")
28 +
29 + def index(self, item):
30 + return self._x.index(item)
31 +
32 + @overload
33 + def __getitem__(self, item) -> T:
34 + ...
35 +
36 + @overload
37 + def __getitem__(self, s: slice, /) -> List[T]:
38 + ...
39 +
40 + def __getitem__(self, item):
41 + return self._x[item]
42 +
43 + @overload
44 + def __setitem__(self, item, value):
45 + ...
46 +
47 + @overload
48 + def __setitem__(self, s: slice, value, /):
49 + ...
50 +
51 + def __setitem__(self, item, value):
52 + raise Exception("Cannot set item in a constant list")
53 +
54 + def __delitem__(self, item):
55 + raise Exception("Cannot delete item from a constant list")
56 +
57 + def __iter__(self):
58 + return iter(self._x)
59 +
60 + def __contains__(self, item):
61 + return item in self._x
62 +
63 + def __len__(self):
64 + return len(self._x)