@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 22de45235c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Push logprob generation to LLMEngine (#3065)

Co-authored-by: Avnish Narayan <avnish@anyscale.com>

Antoni Baum committed Mar 4, 2024 at 11:54 UTC 22de45235c6dd14e901e089971635ec655d5fbe0
13 files changed +555 -335
tests/entrypoints/test_openai_server.py
+58 -3
@@ -213,14 +213,14 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
213 messages=messages,
214 max_tokens=10,
215 logprobs=True,
216 - top_logprobs=10)
216 + top_logprobs=5)
217 assert chat_completion.id is not None
218 assert chat_completion.choices is not None and len(
219 chat_completion.choices) == 1
220 assert chat_completion.choices[0].message is not None
221 assert chat_completion.choices[0].logprobs is not None
222 assert chat_completion.choices[0].logprobs.top_logprobs is not None
223 - assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 10
223 + assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
224 message = chat_completion.choices[0].message
225 assert message.content is not None and len(message.content) >= 10
226 assert message.role == "assistant"
@@ -229,7 +229,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
229 # test multi-turn dialogue
230 messages.append({"role": "user", "content": "express your result in json"})
231 chat_completion = await client.chat.completions.create(
232 - model=MODEL_NAME,
232 + model=model_name,
233 messages=messages,
234 max_tokens=10,
235 )
@@ -237,6 +237,61 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
237 assert message.content is not None and len(message.content) >= 0
238
239
240 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
241 +async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
242 + model_name: str):
243 + messages = [{
244 + "role": "system",
245 + "content": "you are a helpful assistant"
246 + }, {
247 + "role": "user",
248 + "content": "what is 1+1?"
249 + }]
250 +
251 + # Default max_logprobs is 5, so this should raise an error
252 + with pytest.raises((openai.BadRequestError, openai.APIError)):
253 + stream = await client.chat.completions.create(model=model_name,
254 + messages=messages,
255 + max_tokens=10,
256 + logprobs=True,
257 + top_logprobs=10,
258 + stream=True)
259 + async for chunk in stream:
260 + ...
261 +
262 + with pytest.raises(openai.BadRequestError):
263 + await client.chat.completions.create(model=model_name,
264 + messages=messages,
265 + max_tokens=10,
266 + logprobs=True,
267 + top_logprobs=10,
268 + stream=False)
269 +
270 + with pytest.raises((openai.BadRequestError, openai.APIError)):
271 + stream = await client.completions.create(model=model_name,
272 + prompt="Test",
273 + max_tokens=10,
274 + logprobs=10,
275 + stream=True)
276 + async for chunk in stream:
277 + ...
278 +
279 + with pytest.raises(openai.BadRequestError):
280 + await client.completions.create(model=model_name,
281 + prompt="Test",
282 + max_tokens=10,
283 + logprobs=10,
284 + stream=False)
285 +
286 + # the server should still work afterwards
287 + chat_completion = await client.chat.completions.create(model=model_name,
288 + messages=messages,
289 + max_tokens=10,
290 + stream=False)
291 + message = chat_completion.choices[0].message
292 + assert message.content is not None and len(message.content) >= 0
293 +
294 +
295 @pytest.mark.parametrize(
296 # just test 1 lora hereafter
297 "model_name",
tests/samplers/test_logprobs.py
+36 -6
@@ -1,5 +1,6 @@
1 import pytest
2 import torch
3 +from tests.conftest import VllmRunner
4
5 from vllm import SamplingParams
6
@@ -16,6 +17,7 @@ def test_get_prompt_logprobs(
17 example_prompts,
18 ):
19 max_tokens = 5
20 + num_top_logprobs = 6
21 hf_model = hf_runner(model, dtype=dtype)
22 hf_logprobs = hf_model.generate_greedy_logprobs(
23 example_prompts,
@@ -23,19 +25,32 @@ def test_get_prompt_logprobs(
25 )
26 del hf_model
27
26 - vllm_model = vllm_runner(model, dtype=dtype)
28 + vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
29 vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
28 - logprobs=5,
30 + logprobs=num_top_logprobs,
31 prompt_logprobs=5,
32 temperature=0.0)
33 vllm_results = vllm_model.model.generate(
34 example_prompts, sampling_params=vllm_sampling_params)
33 - del vllm_model
35
36 # Test whether logprobs are included in the results.
37 for result in vllm_results:
38 assert result.prompt_logprobs is not None
39 assert result.outputs[0].logprobs is not None
40 + assert len(result.outputs[0].logprobs) == max_tokens
41 + for logprobs in result.outputs[0].logprobs:
42 + assert len(logprobs) == num_top_logprobs
43 + output_text = result.outputs[0].text
44 + output_string_from_most_likely_tokens = []
45 + for top_logprobs in result.outputs[0].logprobs:
46 + top_logprob = next(iter(top_logprobs.values()))
47 + output_string_from_most_likely_tokens.append(
48 + top_logprob.decoded_token)
49 + output_string_from_most_likely_tokens = "".join(
50 + output_string_from_most_likely_tokens)
51 + assert output_text == output_string_from_most_likely_tokens, (
52 + "The output text from the top logprob for each token position "
53 + "should be the same as the output text in the result.")
54
55 # Test whether prompt logprobs are consistent with HF
56 for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
@@ -43,14 +58,29 @@ def test_get_prompt_logprobs(
58 vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
59 for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
60 for token_id, logprob in vllm_prompt_logprob_dict.items():
46 - torch.testing.assert_close(logprob,
61 + torch.testing.assert_close(logprob.logprob,
62 hf_logprob[0][i][token_id].item(),
63 atol=1e-2,
64 rtol=1e-2)
65 vllm_sample_logprobs = vllm_result.outputs[0].logprobs
51 - for i, vllm_sample_logprob_dict in enumerate(vllm_sample_logprobs):
52 - for token_id, logprob in vllm_sample_logprob_dict.items():
66 + for i, top_logprobs in enumerate(vllm_sample_logprobs):
67 + for token_id, sample_logprob in top_logprobs.items():
68 + logprob = sample_logprob.logprob
69 torch.testing.assert_close(logprob,
70 hf_logprob[i][-1][token_id].item(),
71 atol=1e-2,
72 rtol=1e-2)
73 + assert isinstance(sample_logprob.decoded_token, str), \
74 + ("The token should be decoded by the time it is returned "
75 + " to the user.")
76 +
77 +
78 +def test_max_logprobs():
79 + runner = VllmRunner("facebook/opt-125m", max_logprobs=1)
80 + vllm_sampling_params = SamplingParams(logprobs=1)
81 + # should pass
82 + runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
83 +
84 + bad_sampling_params = SamplingParams(logprobs=2)
85 + with pytest.raises(ValueError):
86 + runner.generate(["Hello world"], sampling_params=bad_sampling_params)
tests/worker/spec_decode/utils.py
+7 -5
@@ -4,7 +4,7 @@ from typing import List, Optional, Dict
4 from vllm.worker.worker import Worker
5 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6 from vllm.engine.arg_utils import EngineArgs
7 -from vllm.sequence import SequenceGroupMetadata, SequenceData
7 +from vllm.sequence import Logprob, SequenceGroupMetadata, SequenceData
8 from vllm.sampling_params import SamplingParams
9 from vllm.worker.cache_engine import CacheEngine
10 from vllm.model_executor.utils import set_random_seed
@@ -166,13 +166,15 @@ def create_seq_group_metadata_from_prompts(
166
167
168 def assert_logprobs_dict_allclose(
169 - actual_logprobs: List[Dict[int, float]],
170 - expected_logprobs: List[Dict[int, float]]) -> None:
169 + actual_logprobs: List[Dict[int, Logprob]],
170 + expected_logprobs: List[Dict[int, Logprob]]) -> None:
171 for single_step_actual_logprobs, single_step_expected_logprobs in zip(
172 actual_logprobs, expected_logprobs):
173 assert set(single_step_actual_logprobs.keys()) == set(
174 single_step_expected_logprobs.keys())
175 for token_id in single_step_actual_logprobs:
176 - actual = torch.tensor(single_step_actual_logprobs[token_id])
177 - expected = torch.tensor(single_step_expected_logprobs[token_id])
176 + actual = torch.tensor(
177 + single_step_actual_logprobs[token_id].logprob)
178 + expected = torch.tensor(
179 + single_step_expected_logprobs[token_id].logprob)
180 assert torch.allclose(actual, expected)
vllm/config.py
+2
@@ -79,6 +79,7 @@ class ModelConfig:
79 quantization: Optional[str] = None,
80 enforce_eager: bool = False,
81 max_context_len_to_capture: Optional[int] = None,
82 + max_logprobs: int = 5,
83 ) -> None:
84 self.model = model
85 self.tokenizer = tokenizer
@@ -93,6 +94,7 @@ class ModelConfig:
94 self.quantization = quantization
95 self.enforce_eager = enforce_eager
96 self.max_context_len_to_capture = max_context_len_to_capture
97 + self.max_logprobs = max_logprobs
98
99 if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
100 # download model from ModelScope hub,
vllm/engine/arg_utils.py
+9 -1
@@ -31,6 +31,7 @@ class EngineArgs:
31 max_num_batched_tokens: Optional[int] = None
32 max_num_seqs: int = 256
33 max_paddings: int = 256
34 + max_logprobs: int = 5 # OpenAI default value
35 disable_log_stats: bool = False
36 revision: Optional[str] = None
37 code_revision: Optional[str] = None
@@ -212,6 +213,12 @@ class EngineArgs:
213 type=int,
214 default=EngineArgs.max_paddings,
215 help='maximum number of paddings in a batch')
216 + parser.add_argument(
217 + '--max-logprobs',
218 + type=int,
219 + default=EngineArgs.max_logprobs,
220 + help=('max number of log probs to return logprobs is specified in'
221 + ' SamplingParams'))
222 parser.add_argument('--disable-log-stats',
223 action='store_true',
224 help='disable logging statistics')
@@ -300,7 +307,8 @@ class EngineArgs:
307 self.trust_remote_code, self.download_dir, self.load_format,
308 self.dtype, self.seed, self.revision, self.code_revision,
309 self.tokenizer_revision, self.max_model_len, self.quantization,
303 - self.enforce_eager, self.max_context_len_to_capture)
310 + self.enforce_eager, self.max_context_len_to_capture,
311 + self.max_logprobs)
312 cache_config = CacheConfig(self.block_size,
313 self.gpu_memory_utilization,
314 self.swap_space, self.kv_cache_dtype,
vllm/engine/async_llm_engine.py
+24 -5
@@ -47,7 +47,7 @@ class AsyncStream:
47 self._queue = asyncio.Queue()
48 self._finished = False
49
50 - def put(self, item: RequestOutput) -> None:
50 + def put(self, item: Union[RequestOutput, Exception]) -> None:
51 if self._finished:
52 return
53 self._queue.put_nowait(item)
@@ -110,6 +110,17 @@ class RequestTracker:
110 logger.info(f"Finished request {request_id}.")
111 self.abort_request(request_id)
112
113 + def process_exception(self,
114 + request_id: str,
115 + exception: Exception,
116 + *,
117 + verbose: bool = False) -> None:
118 + """Propagate an exception from the engine."""
119 + self._request_streams[request_id].put(exception)
120 + if verbose:
121 + logger.info(f"Finished request {request_id}.")
122 + self.abort_request(request_id)
123 +
124 def add_request(self, request_id: str,
125 **engine_add_request_kwargs) -> AsyncStream:
126 """Add a request to be sent to the engine on the next background
@@ -377,10 +388,18 @@ class AsyncLLMEngine:
388 for new_request in new_requests:
389 # Add the request into the vLLM engine's waiting queue.
390 # TODO: Maybe add add_request_batch to reduce Ray overhead
380 - if self.engine_use_ray:
381 - await self.engine.add_request.remote(**new_request)
382 - else:
383 - await self.engine.add_request_async(**new_request)
391 + try:
392 + if self.engine_use_ray:
393 + await self.engine.add_request.remote(**new_request)
394 + else:
395 + await self.engine.add_request_async(**new_request)
396 + except ValueError as e:
397 + # TODO: use a vLLM specific error for failed validation
398 + self._request_tracker.process_exception(
399 + new_request["request_id"],
400 + e,
401 + verbose=self.log_requests,
402 + )
403
404 if finished_requests:
405 await self._engine_abort(finished_requests)
vllm/engine/llm_engine.py
+40 -2
@@ -18,7 +18,7 @@ from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
18 from vllm.logger import init_logger
19 from vllm.outputs import RequestOutput
20 from vllm.sampling_params import SamplingParams
21 -from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
21 +from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
22 SequenceGroupOutput, SequenceOutput, SequenceStatus)
23 from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
24 TokenizerGroup)
@@ -473,6 +473,13 @@ class LLMEngine:
473 if lora_request is not None and not self.lora_config:
474 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
475 "not enabled!")
476 + max_logprobs = self.get_model_config().max_logprobs
477 + if (sampling_params.logprobs
478 + and sampling_params.logprobs > max_logprobs) or (
479 + sampling_params.prompt_logprobs
480 + and sampling_params.prompt_logprobs > max_logprobs):
481 + raise ValueError(f"Cannot request more than "
482 + f"{max_logprobs} logprobs.")
483 if arrival_time is None:
484 arrival_time = time.monotonic()
485 prompt_token_ids = self.encode_request(
@@ -583,6 +590,13 @@ class LLMEngine:
590 # Process prompt logprobs
591 prompt_logprobs = outputs.prompt_logprobs
592 if prompt_logprobs is not None:
593 + # We can pick any sequence for the prompt.
594 + seq = next(iter(seq_group.seqs_dict.values()))
595 + all_token_ids = seq.get_token_ids()
596 + for i, prompt_logprobs_for_token in enumerate(prompt_logprobs):
597 + self._decode_logprobs(seq, seq_group.sampling_params,
598 + prompt_logprobs_for_token,
599 + all_token_ids[:i])
600 seq_group.prompt_logprobs = prompt_logprobs
601
602 # Process samples
@@ -930,12 +944,36 @@ class LLMEngine:
944 time_e2e_requests=time_e2e_requests,
945 )
946
947 + def _decode_logprobs(self, seq: Sequence, prms: SamplingParams,
948 + logprobs: Dict[int, Logprob],
949 + all_input_ids: List[int]) -> None:
950 + if not logprobs:
951 + return
952 + for token_id, sample_logprob in logprobs.items():
953 + if (sample_logprob.decoded_token is None and token_id != -1):
954 + all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
955 + _, new_text, prefix_offset, read_offset = detokenize_incrementally(
956 + self.get_tokenizer_for_seq(seq),
957 + all_input_ids=all_input_ids_with_logprob,
958 + prev_tokens=seq.tokens,
959 + prefix_offset=seq.prefix_offset,
960 + read_offset=seq.read_offset,
961 + skip_special_tokens=prms.skip_special_tokens,
962 + spaces_between_special_tokens=prms.
963 + spaces_between_special_tokens,
964 + )
965 + sample_logprob.decoded_token = new_text
966 +
967 def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
968 """Decodes the new token for a sequence."""
969 + all_input_ids = seq.get_token_ids()
970 + self._decode_logprobs(seq, prms, seq.output_logprobs[-1],
971 + all_input_ids)
972 +
973 (new_tokens, new_output_text, prefix_offset,
974 read_offset) = detokenize_incrementally(
975 self.get_tokenizer_for_seq(seq),
938 - all_input_ids=seq.get_token_ids(),
976 + all_input_ids=all_input_ids,
977 prev_tokens=seq.tokens,
978 prefix_offset=seq.prefix_offset,
979 read_offset=seq.read_offset,
vllm/entrypoints/openai/serving_chat.py
+128 -108
@@ -82,8 +82,12 @@ class OpenAIServingChat(OpenAIServing):
82 return self.chat_completion_stream_generator(
83 request, result_generator, request_id)
84 else:
85 - return await self.chat_completion_full_generator(
86 - request, raw_request, result_generator, request_id)
85 + try:
86 + return await self.chat_completion_full_generator(
87 + request, raw_request, result_generator, request_id)
88 + except ValueError as e:
89 + # TODO: Use a vllm-specific Validation Error
90 + return self.create_error_response(str(e))
91
92 def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
93 if request.add_generation_prompt:
@@ -99,117 +103,133 @@ class OpenAIServingChat(OpenAIServing):
103 model_name = request.model
104 created_time = int(time.monotonic())
105 chunk_object_type = "chat.completion.chunk"
102 -
103 - # Send first response for each request.n (index) with the role
104 - role = self.get_chat_request_role(request)
105 - for i in range(request.n):
106 - choice_data = ChatCompletionResponseStreamChoice(
107 - index=i,
108 - delta=DeltaMessage(role=role),
109 - logprobs=None,
110 - finish_reason=None)
111 - chunk = ChatCompletionStreamResponse(id=request_id,
112 - object=chunk_object_type,
113 - created=created_time,
114 - choices=[choice_data],
115 - model=model_name)
116 - data = chunk.model_dump_json(exclude_unset=True)
117 - yield f"data: {data}\n\n"
118 -
119 - # Send response to echo the input portion of the last message
120 - if request.echo:
121 - last_msg_content = ""
122 - if request.messages and isinstance(
123 - request.messages, list) and request.messages[-1].get(
124 - "content") and request.messages[-1].get(
125 - "role") == role:
126 - last_msg_content = request.messages[-1]["content"]
127 -
128 - if last_msg_content:
129 - for i in range(request.n):
130 - choice_data = ChatCompletionResponseStreamChoice(
131 - index=i,
132 - delta=DeltaMessage(content=last_msg_content),
133 - finish_reason=None)
134 - chunk = ChatCompletionStreamResponse(
135 - id=request_id,
136 - object=chunk_object_type,
137 - created=created_time,
138 - choices=[choice_data],
139 - logprobs=None,
140 - model=model_name)
141 - data = chunk.model_dump_json(exclude_unset=True)
142 - yield f"data: {data}\n\n"
106 + first_iteration = True
107
108 # Send response for each token for each request.n (index)
109 previous_texts = [""] * request.n
110 previous_num_tokens = [0] * request.n
111 finish_reason_sent = [False] * request.n
148 - async for res in result_generator:
149 - res: RequestOutput
150 - for output in res.outputs:
151 - i = output.index
152 -
153 - if finish_reason_sent[i]:
154 - continue
155 -
156 - delta_token_ids = output.token_ids[previous_num_tokens[i]:]
157 - top_logprobs = output.logprobs[
158 - previous_num_tokens[i]:] if output.logprobs else None
159 -
160 - if request.logprobs:
161 - logprobs = self._create_logprobs(
162 - token_ids=delta_token_ids,
163 - top_logprobs=top_logprobs,
164 - num_output_top_logprobs=request.logprobs,
165 - initial_text_offset=len(previous_texts[i]),
166 - )
167 - else:
168 - logprobs = None
169 -
170 - delta_text = output.text[len(previous_texts[i]):]
171 - previous_texts[i] = output.text
172 - previous_num_tokens[i] = len(output.token_ids)
173 - if output.finish_reason is None:
174 - # Send token-by-token response for each request.n
175 - choice_data = ChatCompletionResponseStreamChoice(
176 - index=i,
177 - delta=DeltaMessage(content=delta_text),
178 - logprobs=logprobs,
179 - finish_reason=None)
180 - chunk = ChatCompletionStreamResponse(
181 - id=request_id,
182 - object=chunk_object_type,
183 - created=created_time,
184 - choices=[choice_data],
185 - model=model_name)
186 - data = chunk.model_dump_json(exclude_unset=True)
187 - yield f"data: {data}\n\n"
188 - else:
189 - # Send the finish response for each request.n only once
190 - prompt_tokens = len(res.prompt_token_ids)
191 - final_usage = UsageInfo(
192 - prompt_tokens=prompt_tokens,
193 - completion_tokens=previous_num_tokens[i],
194 - total_tokens=prompt_tokens + previous_num_tokens[i],
195 - )
196 - choice_data = ChatCompletionResponseStreamChoice(
197 - index=i,
198 - delta=DeltaMessage(content=delta_text),
199 - logprobs=logprobs,
200 - finish_reason=output.finish_reason)
201 - chunk = ChatCompletionStreamResponse(
202 - id=request_id,
203 - object=chunk_object_type,
204 - created=created_time,
205 - choices=[choice_data],
206 - model=model_name)
207 - if final_usage is not None:
208 - chunk.usage = final_usage
209 - data = chunk.model_dump_json(exclude_unset=True,
210 - exclude_none=True)
211 - yield f"data: {data}\n\n"
212 - finish_reason_sent[i] = True
112 + try:
113 + async for res in result_generator:
114 + res: RequestOutput
115 + # We need to do it here, because if there are exceptions in
116 + # the result_generator, it needs to be sent as the FIRST
117 + # response (by the try...catch).
118 + if first_iteration:
119 + # Send first response for each request.n (index) with the role
120 + role = self.get_chat_request_role(request)
121 + for i in range(request.n):
122 + choice_data = ChatCompletionResponseStreamChoice(
123 + index=i,
124 + delta=DeltaMessage(role=role),
125 + logprobs=None,
126 + finish_reason=None)
127 + chunk = ChatCompletionStreamResponse(
128 + id=request_id,
129 + object=chunk_object_type,
130 + created=created_time,
131 + choices=[choice_data],
132 + model=model_name)
133 + data = chunk.model_dump_json(exclude_unset=True)
134 + yield f"data: {data}\n\n"
135 +
136 + # Send response to echo the input portion of the last message
137 + if request.echo:
138 + last_msg_content = ""
139 + if request.messages and isinstance(
140 + request.messages,
141 + list) and request.messages[-1].get(
142 + "content") and request.messages[-1].get(
143 + "role") == role:
144 + last_msg_content = request.messages[-1]["content"]
145 +
146 + if last_msg_content:
147 + for i in range(request.n):
148 + choice_data = ChatCompletionResponseStreamChoice(
149 + index=i,
150 + delta=DeltaMessage(
151 + content=last_msg_content),
152 + finish_reason=None)
153 + chunk = ChatCompletionStreamResponse(
154 + id=request_id,
155 + object=chunk_object_type,
156 + created=created_time,
157 + choices=[choice_data],
158 + logprobs=None,
159 + model=model_name)
160 + data = chunk.model_dump_json(
161 + exclude_unset=True)
162 + yield f"data: {data}\n\n"
163 + first_iteration = False
164 +
165 + for output in res.outputs:
166 + i = output.index
167 +
168 + if finish_reason_sent[i]:
169 + continue
170 +
171 + delta_token_ids = output.token_ids[previous_num_tokens[i]:]
172 + top_logprobs = output.logprobs[
173 + previous_num_tokens[i]:] if output.logprobs else None
174 +
175 + if request.logprobs:
176 + logprobs = self._create_logprobs(
177 + token_ids=delta_token_ids,
178 + top_logprobs=top_logprobs,
179 + num_output_top_logprobs=request.logprobs,
180 + initial_text_offset=len(previous_texts[i]),
181 + )
182 + else:
183 + logprobs = None
184 +
185 + delta_text = output.text[len(previous_texts[i]):]
186 + previous_texts[i] = output.text
187 + previous_num_tokens[i] = len(output.token_ids)
188 + if output.finish_reason is None:
189 + # Send token-by-token response for each request.n
190 + choice_data = ChatCompletionResponseStreamChoice(
191 + index=i,
192 + delta=DeltaMessage(content=delta_text),
193 + logprobs=logprobs,
194 + finish_reason=None)
195 + chunk = ChatCompletionStreamResponse(
196 + id=request_id,
197 + object=chunk_object_type,
198 + created=created_time,
199 + choices=[choice_data],
200 + model=model_name)
201 + data = chunk.model_dump_json(exclude_unset=True)
202 + yield f"data: {data}\n\n"
203 + else:
204 + # Send the finish response for each request.n only once
205 + prompt_tokens = len(res.prompt_token_ids)
206 + final_usage = UsageInfo(
207 + prompt_tokens=prompt_tokens,
208 + completion_tokens=previous_num_tokens[i],
209 + total_tokens=prompt_tokens +
210 + previous_num_tokens[i],
211 + )
212 + choice_data = ChatCompletionResponseStreamChoice(
213 + index=i,
214 + delta=DeltaMessage(content=delta_text),
215 + logprobs=logprobs,
216 + finish_reason=output.finish_reason)
217 + chunk = ChatCompletionStreamResponse(
218 + id=request_id,
219 + object=chunk_object_type,
220 + created=created_time,
221 + choices=[choice_data],
222 + model=model_name)
223 + if final_usage is not None:
224 + chunk.usage = final_usage
225 + data = chunk.model_dump_json(exclude_unset=True,
226 + exclude_none=True)
227 + yield f"data: {data}\n\n"
228 + finish_reason_sent[i] = True
229 + except ValueError as e:
230 + # TODO: Use a vllm-specific Validation Error
231 + data = self.create_streaming_error_response(str(e))
232 + yield f"data: {data}\n\n"
233 # Send the final done message after all response.n are finished
234 yield "data: [DONE]\n\n"
235
vllm/entrypoints/openai/serving_completion.py
+203 -188
@@ -26,107 +26,6 @@ TypeCreateLogProbsFn = Callable[
26 [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
27
28
29 -async def completion_stream_generator(
30 - request: CompletionRequest,
31 - raw_request: Request,
32 - on_abort,
33 - result_generator: AsyncIterator[Tuple[int, RequestOutput]],
34 - create_logprobs_fn: TypeCreateLogProbsFn,
35 - request_id: str,
36 - created_time: int,
37 - model_name: str,
38 - num_prompts: int,
39 -) -> AsyncGenerator[str, None]:
40 - previous_texts = [""] * request.n * num_prompts
41 - previous_num_tokens = [0] * request.n * num_prompts
42 - has_echoed = [False] * request.n * num_prompts
43 -
44 - async for prompt_idx, res in result_generator:
45 -
46 - # Abort the request if the client disconnects.
47 - if await raw_request.is_disconnected():
48 - await on_abort(f"{request_id}-{prompt_idx}")
49 - raise StopAsyncIteration()
50 -
51 - for output in res.outputs:
52 - i = output.index + prompt_idx * request.n
53 - # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
54 -
55 - if request.echo and request.max_tokens == 0:
56 - # only return the prompt
57 - delta_text = res.prompt
58 - delta_token_ids = res.prompt_token_ids
59 - top_logprobs = res.prompt_logprobs
60 - has_echoed[i] = True
61 - elif request.echo and request.max_tokens > 0 and not has_echoed[i]:
62 - # echo the prompt and first token
63 - delta_text = res.prompt + output.text
64 - delta_token_ids = res.prompt_token_ids + output.token_ids
65 - top_logprobs = res.prompt_logprobs + (output.logprobs or [])
66 - has_echoed[i] = True
67 - else:
68 - # return just the delta
69 - delta_text = output.text[len(previous_texts[i]):]
70 - delta_token_ids = output.token_ids[previous_num_tokens[i]:]
71 - top_logprobs = output.logprobs[
72 - previous_num_tokens[i]:] if output.logprobs else None
73 -
74 - if request.logprobs is not None:
75 - assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
76 - logprobs = create_logprobs_fn(
77 - token_ids=delta_token_ids,
78 - top_logprobs=top_logprobs,
79 - num_output_top_logprobs=request.logprobs,
80 - initial_text_offset=len(previous_texts[i]),
81 - )
82 - else:
83 - logprobs = None
84 -
85 - previous_texts[i] = output.text
86 - previous_num_tokens[i] = len(output.token_ids)
87 - finish_reason = output.finish_reason
88 - response_json = CompletionStreamResponse(
89 - id=request_id,
90 - created=created_time,
91 - model=model_name,
92 - choices=[
93 - CompletionResponseStreamChoice(
94 - index=i,
95 - text=delta_text,
96 - logprobs=logprobs,
97 - finish_reason=finish_reason,
98 - )
99 - ]).model_dump_json()
100 - yield f"data: {response_json}\n\n"
101 -
102 - if output.finish_reason is not None: # return final usage
103 - logprobs = LogProbs() if request.logprobs is not None else None
104 - prompt_tokens = len(res.prompt_token_ids)
105 - completion_tokens = len(output.token_ids)
106 - final_usage = UsageInfo(
107 - prompt_tokens=prompt_tokens,
108 - completion_tokens=completion_tokens,
109 - total_tokens=prompt_tokens + completion_tokens,
110 - )
111 - response_json = CompletionStreamResponse(
112 - id=request_id,
113 - created=created_time,
114 - model=model_name,
115 - choices=[
116 - CompletionResponseStreamChoice(
117 - index=i,
118 - text="",
119 - logprobs=logprobs,
120 - finish_reason=output.finish_reason,
121 - )
122 - ],
123 - usage=final_usage,
124 - ).model_dump_json()
125 - yield f"data: {response_json}\n\n"
126 -
127 - yield "data: [DONE]\n\n"
128 -
129 -
29 def parse_prompt_format(prompt) -> Tuple[bool, list]:
30 # get the prompt, openai supports the following
31 # "a string, array of strings, array of tokens, or array of token arrays."
@@ -151,73 +50,6 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
50 return prompt_is_tokens, prompts
51
52
154 -def request_output_to_completion_response(
155 - final_res_batch: List[RequestOutput],
156 - request: CompletionRequest,
157 - create_logprobs_fn: TypeCreateLogProbsFn,
158 - request_id: str,
159 - created_time: int,
160 - model_name: str,
161 -) -> CompletionResponse:
162 - choices = []
163 - num_prompt_tokens = 0
164 - num_generated_tokens = 0
165 - for final_res in final_res_batch:
166 - assert final_res is not None
167 - prompt_token_ids = final_res.prompt_token_ids
168 - prompt_logprobs = final_res.prompt_logprobs
169 - prompt_text = final_res.prompt
170 -
171 - for output in final_res.outputs:
172 - if request.echo and request.max_tokens == 0:
173 - token_ids = prompt_token_ids
174 - top_logprobs = prompt_logprobs
175 - output_text = prompt_text
176 - elif request.echo and request.max_tokens > 0:
177 - token_ids = prompt_token_ids + output.token_ids
178 - top_logprobs = prompt_logprobs + output.logprobs
179 - output_text = prompt_text + output.text
180 - else:
181 - token_ids = output.token_ids
182 - top_logprobs = output.logprobs
183 - output_text = output.text
184 -
185 - if request.logprobs is not None:
186 - logprobs = create_logprobs_fn(
187 - token_ids=token_ids,
188 - top_logprobs=top_logprobs,
189 - num_output_top_logprobs=request.logprobs,
190 - )
191 - else:
192 - logprobs = None
193 -
194 - choice_data = CompletionResponseChoice(
195 - index=len(choices),
196 - text=output_text,
197 - logprobs=logprobs,
198 - finish_reason=output.finish_reason,
199 - )
200 - choices.append(choice_data)
201 -
202 - num_prompt_tokens += len(prompt_token_ids)
203 - num_generated_tokens += sum(
204 - len(output.token_ids) for output in final_res.outputs)
205 -
206 - usage = UsageInfo(
207 - prompt_tokens=num_prompt_tokens,
208 - completion_tokens=num_generated_tokens,
209 - total_tokens=num_prompt_tokens + num_generated_tokens,
210 - )
211 -
212 - return CompletionResponse(
213 - id=request_id,
214 - created=created_time,
215 - model=model_name,
216 - choices=choices,
217 - usage=usage,
218 - )
219 -
220 -
53 def merge_async_iterators(*iterators):
54 """Merge multiple asynchronous iterators into a single iterator.
55
@@ -230,8 +62,11 @@ def merge_async_iterators(*iterators):
62 finished = [False] * len(iterators)
63
64 async def producer(i, iterator):
233 - async for item in iterator:
234 - await queue.put((i, item))
65 + try:
66 + async for item in iterator:
67 + await queue.put((i, item))
68 + except Exception as e:
69 + await queue.put(e)
70 finished[i] = True
71
72 _tasks = [
@@ -242,6 +77,8 @@ def merge_async_iterators(*iterators):
77 async def consumer():
78 while not all(finished) or not queue.empty():
79 item = await queue.get()
80 + if isinstance(item, Exception):
81 + raise item
82 yield item
83 await asyncio.gather(*_tasks)
84
@@ -312,6 +149,7 @@ class OpenAIServingCompletion(OpenAIServing):
149 prompt_token_ids=input_ids,
150 lora_request=lora_request))
151 except ValueError as e:
152 + # TODO: Use a vllm-specific Validation Error
153 return self.create_error_response(str(e))
154
155 result_generator: AsyncIterator[Tuple[
@@ -325,27 +163,28 @@ class OpenAIServingCompletion(OpenAIServing):
163
164 # Streaming response
165 if stream:
328 - return completion_stream_generator(request,
329 - raw_request,
330 - self.engine.abort,
331 - result_generator,
332 - self._create_logprobs,
333 - request_id,
334 - created_time,
335 - model_name,
336 - num_prompts=len(prompts))
166 + return self.completion_stream_generator(request,
167 + raw_request,
168 + result_generator,
169 + request_id,
170 + created_time,
171 + model_name,
172 + num_prompts=len(prompts))
173
174 # Non-streaming response
175 final_res_batch: RequestOutput = [None] * len(prompts)
340 - async for i, res in result_generator:
341 - if await raw_request.is_disconnected():
342 - # Abort the request if the client disconnects.
343 - await self.engine.abort(f"{request_id}-{i}")
344 - return self.create_error_response("Client disconnected")
345 - final_res_batch[i] = res
346 - response = request_output_to_completion_response(
347 - final_res_batch, request, self._create_logprobs, request_id,
348 - created_time, model_name)
176 + try:
177 + async for i, res in result_generator:
178 + if await raw_request.is_disconnected():
179 + # Abort the request if the client disconnects.
180 + await self.engine.abort(f"{request_id}-{i}")
181 + return self.create_error_response("Client disconnected")
182 + final_res_batch[i] = res
183 + response = self.request_output_to_completion_response(
184 + final_res_batch, request, request_id, created_time, model_name)
185 + except ValueError as e:
186 + # TODO: Use a vllm-specific Validation Error
187 + return self.create_error_response(str(e))
188
189 # When user requests streaming but we don't stream, we still need to
190 # return a streaming response with a single event.
@@ -359,3 +198,179 @@ class OpenAIServingCompletion(OpenAIServing):
198 return fake_stream_generator()
199
200 return response
201 +
202 + async def completion_stream_generator(
203 + self,
204 + request: CompletionRequest,
205 + raw_request: Request,
206 + result_generator: AsyncIterator[Tuple[int, RequestOutput]],
207 + request_id: str,
208 + created_time: int,
209 + model_name: str,
210 + num_prompts: int,
211 + ) -> AsyncGenerator[str, None]:
212 + previous_texts = [""] * request.n * num_prompts
213 + previous_num_tokens = [0] * request.n * num_prompts
214 + has_echoed = [False] * request.n * num_prompts
215 +
216 + try:
217 + async for prompt_idx, res in result_generator:
218 +
219 + # Abort the request if the client disconnects.
220 + if await raw_request.is_disconnected():
221 + await self.engine.abort(f"{request_id}-{prompt_idx}")
222 + raise StopAsyncIteration()
223 +
224 + for output in res.outputs:
225 + i = output.index + prompt_idx * request.n
226 + # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
227 +
228 + if request.echo and request.max_tokens == 0:
229 + # only return the prompt
230 + delta_text = res.prompt
231 + delta_token_ids = res.prompt_token_ids
232 + top_logprobs = res.prompt_logprobs
233 + has_echoed[i] = True
234 + elif request.echo and request.max_tokens > 0 and not has_echoed[
235 + i]:
236 + # echo the prompt and first token
237 + delta_text = res.prompt + output.text
238 + delta_token_ids = res.prompt_token_ids + output.token_ids
239 + top_logprobs = res.prompt_logprobs + (output.logprobs
240 + or [])
241 + has_echoed[i] = True
242 + else:
243 + # return just the delta
244 + delta_text = output.text[len(previous_texts[i]):]
245 + delta_token_ids = output.token_ids[
246 + previous_num_tokens[i]:]
247 + top_logprobs = output.logprobs[previous_num_tokens[
248 + i]:] if output.logprobs else None
249 +
250 + if request.logprobs is not None:
251 + assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
252 + logprobs = self._create_logprobs(
253 + token_ids=delta_token_ids,
254 + top_logprobs=top_logprobs,
255 + num_output_top_logprobs=request.logprobs,
256 + initial_text_offset=len(previous_texts[i]),
257 + )
258 + else:
259 + logprobs = None
260 +
261 + previous_texts[i] = output.text
262 + previous_num_tokens[i] = len(output.token_ids)
263 + finish_reason = output.finish_reason
264 + response_json = CompletionStreamResponse(
265 + id=request_id,
266 + created=created_time,
267 + model=model_name,
268 + choices=[
269 + CompletionResponseStreamChoice(
270 + index=i,
271 + text=delta_text,
272 + logprobs=logprobs,
273 + finish_reason=finish_reason,
274 + )
275 + ]).model_dump_json()
276 + yield f"data: {response_json}\n\n"
277 +
278 + if output.finish_reason is not None: # return final usage
279 + logprobs = LogProbs(
280 + ) if request.logprobs is not None else None
281 + prompt_tokens = len(res.prompt_token_ids)
282 + completion_tokens = len(output.token_ids)
283 + final_usage = UsageInfo(
284 + prompt_tokens=prompt_tokens,
285 + completion_tokens=completion_tokens,
286 + total_tokens=prompt_tokens + completion_tokens,
287 + )
288 + response_json = CompletionStreamResponse(
289 + id=request_id,
290 + created=created_time,
291 + model=model_name,
292 + choices=[
293 + CompletionResponseStreamChoice(
294 + index=i,
295 + text="",
296 + logprobs=logprobs,
297 + finish_reason=output.finish_reason,
298 + )
299 + ],
300 + usage=final_usage,
301 + ).model_dump_json()
302 + yield f"data: {response_json}\n\n"
303 + except ValueError as e:
304 + # TODO: Use a vllm-specific Validation Error
305 + data = self.create_streaming_error_response(str(e))
306 + print("yield", f"data: {data}\n\n")
307 + yield f"data: {data}\n\n"
308 +
309 + print("yield", "data: [DONE]\n\n")
310 + yield "data: [DONE]\n\n"
311 +
312 + def request_output_to_completion_response(
313 + self,
314 + final_res_batch: List[RequestOutput],
315 + request: CompletionRequest,
316 + request_id: str,
317 + created_time: int,
318 + model_name: str,
319 + ) -> CompletionResponse:
320 + choices = []
321 + num_prompt_tokens = 0
322 + num_generated_tokens = 0
323 + for final_res in final_res_batch:
324 + assert final_res is not None
325 + prompt_token_ids = final_res.prompt_token_ids
326 + prompt_logprobs = final_res.prompt_logprobs
327 + prompt_text = final_res.prompt
328 +
329 + for output in final_res.outputs:
330 + if request.echo and request.max_tokens == 0:
331 + token_ids = prompt_token_ids
332 + top_logprobs = prompt_logprobs
333 + output_text = prompt_text
334 + elif request.echo and request.max_tokens > 0:
335 + token_ids = prompt_token_ids + output.token_ids
336 + top_logprobs = prompt_logprobs + output.logprobs
337 + output_text = prompt_text + output.text
338 + else:
339 + token_ids = output.token_ids
340 + top_logprobs = output.logprobs
341 + output_text = output.text
342 +
343 + if request.logprobs is not None:
344 + logprobs = self._create_logprobs(
345 + token_ids=token_ids,
346 + top_logprobs=top_logprobs,
347 + num_output_top_logprobs=request.logprobs,
348 + )
349 + else:
350 + logprobs = None
351 +
352 + choice_data = CompletionResponseChoice(
353 + index=len(choices),
354 + text=output_text,
355 + logprobs=logprobs,
356 + finish_reason=output.finish_reason,
357 + )
358 + choices.append(choice_data)
359 +
360 + num_prompt_tokens += len(prompt_token_ids)
361 + num_generated_tokens += sum(
362 + len(output.token_ids) for output in final_res.outputs)
363 +
364 + usage = UsageInfo(
365 + prompt_tokens=num_prompt_tokens,
366 + completion_tokens=num_generated_tokens,
367 + total_tokens=num_prompt_tokens + num_generated_tokens,
368 + )
369 +
370 + return CompletionResponse(
371 + id=request_id,
372 + created=created_time,
373 + model=model_name,
374 + choices=choices,
375 + usage=usage,
376 + )
vllm/entrypoints/openai/serving_engine.py
+19 -4
@@ -1,4 +1,5 @@
1 import asyncio
2 +import json
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 from typing import Dict, List, Optional, Union
@@ -11,6 +12,7 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
12 ModelCard, ModelList,
13 ModelPermission)
14 from vllm.lora.request import LoRARequest
15 +from vllm.sequence import Logprob
16
17 logger = init_logger(__name__)
18
@@ -83,7 +85,7 @@ class OpenAIServing:
85 def _create_logprobs(
86 self,
87 token_ids: List[int],
86 - top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
88 + top_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None,
89 num_output_top_logprobs: Optional[int] = None,
90 initial_text_offset: int = 0,
91 ) -> LogProbs:
@@ -95,10 +97,10 @@ class OpenAIServing:
97 for i, token_id in enumerate(token_ids):
98 step_top_logprobs = top_logprobs[i]
99 if step_top_logprobs is not None:
98 - token_logprob = step_top_logprobs[token_id]
100 + token_logprob = step_top_logprobs[token_id].logprob
101 else:
102 token_logprob = None
101 - token = self.tokenizer.convert_ids_to_tokens(token_id)
103 + token = step_top_logprobs[token_id].decoded_token
104 logprobs.tokens.append(token)
105 logprobs.token_logprobs.append(token_logprob)
106 if len(logprobs.text_offset) == 0:
@@ -110,7 +112,7 @@ class OpenAIServing:
112
113 if num_output_top_logprobs:
114 logprobs.top_logprobs.append({
113 - self.tokenizer.convert_ids_to_tokens(i): p
115 + p.decoded_token: p.logprob
116 for i, p in step_top_logprobs.items()
117 } if step_top_logprobs else None)
118 return logprobs
@@ -124,6 +126,19 @@ class OpenAIServing:
126 type=err_type,
127 code=status_code.value)
128
129 + def create_streaming_error_response(
130 + self,
131 + message: str,
132 + err_type: str = "BadRequestError",
133 + status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> str:
134 + json_str = json.dumps({
135 + "error":
136 + self.create_error_response(message=message,
137 + err_type=err_type,
138 + status_code=status_code).model_dump()
139 + })
140 + return json_str
141 +
142 async def _check_model(self, request) -> Optional[ErrorResponse]:
143 if request.model == self.served_model:
144 return
vllm/model_executor/layers/sampler.py
+11 -4
@@ -8,8 +8,9 @@ from vllm.model_executor.parallel_utils.communication_op import (
8 tensor_model_parallel_gather)
9 from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
10 from vllm.sampling_params import SamplingParams, SamplingType
11 -from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12 - SequenceData, SequenceGroupOutput, SequenceOutput)
11 +from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
12 + SamplerOutput, SequenceData, SequenceGroupOutput,
13 + SequenceOutput)
14 from vllm.utils import is_neuron
15
16
@@ -528,7 +529,10 @@ def _get_logprobs(
529 prompt_logprobs_dict.update(
530 zip(top_token_ids[sample_idx, :num_logprobs].tolist(),
531 top_logprobs[sample_idx, :num_logprobs].tolist()))
531 - group_prompt_logprobs.append(prompt_logprobs_dict)
532 + group_prompt_logprobs.append({
533 + token_id: Logprob(logprob)
534 + for token_id, logprob in prompt_logprobs_dict.items()
535 + })
536 sample_idx += 1
537 query_result_idx += 1
538 result_prompt_logprobs.append(group_prompt_logprobs)
@@ -553,7 +557,10 @@ def _get_logprobs(
557 parent_id, :num_logprobs].tolist(),
558 top_logprobs[sample_idx +
559 parent_id, :num_logprobs].tolist()))
556 - group_sample_logprobs.append(sample_logprobs_dict)
560 + group_sample_logprobs.append({
561 + token_id: Logprob(logprob)
562 + for token_id, logprob in sample_logprobs_dict.items()
563 + })
564 result_sample_logprobs.append(group_sample_logprobs)
565 sample_idx += len(seq_ids)
566
vllm/sequence.py
+17 -8
@@ -8,8 +8,16 @@ from vllm.block import LogicalTokenBlock
8 from vllm.sampling_params import SamplingParams
9 from vllm.lora.request import LoRARequest
10
11 -PromptLogprobs = List[Optional[Dict[int, float]]]
12 -SampleLogprobs = List[Dict[int, float]]
11 +
12 +@dataclass
13 +class Logprob:
14 + """Infos for supporting OpenAI compatible logprobs."""
15 + logprob: float
16 + decoded_token: Optional[str] = None
17 +
18 +
19 +PromptLogprobs = List[Optional[Dict[int, Logprob]]]
20 +SampleLogprobs = List[Dict[int, Logprob]]
21
22
23 class SequenceStatus(enum.Enum):
@@ -196,12 +204,12 @@ class Sequence:
204 def append_token_id(
205 self,
206 token_id: int,
199 - logprobs: Dict[int, float],
207 + logprobs: Dict[int, Logprob],
208 ) -> None:
209 assert token_id in logprobs
210 self._append_tokens_to_blocks([token_id])
211 self.output_logprobs.append(logprobs)
204 - self.data.append_token_id(token_id, logprobs[token_id])
212 + self.data.append_token_id(token_id, logprobs[token_id].logprob)
213
214 def get_len(self) -> int:
215 return self.data.get_len()
@@ -456,7 +464,7 @@ class SequenceOutput:
464 self,
465 parent_seq_id: int,
466 output_token: int,
459 - logprobs: Dict[int, float],
467 + logprobs: Dict[int, Logprob],
468 ) -> None:
469 self.parent_seq_id = parent_seq_id
470 self.output_token = output_token
@@ -470,9 +478,10 @@ class SequenceOutput:
478 def __eq__(self, other: object) -> bool:
479 if not isinstance(other, SequenceOutput):
480 raise NotImplementedError()
473 - return (self.parent_seq_id == other.parent_seq_id
474 - and self.output_token == other.output_token
475 - and self.logprobs == other.logprobs)
481 + equal = (self.parent_seq_id == other.parent_seq_id
482 + and self.output_token == other.output_token)
483 + log_probs_equal = other.logprobs == self.logprobs
484 + return equal and log_probs_equal
485
486
487 class SequenceGroupOutput:
vllm/worker/spec_decode/multi_step_worker.py
+1 -1
@@ -77,7 +77,7 @@ class MultiStepWorker(Worker):
77 token_id = seq_output.output_token
78 token_logprob = seq_output.logprobs[token_id]
79
80 - seq.append_token_id(token_id, token_logprob)
80 + seq.append_token_id(token_id, token_logprob.logprob)
81
82 def _shallow_copy_inputs(
83 self, seq_group_metadata_list: List[SequenceGroupMetadata]