A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Push logprob generation to LLMEngine (#3065)
Co-authored-by: Avnish Narayan <avnish@anyscale.com>
Antoni Baum committed
Mar 4, 2024 at 11:54 UTC
22de45235c6dd14e901e089971635ec655d5fbe0
13 files changed
+555
-335
tests/entrypoints/test_openai_server.py
+58
-3
@@ -213,14 +213,14 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
213
messages=messages,
214
max_tokens=10,
215
logprobs=True,
216
- top_logprobs=10)
216
+ top_logprobs=5)
217
assert chat_completion.id is not None
218
assert chat_completion.choices is not None and len(
219
chat_completion.choices) == 1
220
assert chat_completion.choices[0].message is not None
221
assert chat_completion.choices[0].logprobs is not None
222
assert chat_completion.choices[0].logprobs.top_logprobs is not None
223
- assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 10
223
+ assert len(chat_completion.choices[0].logprobs.top_logprobs[0]) == 5
224
message = chat_completion.choices[0].message
225
assert message.content is not None and len(message.content) >= 10
226
assert message.role == "assistant"
@@ -229,7 +229,7 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
229
# test multi-turn dialogue
230
messages.append({"role": "user", "content": "express your result in json"})
231
chat_completion = await client.chat.completions.create(
232
- model=MODEL_NAME,
232
+ model=model_name,
233
messages=messages,
234
max_tokens=10,
235
)
@@ -237,6 +237,61 @@ async def test_single_chat_session(server, client: openai.AsyncOpenAI,
237
assert message.content is not None and len(message.content) >= 0
238
239
240
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
241
+async def test_too_many_logprobs(server, client: openai.AsyncOpenAI,
242
+ model_name: str):
243
+ messages = [{
244
+ "role": "system",
245
+ "content": "you are a helpful assistant"
246
+ }, {
247
+ "role": "user",
248
+ "content": "what is 1+1?"
249
+ }]
250
+
251
+ # Default max_logprobs is 5, so this should raise an error
252
+ with pytest.raises((openai.BadRequestError, openai.APIError)):
253
+ stream = await client.chat.completions.create(model=model_name,
254
+ messages=messages,
255
+ max_tokens=10,
256
+ logprobs=True,
257
+ top_logprobs=10,
258
+ stream=True)
259
+ async for chunk in stream:
260
+ ...
261
+
262
+ with pytest.raises(openai.BadRequestError):
263
+ await client.chat.completions.create(model=model_name,
264
+ messages=messages,
265
+ max_tokens=10,
266
+ logprobs=True,
267
+ top_logprobs=10,
268
+ stream=False)
269
+
270
+ with pytest.raises((openai.BadRequestError, openai.APIError)):
271
+ stream = await client.completions.create(model=model_name,
272
+ prompt="Test",
273
+ max_tokens=10,
274
+ logprobs=10,
275
+ stream=True)
276
+ async for chunk in stream:
277
+ ...
278
+
279
+ with pytest.raises(openai.BadRequestError):
280
+ await client.completions.create(model=model_name,
281
+ prompt="Test",
282
+ max_tokens=10,
283
+ logprobs=10,
284
+ stream=False)
285
+
286
+ # the server should still work afterwards
287
+ chat_completion = await client.chat.completions.create(model=model_name,
288
+ messages=messages,
289
+ max_tokens=10,
290
+ stream=False)
291
+ message = chat_completion.choices[0].message
292
+ assert message.content is not None and len(message.content) >= 0
293
+
294
+
295
@pytest.mark.parametrize(
296
# just test 1 lora hereafter
297
"model_name",
tests/samplers/test_logprobs.py
+36
-6
@@ -1,5 +1,6 @@
1
import pytest
2
import torch
3
+from tests.conftest import VllmRunner
4
5
from vllm import SamplingParams
6
@@ -16,6 +17,7 @@ def test_get_prompt_logprobs(
17
example_prompts,
18
):
19
max_tokens = 5
20
+ num_top_logprobs = 6
21
hf_model = hf_runner(model, dtype=dtype)
22
hf_logprobs = hf_model.generate_greedy_logprobs(
23
example_prompts,
@@ -23,19 +25,32 @@ def test_get_prompt_logprobs(
25
)
26
del hf_model
27
26
- vllm_model = vllm_runner(model, dtype=dtype)
28
+ vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
29
vllm_sampling_params = SamplingParams(max_tokens=max_tokens,
28
- logprobs=5,
30
+ logprobs=num_top_logprobs,
31
prompt_logprobs=5,
32
temperature=0.0)
33
vllm_results = vllm_model.model.generate(
34
example_prompts, sampling_params=vllm_sampling_params)
33
- del vllm_model
35
36
# Test whether logprobs are included in the results.
37
for result in vllm_results:
38
assert result.prompt_logprobs is not None
39
assert result.outputs[0].logprobs is not None
40
+ assert len(result.outputs[0].logprobs) == max_tokens
41
+ for logprobs in result.outputs[0].logprobs:
42
+ assert len(logprobs) == num_top_logprobs
43
+ output_text = result.outputs[0].text
44
+ output_string_from_most_likely_tokens = []
45
+ for top_logprobs in result.outputs[0].logprobs:
46
+ top_logprob = next(iter(top_logprobs.values()))
47
+ output_string_from_most_likely_tokens.append(
48
+ top_logprob.decoded_token)
49
+ output_string_from_most_likely_tokens = "".join(
50
+ output_string_from_most_likely_tokens)
51
+ assert output_text == output_string_from_most_likely_tokens, (
52
+ "The output text from the top logprob for each token position "
53
+ "should be the same as the output text in the result.")
54
55
# Test whether prompt logprobs are consistent with HF
56
for vllm_result, hf_logprob in zip(vllm_results, hf_logprobs):
@@ -43,14 +58,29 @@ def test_get_prompt_logprobs(
58
vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
59
for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
60
for token_id, logprob in vllm_prompt_logprob_dict.items():
46
- torch.testing.assert_close(logprob,
61
+ torch.testing.assert_close(logprob.logprob,
62
hf_logprob[0][i][token_id].item(),
63
atol=1e-2,
64
rtol=1e-2)
65
vllm_sample_logprobs = vllm_result.outputs[0].logprobs
51
- for i, vllm_sample_logprob_dict in enumerate(vllm_sample_logprobs):
52
- for token_id, logprob in vllm_sample_logprob_dict.items():
66
+ for i, top_logprobs in enumerate(vllm_sample_logprobs):
67
+ for token_id, sample_logprob in top_logprobs.items():
68
+ logprob = sample_logprob.logprob
69
torch.testing.assert_close(logprob,
70
hf_logprob[i][-1][token_id].item(),
71
atol=1e-2,
72
rtol=1e-2)
73
+ assert isinstance(sample_logprob.decoded_token, str), \
74
+ ("The token should be decoded by the time it is returned "
75
+ " to the user.")
76
+
77
+
78
+def test_max_logprobs():
79
+ runner = VllmRunner("facebook/opt-125m", max_logprobs=1)
80
+ vllm_sampling_params = SamplingParams(logprobs=1)
81
+ # should pass
82
+ runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
83
+
84
+ bad_sampling_params = SamplingParams(logprobs=2)
85
+ with pytest.raises(ValueError):
86
+ runner.generate(["Hello world"], sampling_params=bad_sampling_params)
tests/worker/spec_decode/utils.py
+7
-5
@@ -4,7 +4,7 @@ from typing import List, Optional, Dict
4
from vllm.worker.worker import Worker
5
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
6
from vllm.engine.arg_utils import EngineArgs
7
-from vllm.sequence import SequenceGroupMetadata, SequenceData
7
+from vllm.sequence import Logprob, SequenceGroupMetadata, SequenceData
8
from vllm.sampling_params import SamplingParams
9
from vllm.worker.cache_engine import CacheEngine
10
from vllm.model_executor.utils import set_random_seed
@@ -166,13 +166,15 @@ def create_seq_group_metadata_from_prompts(
166
167
168
def assert_logprobs_dict_allclose(
169
- actual_logprobs: List[Dict[int, float]],
170
- expected_logprobs: List[Dict[int, float]]) -> None:
169
+ actual_logprobs: List[Dict[int, Logprob]],
170
+ expected_logprobs: List[Dict[int, Logprob]]) -> None:
171
for single_step_actual_logprobs, single_step_expected_logprobs in zip(
172
actual_logprobs, expected_logprobs):
173
assert set(single_step_actual_logprobs.keys()) == set(
174
single_step_expected_logprobs.keys())
175
for token_id in single_step_actual_logprobs:
176
- actual = torch.tensor(single_step_actual_logprobs[token_id])
177
- expected = torch.tensor(single_step_expected_logprobs[token_id])
176
+ actual = torch.tensor(
177
+ single_step_actual_logprobs[token_id].logprob)
178
+ expected = torch.tensor(
179
+ single_step_expected_logprobs[token_id].logprob)
180
assert torch.allclose(actual, expected)
vllm/config.py
+2
@@ -79,6 +79,7 @@ class ModelConfig:
79
quantization: Optional[str] = None,
80
enforce_eager: bool = False,
81
max_context_len_to_capture: Optional[int] = None,
82
+ max_logprobs: int = 5,
83
) -> None:
84
self.model = model
85
self.tokenizer = tokenizer
@@ -93,6 +94,7 @@ class ModelConfig:
94
self.quantization = quantization
95
self.enforce_eager = enforce_eager
96
self.max_context_len_to_capture = max_context_len_to_capture
97
+ self.max_logprobs = max_logprobs
98
99
if os.environ.get("VLLM_USE_MODELSCOPE", "False").lower() == "true":
100
# download model from ModelScope hub,
vllm/engine/arg_utils.py
+9
-1
@@ -31,6 +31,7 @@ class EngineArgs:
31
max_num_batched_tokens: Optional[int] = None
32
max_num_seqs: int = 256
33
max_paddings: int = 256
34
+ max_logprobs: int = 5 # OpenAI default value
35
disable_log_stats: bool = False
36
revision: Optional[str] = None
37
code_revision: Optional[str] = None
@@ -212,6 +213,12 @@ class EngineArgs:
213
type=int,
214
default=EngineArgs.max_paddings,
215
help='maximum number of paddings in a batch')
216
+ parser.add_argument(
217
+ '--max-logprobs',
218
+ type=int,
219
+ default=EngineArgs.max_logprobs,
220
+ help=('max number of log probs to return logprobs is specified in'
221
+ ' SamplingParams'))
222
parser.add_argument('--disable-log-stats',
223
action='store_true',
224
help='disable logging statistics')
@@ -300,7 +307,8 @@ class EngineArgs:
307
self.trust_remote_code, self.download_dir, self.load_format,
308
self.dtype, self.seed, self.revision, self.code_revision,
309
self.tokenizer_revision, self.max_model_len, self.quantization,
303
- self.enforce_eager, self.max_context_len_to_capture)
310
+ self.enforce_eager, self.max_context_len_to_capture,
311
+ self.max_logprobs)
312
cache_config = CacheConfig(self.block_size,
313
self.gpu_memory_utilization,
314
self.swap_space, self.kv_cache_dtype,
vllm/engine/async_llm_engine.py
+24
-5
@@ -47,7 +47,7 @@ class AsyncStream:
47
self._queue = asyncio.Queue()
48
self._finished = False
49
50
- def put(self, item: RequestOutput) -> None:
50
+ def put(self, item: Union[RequestOutput, Exception]) -> None:
51
if self._finished:
52
return
53
self._queue.put_nowait(item)
@@ -110,6 +110,17 @@ class RequestTracker:
110
logger.info(f"Finished request {request_id}.")
111
self.abort_request(request_id)
112
113
+ def process_exception(self,
114
+ request_id: str,
115
+ exception: Exception,
116
+ *,
117
+ verbose: bool = False) -> None:
118
+ """Propagate an exception from the engine."""
119
+ self._request_streams[request_id].put(exception)
120
+ if verbose:
121
+ logger.info(f"Finished request {request_id}.")
122
+ self.abort_request(request_id)
123
+
124
def add_request(self, request_id: str,
125
**engine_add_request_kwargs) -> AsyncStream:
126
"""Add a request to be sent to the engine on the next background
@@ -377,10 +388,18 @@ class AsyncLLMEngine:
388
for new_request in new_requests:
389
# Add the request into the vLLM engine's waiting queue.
390
# TODO: Maybe add add_request_batch to reduce Ray overhead
380
- if self.engine_use_ray:
381
- await self.engine.add_request.remote(**new_request)
382
- else:
383
- await self.engine.add_request_async(**new_request)
391
+ try:
392
+ if self.engine_use_ray:
393
+ await self.engine.add_request.remote(**new_request)
394
+ else:
395
+ await self.engine.add_request_async(**new_request)
396
+ except ValueError as e:
397
+ # TODO: use a vLLM specific error for failed validation
398
+ self._request_tracker.process_exception(
399
+ new_request["request_id"],
400
+ e,
401
+ verbose=self.log_requests,
402
+ )
403
404
if finished_requests:
405
await self._engine_abort(finished_requests)
vllm/engine/llm_engine.py
+40
-2
@@ -18,7 +18,7 @@ from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
18
from vllm.logger import init_logger
19
from vllm.outputs import RequestOutput
20
from vllm.sampling_params import SamplingParams
21
-from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
21
+from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
22
SequenceGroupOutput, SequenceOutput, SequenceStatus)
23
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
24
TokenizerGroup)
@@ -473,6 +473,13 @@ class LLMEngine:
473
if lora_request is not None and not self.lora_config:
474
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
475
"not enabled!")
476
+ max_logprobs = self.get_model_config().max_logprobs
477
+ if (sampling_params.logprobs
478
+ and sampling_params.logprobs > max_logprobs) or (
479
+ sampling_params.prompt_logprobs
480
+ and sampling_params.prompt_logprobs > max_logprobs):
481
+ raise ValueError(f"Cannot request more than "
482
+ f"{max_logprobs} logprobs.")
483
if arrival_time is None:
484
arrival_time = time.monotonic()
485
prompt_token_ids = self.encode_request(
@@ -583,6 +590,13 @@ class LLMEngine:
590
# Process prompt logprobs
591
prompt_logprobs = outputs.prompt_logprobs
592
if prompt_logprobs is not None:
593
+ # We can pick any sequence for the prompt.
594
+ seq = next(iter(seq_group.seqs_dict.values()))
595
+ all_token_ids = seq.get_token_ids()
596
+ for i, prompt_logprobs_for_token in enumerate(prompt_logprobs):
597
+ self._decode_logprobs(seq, seq_group.sampling_params,
598
+ prompt_logprobs_for_token,
599
+ all_token_ids[:i])
600
seq_group.prompt_logprobs = prompt_logprobs
601
602
# Process samples
@@ -930,12 +944,36 @@ class LLMEngine:
944
time_e2e_requests=time_e2e_requests,
945
)
946
947
+ def _decode_logprobs(self, seq: Sequence, prms: SamplingParams,
948
+ logprobs: Dict[int, Logprob],
949
+ all_input_ids: List[int]) -> None:
950
+ if not logprobs:
951
+ return
952
+ for token_id, sample_logprob in logprobs.items():
953
+ if (sample_logprob.decoded_token is None and token_id != -1):
954
+ all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
955
+ _, new_text, prefix_offset, read_offset = detokenize_incrementally(
956
+ self.get_tokenizer_for_seq(seq),
957
+ all_input_ids=all_input_ids_with_logprob,
958
+ prev_tokens=seq.tokens,
959
+ prefix_offset=seq.prefix_offset,
960
+ read_offset=seq.read_offset,
961
+ skip_special_tokens=prms.skip_special_tokens,
962
+ spaces_between_special_tokens=prms.
963
+ spaces_between_special_tokens,
964
+ )
965
+ sample_logprob.decoded_token = new_text
966
+
967
def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
968
"""Decodes the new token for a sequence."""
969
+ all_input_ids = seq.get_token_ids()
970
+ self._decode_logprobs(seq, prms, seq.output_logprobs[-1],
971
+ all_input_ids)
972
+
973
(new_tokens, new_output_text, prefix_offset,
974
read_offset) = detokenize_incrementally(
975
self.get_tokenizer_for_seq(seq),
938
- all_input_ids=seq.get_token_ids(),
976
+ all_input_ids=all_input_ids,
977
prev_tokens=seq.tokens,
978
prefix_offset=seq.prefix_offset,
979
read_offset=seq.read_offset,
vllm/entrypoints/openai/serving_chat.py
+128
-108
@@ -82,8 +82,12 @@ class OpenAIServingChat(OpenAIServing):
82
return self.chat_completion_stream_generator(
83
request, result_generator, request_id)
84
else:
85
- return await self.chat_completion_full_generator(
86
- request, raw_request, result_generator, request_id)
85
+ try:
86
+ return await self.chat_completion_full_generator(
87
+ request, raw_request, result_generator, request_id)
88
+ except ValueError as e:
89
+ # TODO: Use a vllm-specific Validation Error
90
+ return self.create_error_response(str(e))
91
92
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
93
if request.add_generation_prompt:
@@ -99,117 +103,133 @@ class OpenAIServingChat(OpenAIServing):
103
model_name = request.model
104
created_time = int(time.monotonic())
105
chunk_object_type = "chat.completion.chunk"
102
-
103
- # Send first response for each request.n (index) with the role
104
- role = self.get_chat_request_role(request)
105
- for i in range(request.n):
106
- choice_data = ChatCompletionResponseStreamChoice(
107
- index=i,
108
- delta=DeltaMessage(role=role),
109
- logprobs=None,
110
- finish_reason=None)
111
- chunk = ChatCompletionStreamResponse(id=request_id,
112
- object=chunk_object_type,
113
- created=created_time,
114
- choices=[choice_data],
115
- model=model_name)
116
- data = chunk.model_dump_json(exclude_unset=True)
117
- yield f"data: {data}\n\n"
118
-
119
- # Send response to echo the input portion of the last message
120
- if request.echo:
121
- last_msg_content = ""
122
- if request.messages and isinstance(
123
- request.messages, list) and request.messages[-1].get(
124
- "content") and request.messages[-1].get(
125
- "role") == role:
126
- last_msg_content = request.messages[-1]["content"]
127
-
128
- if last_msg_content:
129
- for i in range(request.n):
130
- choice_data = ChatCompletionResponseStreamChoice(
131
- index=i,
132
- delta=DeltaMessage(content=last_msg_content),
133
- finish_reason=None)
134
- chunk = ChatCompletionStreamResponse(
135
- id=request_id,
136
- object=chunk_object_type,
137
- created=created_time,
138
- choices=[choice_data],
139
- logprobs=None,
140
- model=model_name)
141
- data = chunk.model_dump_json(exclude_unset=True)
142
- yield f"data: {data}\n\n"
106
+ first_iteration = True
107
108
# Send response for each token for each request.n (index)
109
previous_texts = [""] * request.n
110
previous_num_tokens = [0] * request.n
111
finish_reason_sent = [False] * request.n
148
- async for res in result_generator:
149
- res: RequestOutput
150
- for output in res.outputs:
151
- i = output.index
152
-
153
- if finish_reason_sent[i]:
154
- continue
155
-
156
- delta_token_ids = output.token_ids[previous_num_tokens[i]:]
157
- top_logprobs = output.logprobs[
158
- previous_num_tokens[i]:] if output.logprobs else None
159
-
160
- if request.logprobs:
161
- logprobs = self._create_logprobs(
162
- token_ids=delta_token_ids,
163
- top_logprobs=top_logprobs,
164
- num_output_top_logprobs=request.logprobs,
165
- initial_text_offset=len(previous_texts[i]),
166
- )
167
- else:
168
- logprobs = None
169
-
170
- delta_text = output.text[len(previous_texts[i]):]
171
- previous_texts[i] = output.text
172
- previous_num_tokens[i] = len(output.token_ids)
173
- if output.finish_reason is None:
174
- # Send token-by-token response for each request.n
175
- choice_data = ChatCompletionResponseStreamChoice(
176
- index=i,
177
- delta=DeltaMessage(content=delta_text),
178
- logprobs=logprobs,
179
- finish_reason=None)
180
- chunk = ChatCompletionStreamResponse(
181
- id=request_id,
182
- object=chunk_object_type,
183
- created=created_time,
184
- choices=[choice_data],
185
- model=model_name)
186
- data = chunk.model_dump_json(exclude_unset=True)
187
- yield f"data: {data}\n\n"
188
- else:
189
- # Send the finish response for each request.n only once
190
- prompt_tokens = len(res.prompt_token_ids)
191
- final_usage = UsageInfo(
192
- prompt_tokens=prompt_tokens,
193
- completion_tokens=previous_num_tokens[i],
194
- total_tokens=prompt_tokens + previous_num_tokens[i],
195
- )
196
- choice_data = ChatCompletionResponseStreamChoice(
197
- index=i,
198
- delta=DeltaMessage(content=delta_text),
199
- logprobs=logprobs,
200
- finish_reason=output.finish_reason)
201
- chunk = ChatCompletionStreamResponse(
202
- id=request_id,
203
- object=chunk_object_type,
204
- created=created_time,
205
- choices=[choice_data],
206
- model=model_name)
207
- if final_usage is not None:
208
- chunk.usage = final_usage
209
- data = chunk.model_dump_json(exclude_unset=True,
210
- exclude_none=True)
211
- yield f"data: {data}\n\n"
212
- finish_reason_sent[i] = True
112
+ try:
113
+ async for res in result_generator:
114
+ res: RequestOutput
115
+ # We need to do it here, because if there are exceptions in
116
+ # the result_generator, it needs to be sent as the FIRST
117
+ # response (by the try...catch).
118
+ if first_iteration:
119
+ # Send first response for each request.n (index) with the role
120
+ role = self.get_chat_request_role(request)
121
+ for i in range(request.n):
122
+ choice_data = ChatCompletionResponseStreamChoice(
123
+ index=i,
124
+ delta=DeltaMessage(role=role),
125
+ logprobs=None,
126
+ finish_reason=None)
127
+ chunk = ChatCompletionStreamResponse(
128
+ id=request_id,
129
+ object=chunk_object_type,
130
+ created=created_time,
131
+ choices=[choice_data],
132
+ model=model_name)
133
+ data = chunk.model_dump_json(exclude_unset=True)
134
+ yield f"data: {data}\n\n"
135
+
136
+ # Send response to echo the input portion of the last message
137
+ if request.echo:
138
+ last_msg_content = ""
139
+ if request.messages and isinstance(
140
+ request.messages,
141
+ list) and request.messages[-1].get(
142
+ "content") and request.messages[-1].get(
143
+ "role") == role:
144
+ last_msg_content = request.messages[-1]["content"]
145
+
146
+ if last_msg_content:
147
+ for i in range(request.n):
148
+ choice_data = ChatCompletionResponseStreamChoice(
149
+ index=i,
150
+ delta=DeltaMessage(
151
+ content=last_msg_content),
152
+ finish_reason=None)
153
+ chunk = ChatCompletionStreamResponse(
154
+ id=request_id,
155
+ object=chunk_object_type,
156
+ created=created_time,
157
+ choices=[choice_data],
158
+ logprobs=None,
159
+ model=model_name)
160
+ data = chunk.model_dump_json(
161
+ exclude_unset=True)
162
+ yield f"data: {data}\n\n"
163
+ first_iteration = False
164
+
165
+ for output in res.outputs:
166
+ i = output.index
167
+
168
+ if finish_reason_sent[i]:
169
+ continue
170
+
171
+ delta_token_ids = output.token_ids[previous_num_tokens[i]:]
172
+ top_logprobs = output.logprobs[
173
+ previous_num_tokens[i]:] if output.logprobs else None
174
+
175
+ if request.logprobs:
176
+ logprobs = self._create_logprobs(
177
+ token_ids=delta_token_ids,
178
+ top_logprobs=top_logprobs,
179
+ num_output_top_logprobs=request.logprobs,
180
+ initial_text_offset=len(previous_texts[i]),
181
+ )
182
+ else:
183
+ logprobs = None
184
+
185
+ delta_text = output.text[len(previous_texts[i]):]
186
+ previous_texts[i] = output.text
187
+ previous_num_tokens[i] = len(output.token_ids)
188
+ if output.finish_reason is None:
189
+ # Send token-by-token response for each request.n
190
+ choice_data = ChatCompletionResponseStreamChoice(
191
+ index=i,
192
+ delta=DeltaMessage(content=delta_text),
193
+ logprobs=logprobs,
194
+ finish_reason=None)
195
+ chunk = ChatCompletionStreamResponse(
196
+ id=request_id,
197
+ object=chunk_object_type,
198
+ created=created_time,
199
+ choices=[choice_data],
200
+ model=model_name)
201
+ data = chunk.model_dump_json(exclude_unset=True)
202
+ yield f"data: {data}\n\n"
203
+ else:
204
+ # Send the finish response for each request.n only once
205
+ prompt_tokens = len(res.prompt_token_ids)
206
+ final_usage = UsageInfo(
207
+ prompt_tokens=prompt_tokens,
208
+ completion_tokens=previous_num_tokens[i],
209
+ total_tokens=prompt_tokens +
210
+ previous_num_tokens[i],
211
+ )
212
+ choice_data = ChatCompletionResponseStreamChoice(
213
+ index=i,
214
+ delta=DeltaMessage(content=delta_text),
215
+ logprobs=logprobs,
216
+ finish_reason=output.finish_reason)
217
+ chunk = ChatCompletionStreamResponse(
218
+ id=request_id,
219
+ object=chunk_object_type,
220
+ created=created_time,
221
+ choices=[choice_data],
222
+ model=model_name)
223
+ if final_usage is not None:
224
+ chunk.usage = final_usage
225
+ data = chunk.model_dump_json(exclude_unset=True,
226
+ exclude_none=True)
227
+ yield f"data: {data}\n\n"
228
+ finish_reason_sent[i] = True
229
+ except ValueError as e:
230
+ # TODO: Use a vllm-specific Validation Error
231
+ data = self.create_streaming_error_response(str(e))
232
+ yield f"data: {data}\n\n"
233
# Send the final done message after all response.n are finished
234
yield "data: [DONE]\n\n"
235
vllm/entrypoints/openai/serving_completion.py
+203
-188
@@ -26,107 +26,6 @@ TypeCreateLogProbsFn = Callable[
26
[TypeTokenIDs, TypeTopLogProbs, Optional[int], int], LogProbs]
27
28
29
-async def completion_stream_generator(
30
- request: CompletionRequest,
31
- raw_request: Request,
32
- on_abort,
33
- result_generator: AsyncIterator[Tuple[int, RequestOutput]],
34
- create_logprobs_fn: TypeCreateLogProbsFn,
35
- request_id: str,
36
- created_time: int,
37
- model_name: str,
38
- num_prompts: int,
39
-) -> AsyncGenerator[str, None]:
40
- previous_texts = [""] * request.n * num_prompts
41
- previous_num_tokens = [0] * request.n * num_prompts
42
- has_echoed = [False] * request.n * num_prompts
43
-
44
- async for prompt_idx, res in result_generator:
45
-
46
- # Abort the request if the client disconnects.
47
- if await raw_request.is_disconnected():
48
- await on_abort(f"{request_id}-{prompt_idx}")
49
- raise StopAsyncIteration()
50
-
51
- for output in res.outputs:
52
- i = output.index + prompt_idx * request.n
53
- # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
54
-
55
- if request.echo and request.max_tokens == 0:
56
- # only return the prompt
57
- delta_text = res.prompt
58
- delta_token_ids = res.prompt_token_ids
59
- top_logprobs = res.prompt_logprobs
60
- has_echoed[i] = True
61
- elif request.echo and request.max_tokens > 0 and not has_echoed[i]:
62
- # echo the prompt and first token
63
- delta_text = res.prompt + output.text
64
- delta_token_ids = res.prompt_token_ids + output.token_ids
65
- top_logprobs = res.prompt_logprobs + (output.logprobs or [])
66
- has_echoed[i] = True
67
- else:
68
- # return just the delta
69
- delta_text = output.text[len(previous_texts[i]):]
70
- delta_token_ids = output.token_ids[previous_num_tokens[i]:]
71
- top_logprobs = output.logprobs[
72
- previous_num_tokens[i]:] if output.logprobs else None
73
-
74
- if request.logprobs is not None:
75
- assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
76
- logprobs = create_logprobs_fn(
77
- token_ids=delta_token_ids,
78
- top_logprobs=top_logprobs,
79
- num_output_top_logprobs=request.logprobs,
80
- initial_text_offset=len(previous_texts[i]),
81
- )
82
- else:
83
- logprobs = None
84
-
85
- previous_texts[i] = output.text
86
- previous_num_tokens[i] = len(output.token_ids)
87
- finish_reason = output.finish_reason
88
- response_json = CompletionStreamResponse(
89
- id=request_id,
90
- created=created_time,
91
- model=model_name,
92
- choices=[
93
- CompletionResponseStreamChoice(
94
- index=i,
95
- text=delta_text,
96
- logprobs=logprobs,
97
- finish_reason=finish_reason,
98
- )
99
- ]).model_dump_json()
100
- yield f"data: {response_json}\n\n"
101
-
102
- if output.finish_reason is not None: # return final usage
103
- logprobs = LogProbs() if request.logprobs is not None else None
104
- prompt_tokens = len(res.prompt_token_ids)
105
- completion_tokens = len(output.token_ids)
106
- final_usage = UsageInfo(
107
- prompt_tokens=prompt_tokens,
108
- completion_tokens=completion_tokens,
109
- total_tokens=prompt_tokens + completion_tokens,
110
- )
111
- response_json = CompletionStreamResponse(
112
- id=request_id,
113
- created=created_time,
114
- model=model_name,
115
- choices=[
116
- CompletionResponseStreamChoice(
117
- index=i,
118
- text="",
119
- logprobs=logprobs,
120
- finish_reason=output.finish_reason,
121
- )
122
- ],
123
- usage=final_usage,
124
- ).model_dump_json()
125
- yield f"data: {response_json}\n\n"
126
-
127
- yield "data: [DONE]\n\n"
128
-
129
-
29
def parse_prompt_format(prompt) -> Tuple[bool, list]:
30
# get the prompt, openai supports the following
31
# "a string, array of strings, array of tokens, or array of token arrays."
@@ -151,73 +50,6 @@ def parse_prompt_format(prompt) -> Tuple[bool, list]:
50
return prompt_is_tokens, prompts
51
52
154
-def request_output_to_completion_response(
155
- final_res_batch: List[RequestOutput],
156
- request: CompletionRequest,
157
- create_logprobs_fn: TypeCreateLogProbsFn,
158
- request_id: str,
159
- created_time: int,
160
- model_name: str,
161
-) -> CompletionResponse:
162
- choices = []
163
- num_prompt_tokens = 0
164
- num_generated_tokens = 0
165
- for final_res in final_res_batch:
166
- assert final_res is not None
167
- prompt_token_ids = final_res.prompt_token_ids
168
- prompt_logprobs = final_res.prompt_logprobs
169
- prompt_text = final_res.prompt
170
-
171
- for output in final_res.outputs:
172
- if request.echo and request.max_tokens == 0:
173
- token_ids = prompt_token_ids
174
- top_logprobs = prompt_logprobs
175
- output_text = prompt_text
176
- elif request.echo and request.max_tokens > 0:
177
- token_ids = prompt_token_ids + output.token_ids
178
- top_logprobs = prompt_logprobs + output.logprobs
179
- output_text = prompt_text + output.text
180
- else:
181
- token_ids = output.token_ids
182
- top_logprobs = output.logprobs
183
- output_text = output.text
184
-
185
- if request.logprobs is not None:
186
- logprobs = create_logprobs_fn(
187
- token_ids=token_ids,
188
- top_logprobs=top_logprobs,
189
- num_output_top_logprobs=request.logprobs,
190
- )
191
- else:
192
- logprobs = None
193
-
194
- choice_data = CompletionResponseChoice(
195
- index=len(choices),
196
- text=output_text,
197
- logprobs=logprobs,
198
- finish_reason=output.finish_reason,
199
- )
200
- choices.append(choice_data)
201
-
202
- num_prompt_tokens += len(prompt_token_ids)
203
- num_generated_tokens += sum(
204
- len(output.token_ids) for output in final_res.outputs)
205
-
206
- usage = UsageInfo(
207
- prompt_tokens=num_prompt_tokens,
208
- completion_tokens=num_generated_tokens,
209
- total_tokens=num_prompt_tokens + num_generated_tokens,
210
- )
211
-
212
- return CompletionResponse(
213
- id=request_id,
214
- created=created_time,
215
- model=model_name,
216
- choices=choices,
217
- usage=usage,
218
- )
219
-
220
-
53
def merge_async_iterators(*iterators):
54
"""Merge multiple asynchronous iterators into a single iterator.
55
@@ -230,8 +62,11 @@ def merge_async_iterators(*iterators):
62
finished = [False] * len(iterators)
63
64
async def producer(i, iterator):
233
- async for item in iterator:
234
- await queue.put((i, item))
65
+ try:
66
+ async for item in iterator:
67
+ await queue.put((i, item))
68
+ except Exception as e:
69
+ await queue.put(e)
70
finished[i] = True
71
72
_tasks = [
@@ -242,6 +77,8 @@ def merge_async_iterators(*iterators):
77
async def consumer():
78
while not all(finished) or not queue.empty():
79
item = await queue.get()
80
+ if isinstance(item, Exception):
81
+ raise item
82
yield item
83
await asyncio.gather(*_tasks)
84
@@ -312,6 +149,7 @@ class OpenAIServingCompletion(OpenAIServing):
149
prompt_token_ids=input_ids,
150
lora_request=lora_request))
151
except ValueError as e:
152
+ # TODO: Use a vllm-specific Validation Error
153
return self.create_error_response(str(e))
154
155
result_generator: AsyncIterator[Tuple[
@@ -325,27 +163,28 @@ class OpenAIServingCompletion(OpenAIServing):
163
164
# Streaming response
165
if stream:
328
- return completion_stream_generator(request,
329
- raw_request,
330
- self.engine.abort,
331
- result_generator,
332
- self._create_logprobs,
333
- request_id,
334
- created_time,
335
- model_name,
336
- num_prompts=len(prompts))
166
+ return self.completion_stream_generator(request,
167
+ raw_request,
168
+ result_generator,
169
+ request_id,
170
+ created_time,
171
+ model_name,
172
+ num_prompts=len(prompts))
173
174
# Non-streaming response
175
final_res_batch: RequestOutput = [None] * len(prompts)
340
- async for i, res in result_generator:
341
- if await raw_request.is_disconnected():
342
- # Abort the request if the client disconnects.
343
- await self.engine.abort(f"{request_id}-{i}")
344
- return self.create_error_response("Client disconnected")
345
- final_res_batch[i] = res
346
- response = request_output_to_completion_response(
347
- final_res_batch, request, self._create_logprobs, request_id,
348
- created_time, model_name)
176
+ try:
177
+ async for i, res in result_generator:
178
+ if await raw_request.is_disconnected():
179
+ # Abort the request if the client disconnects.
180
+ await self.engine.abort(f"{request_id}-{i}")
181
+ return self.create_error_response("Client disconnected")
182
+ final_res_batch[i] = res
183
+ response = self.request_output_to_completion_response(
184
+ final_res_batch, request, request_id, created_time, model_name)
185
+ except ValueError as e:
186
+ # TODO: Use a vllm-specific Validation Error
187
+ return self.create_error_response(str(e))
188
189
# When user requests streaming but we don't stream, we still need to
190
# return a streaming response with a single event.
@@ -359,3 +198,179 @@ class OpenAIServingCompletion(OpenAIServing):
198
return fake_stream_generator()
199
200
return response
201
+
202
+ async def completion_stream_generator(
203
+ self,
204
+ request: CompletionRequest,
205
+ raw_request: Request,
206
+ result_generator: AsyncIterator[Tuple[int, RequestOutput]],
207
+ request_id: str,
208
+ created_time: int,
209
+ model_name: str,
210
+ num_prompts: int,
211
+ ) -> AsyncGenerator[str, None]:
212
+ previous_texts = [""] * request.n * num_prompts
213
+ previous_num_tokens = [0] * request.n * num_prompts
214
+ has_echoed = [False] * request.n * num_prompts
215
+
216
+ try:
217
+ async for prompt_idx, res in result_generator:
218
+
219
+ # Abort the request if the client disconnects.
220
+ if await raw_request.is_disconnected():
221
+ await self.engine.abort(f"{request_id}-{prompt_idx}")
222
+ raise StopAsyncIteration()
223
+
224
+ for output in res.outputs:
225
+ i = output.index + prompt_idx * request.n
226
+ # TODO(simon): optimize the performance by avoiding full text O(n^2) sending.
227
+
228
+ if request.echo and request.max_tokens == 0:
229
+ # only return the prompt
230
+ delta_text = res.prompt
231
+ delta_token_ids = res.prompt_token_ids
232
+ top_logprobs = res.prompt_logprobs
233
+ has_echoed[i] = True
234
+ elif request.echo and request.max_tokens > 0 and not has_echoed[
235
+ i]:
236
+ # echo the prompt and first token
237
+ delta_text = res.prompt + output.text
238
+ delta_token_ids = res.prompt_token_ids + output.token_ids
239
+ top_logprobs = res.prompt_logprobs + (output.logprobs
240
+ or [])
241
+ has_echoed[i] = True
242
+ else:
243
+ # return just the delta
244
+ delta_text = output.text[len(previous_texts[i]):]
245
+ delta_token_ids = output.token_ids[
246
+ previous_num_tokens[i]:]
247
+ top_logprobs = output.logprobs[previous_num_tokens[
248
+ i]:] if output.logprobs else None
249
+
250
+ if request.logprobs is not None:
251
+ assert top_logprobs is not None, "top_logprobs must be provided when logprobs is requested"
252
+ logprobs = self._create_logprobs(
253
+ token_ids=delta_token_ids,
254
+ top_logprobs=top_logprobs,
255
+ num_output_top_logprobs=request.logprobs,
256
+ initial_text_offset=len(previous_texts[i]),
257
+ )
258
+ else:
259
+ logprobs = None
260
+
261
+ previous_texts[i] = output.text
262
+ previous_num_tokens[i] = len(output.token_ids)
263
+ finish_reason = output.finish_reason
264
+ response_json = CompletionStreamResponse(
265
+ id=request_id,
266
+ created=created_time,
267
+ model=model_name,
268
+ choices=[
269
+ CompletionResponseStreamChoice(
270
+ index=i,
271
+ text=delta_text,
272
+ logprobs=logprobs,
273
+ finish_reason=finish_reason,
274
+ )
275
+ ]).model_dump_json()
276
+ yield f"data: {response_json}\n\n"
277
+
278
+ if output.finish_reason is not None: # return final usage
279
+ logprobs = LogProbs(
280
+ ) if request.logprobs is not None else None
281
+ prompt_tokens = len(res.prompt_token_ids)
282
+ completion_tokens = len(output.token_ids)
283
+ final_usage = UsageInfo(
284
+ prompt_tokens=prompt_tokens,
285
+ completion_tokens=completion_tokens,
286
+ total_tokens=prompt_tokens + completion_tokens,
287
+ )
288
+ response_json = CompletionStreamResponse(
289
+ id=request_id,
290
+ created=created_time,
291
+ model=model_name,
292
+ choices=[
293
+ CompletionResponseStreamChoice(
294
+ index=i,
295
+ text="",
296
+ logprobs=logprobs,
297
+ finish_reason=output.finish_reason,
298
+ )
299
+ ],
300
+ usage=final_usage,
301
+ ).model_dump_json()
302
+ yield f"data: {response_json}\n\n"
303
+ except ValueError as e:
304
+ # TODO: Use a vllm-specific Validation Error
305
+ data = self.create_streaming_error_response(str(e))
306
+ print("yield", f"data: {data}\n\n")
307
+ yield f"data: {data}\n\n"
308
+
309
+ print("yield", "data: [DONE]\n\n")
310
+ yield "data: [DONE]\n\n"
311
+
312
+ def request_output_to_completion_response(
313
+ self,
314
+ final_res_batch: List[RequestOutput],
315
+ request: CompletionRequest,
316
+ request_id: str,
317
+ created_time: int,
318
+ model_name: str,
319
+ ) -> CompletionResponse:
320
+ choices = []
321
+ num_prompt_tokens = 0
322
+ num_generated_tokens = 0
323
+ for final_res in final_res_batch:
324
+ assert final_res is not None
325
+ prompt_token_ids = final_res.prompt_token_ids
326
+ prompt_logprobs = final_res.prompt_logprobs
327
+ prompt_text = final_res.prompt
328
+
329
+ for output in final_res.outputs:
330
+ if request.echo and request.max_tokens == 0:
331
+ token_ids = prompt_token_ids
332
+ top_logprobs = prompt_logprobs
333
+ output_text = prompt_text
334
+ elif request.echo and request.max_tokens > 0:
335
+ token_ids = prompt_token_ids + output.token_ids
336
+ top_logprobs = prompt_logprobs + output.logprobs
337
+ output_text = prompt_text + output.text
338
+ else:
339
+ token_ids = output.token_ids
340
+ top_logprobs = output.logprobs
341
+ output_text = output.text
342
+
343
+ if request.logprobs is not None:
344
+ logprobs = self._create_logprobs(
345
+ token_ids=token_ids,
346
+ top_logprobs=top_logprobs,
347
+ num_output_top_logprobs=request.logprobs,
348
+ )
349
+ else:
350
+ logprobs = None
351
+
352
+ choice_data = CompletionResponseChoice(
353
+ index=len(choices),
354
+ text=output_text,
355
+ logprobs=logprobs,
356
+ finish_reason=output.finish_reason,
357
+ )
358
+ choices.append(choice_data)
359
+
360
+ num_prompt_tokens += len(prompt_token_ids)
361
+ num_generated_tokens += sum(
362
+ len(output.token_ids) for output in final_res.outputs)
363
+
364
+ usage = UsageInfo(
365
+ prompt_tokens=num_prompt_tokens,
366
+ completion_tokens=num_generated_tokens,
367
+ total_tokens=num_prompt_tokens + num_generated_tokens,
368
+ )
369
+
370
+ return CompletionResponse(
371
+ id=request_id,
372
+ created=created_time,
373
+ model=model_name,
374
+ choices=choices,
375
+ usage=usage,
376
+ )
vllm/entrypoints/openai/serving_engine.py
+19
-4
@@ -1,4 +1,5 @@
1
import asyncio
2
+import json
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
from typing import Dict, List, Optional, Union
@@ -11,6 +12,7 @@ from vllm.entrypoints.openai.protocol import (CompletionRequest,
12
ModelCard, ModelList,
13
ModelPermission)
14
from vllm.lora.request import LoRARequest
15
+from vllm.sequence import Logprob
16
17
logger = init_logger(__name__)
18
@@ -83,7 +85,7 @@ class OpenAIServing:
85
def _create_logprobs(
86
self,
87
token_ids: List[int],
86
- top_logprobs: Optional[List[Optional[Dict[int, float]]]] = None,
88
+ top_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None,
89
num_output_top_logprobs: Optional[int] = None,
90
initial_text_offset: int = 0,
91
) -> LogProbs:
@@ -95,10 +97,10 @@ class OpenAIServing:
97
for i, token_id in enumerate(token_ids):
98
step_top_logprobs = top_logprobs[i]
99
if step_top_logprobs is not None:
98
- token_logprob = step_top_logprobs[token_id]
100
+ token_logprob = step_top_logprobs[token_id].logprob
101
else:
102
token_logprob = None
101
- token = self.tokenizer.convert_ids_to_tokens(token_id)
103
+ token = step_top_logprobs[token_id].decoded_token
104
logprobs.tokens.append(token)
105
logprobs.token_logprobs.append(token_logprob)
106
if len(logprobs.text_offset) == 0:
@@ -110,7 +112,7 @@ class OpenAIServing:
112
113
if num_output_top_logprobs:
114
logprobs.top_logprobs.append({
113
- self.tokenizer.convert_ids_to_tokens(i): p
115
+ p.decoded_token: p.logprob
116
for i, p in step_top_logprobs.items()
117
} if step_top_logprobs else None)
118
return logprobs
@@ -124,6 +126,19 @@ class OpenAIServing:
126
type=err_type,
127
code=status_code.value)
128
129
+ def create_streaming_error_response(
130
+ self,
131
+ message: str,
132
+ err_type: str = "BadRequestError",
133
+ status_code: HTTPStatus = HTTPStatus.BAD_REQUEST) -> str:
134
+ json_str = json.dumps({
135
+ "error":
136
+ self.create_error_response(message=message,
137
+ err_type=err_type,
138
+ status_code=status_code).model_dump()
139
+ })
140
+ return json_str
141
+
142
async def _check_model(self, request) -> Optional[ErrorResponse]:
143
if request.model == self.served_model:
144
return
vllm/model_executor/layers/sampler.py
+11
-4
@@ -8,8 +8,9 @@ from vllm.model_executor.parallel_utils.communication_op import (
8
tensor_model_parallel_gather)
9
from vllm.model_executor.sampling_metadata import SamplingMetadata, SamplingTensors
10
from vllm.sampling_params import SamplingParams, SamplingType
11
-from vllm.sequence import (PromptLogprobs, SampleLogprobs, SamplerOutput,
12
- SequenceData, SequenceGroupOutput, SequenceOutput)
11
+from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
12
+ SamplerOutput, SequenceData, SequenceGroupOutput,
13
+ SequenceOutput)
14
from vllm.utils import is_neuron
15
16
@@ -528,7 +529,10 @@ def _get_logprobs(
529
prompt_logprobs_dict.update(
530
zip(top_token_ids[sample_idx, :num_logprobs].tolist(),
531
top_logprobs[sample_idx, :num_logprobs].tolist()))
531
- group_prompt_logprobs.append(prompt_logprobs_dict)
532
+ group_prompt_logprobs.append({
533
+ token_id: Logprob(logprob)
534
+ for token_id, logprob in prompt_logprobs_dict.items()
535
+ })
536
sample_idx += 1
537
query_result_idx += 1
538
result_prompt_logprobs.append(group_prompt_logprobs)
@@ -553,7 +557,10 @@ def _get_logprobs(
557
parent_id, :num_logprobs].tolist(),
558
top_logprobs[sample_idx +
559
parent_id, :num_logprobs].tolist()))
556
- group_sample_logprobs.append(sample_logprobs_dict)
560
+ group_sample_logprobs.append({
561
+ token_id: Logprob(logprob)
562
+ for token_id, logprob in sample_logprobs_dict.items()
563
+ })
564
result_sample_logprobs.append(group_sample_logprobs)
565
sample_idx += len(seq_ids)
566
vllm/sequence.py
+17
-8
@@ -8,8 +8,16 @@ from vllm.block import LogicalTokenBlock
8
from vllm.sampling_params import SamplingParams
9
from vllm.lora.request import LoRARequest
10
11
-PromptLogprobs = List[Optional[Dict[int, float]]]
12
-SampleLogprobs = List[Dict[int, float]]
11
+
12
+@dataclass
13
+class Logprob:
14
+ """Infos for supporting OpenAI compatible logprobs."""
15
+ logprob: float
16
+ decoded_token: Optional[str] = None
17
+
18
+
19
+PromptLogprobs = List[Optional[Dict[int, Logprob]]]
20
+SampleLogprobs = List[Dict[int, Logprob]]
21
22
23
class SequenceStatus(enum.Enum):
@@ -196,12 +204,12 @@ class Sequence:
204
def append_token_id(
205
self,
206
token_id: int,
199
- logprobs: Dict[int, float],
207
+ logprobs: Dict[int, Logprob],
208
) -> None:
209
assert token_id in logprobs
210
self._append_tokens_to_blocks([token_id])
211
self.output_logprobs.append(logprobs)
204
- self.data.append_token_id(token_id, logprobs[token_id])
212
+ self.data.append_token_id(token_id, logprobs[token_id].logprob)
213
214
def get_len(self) -> int:
215
return self.data.get_len()
@@ -456,7 +464,7 @@ class SequenceOutput:
464
self,
465
parent_seq_id: int,
466
output_token: int,
459
- logprobs: Dict[int, float],
467
+ logprobs: Dict[int, Logprob],
468
) -> None:
469
self.parent_seq_id = parent_seq_id
470
self.output_token = output_token
@@ -470,9 +478,10 @@ class SequenceOutput:
478
def __eq__(self, other: object) -> bool:
479
if not isinstance(other, SequenceOutput):
480
raise NotImplementedError()
473
- return (self.parent_seq_id == other.parent_seq_id
474
- and self.output_token == other.output_token
475
- and self.logprobs == other.logprobs)
481
+ equal = (self.parent_seq_id == other.parent_seq_id
482
+ and self.output_token == other.output_token)
483
+ log_probs_equal = other.logprobs == self.logprobs
484
+ return equal and log_probs_equal
485
486
487
class SequenceGroupOutput:
vllm/worker/spec_decode/multi_step_worker.py
+1
-1
@@ -77,7 +77,7 @@ class MultiStepWorker(Worker):
77
token_id = seq_output.output_token
78
token_logprob = seq_output.logprobs[token_id]
79
80
- seq.append_token_id(token_id, token_logprob)
80
+ seq.append_token_id(token_id, token_logprob.logprob)
81
82
def _shallow_copy_inputs(
83
self, seq_group_metadata_list: List[SequenceGroupMetadata]