1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import List
3
+import math
4
+from typing import Dict, List, Optional
5
6
import pytest
6
-from transformers import AutoTokenizer
7
8
-from vllm.engine.arg_utils import EngineArgs
8
+from tests.v1.engine.utils import (NUM_PROMPT_LOGPROBS_UNDER_TEST,
9
+ NUM_SAMPLE_LOGPROBS_UNDER_TEST,
10
+ STOP_STRINGS,
11
+ DummyOutputProcessorTestVectors,
12
+ MockEngineCore)
13
from vllm.sampling_params import RequestOutputKind, SamplingParams
10
-from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
11
-from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
14
+from vllm.sequence import PromptLogprobs, SampleLogprobs
15
+from vllm.transformers_utils.tokenizer import AnyTokenizer
16
+from vllm.v1.engine import EngineCoreRequest
17
from vllm.v1.engine.output_processor import OutputProcessor
18
14
-TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
15
-VLLM_CONFIG = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
16
-TOKENIZER_GROUP = init_tokenizer_from_configs(VLLM_CONFIG.model_config,
17
- VLLM_CONFIG.scheduler_config,
18
- VLLM_CONFIG.parallel_config,
19
- VLLM_CONFIG.lora_config)
20
-tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
21
-
22
-FULL_STRINGS = [
23
- "My name is Robert from Neural Magic and I love working on vLLM so much!",
24
- "Red Hat is the best open source company by far across Linux, K8s, and AI.",
25
- "Nick is the name of my brother in addition to my colleague from Red Hat.",
26
-]
27
-
28
-STOP_STRINGS = ["I love working on", "company by far", "brother in"]
29
-
30
-FULL_TOKENS = [tokenizer(text).input_ids for text in FULL_STRINGS]
31
-PROMPT_LEN = 5
32
-PROMPT_TOKENS = [
33
- tokenizer(text).input_ids[:PROMPT_LEN] for text in FULL_STRINGS
34
-]
35
-GENERATION_TOKENS = [
36
- tokenizer(text).input_ids[PROMPT_LEN:] for text in FULL_STRINGS
37
-]
38
-PROMPT_STRINGS = [
39
- tokenizer.decode(prompt_tokens, skip_special_tokens=True)
40
- for prompt_tokens in PROMPT_TOKENS
41
-]
42
-PROMPT_STRINGS_LEN = [len(prompt_string) for prompt_string in PROMPT_STRINGS]
43
-GENERATION_STRINGS = [
44
- text[prompt_len:]
45
- for text, prompt_len in zip(FULL_STRINGS, PROMPT_STRINGS_LEN)
46
-]
47
-
48
-
49
-class MockEngineCore:
50
- """Mock outputs form premade tokens lists."""
51
-
52
- def __init__(self, tokens_list: List[List[int]]):
53
- self.tokens_list = tokens_list
54
- self.current_idx = 0
55
-
56
- def get_outputs(self) -> List[EngineCoreOutput]:
57
- token_idx = self.current_idx
58
- self.current_idx += 1
59
-
60
- outputs = []
61
- for req_idx, token_ids in enumerate(self.tokens_list):
62
- if len(token_ids) > token_idx:
63
- output = EngineCoreOutput(request_id=f"request-{req_idx}",
64
- new_token_ids=[token_ids[token_idx]],
65
- finished=False)
66
- if token_idx == len(token_ids) - 1:
67
- output.finished = True
68
- output.finish_reason = "stopped"
69
- outputs.append(output)
70
-
71
- return outputs
19
+
20
+def _ref_convert_id_to_token(
21
+ tokenizer: AnyTokenizer,
22
+ token_id: int,
23
+) -> str:
24
+ """Reference impl of logprobs detokenization.
25
+
26
+ Args:
27
+ tokenizer: tokenizer used by the model under test
28
+ token_id: convert this token id
29
+
30
+ Returns:
31
+ String representation of input token id
32
+ """
33
+ return tokenizer.convert_ids_to_tokens(token_id) or ""
34
35
36
@pytest.mark.parametrize(
37
"request_output_kind",
38
[RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
77
-def test_incremental_detokenization(request_output_kind: RequestOutputKind):
78
- output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
79
- engine_core = MockEngineCore(GENERATION_TOKENS)
39
+def test_incremental_detokenization(request_output_kind: RequestOutputKind,
40
+ dummy_test_vectors):
41
+ output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
42
+ log_stats=False)
43
+ engine_core = MockEngineCore(
44
+ tokens_list=dummy_test_vectors.generation_tokens)
45
46
# Make N requests.
47
requests = [
59
spaces_between_special_tokens=False,
60
output_kind=request_output_kind,
61
stop=[],
97
- include_stop_str_in_output=False))
98
- for idx, (
99
- prompt,
100
- prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
62
+ include_stop_str_in_output=False,
63
+ )) for idx, (prompt, prompt_tokens) in enumerate(
64
+ zip(dummy_test_vectors.prompt_strings,
65
+ dummy_test_vectors.prompt_tokens))
66
]
67
68
# Add requests to the detokenizer.
78
break
79
80
# Step the Detokenizer.
116
- processed_outputs = output_processor.process_outputs(outputs, )
81
+ processed_outputs = output_processor.process_outputs(outputs)
82
request_outputs = processed_outputs.request_outputs
83
requests_to_abort = processed_outputs.reqs_to_abort
84
assert len(requests_to_abort) == 0
97
98
# Confirmed tracked values matches what we expected.
99
for idx, (ref_gen_str, ref_gen_toks) in enumerate(
135
- zip(GENERATION_STRINGS, GENERATION_TOKENS)):
100
+ zip(dummy_test_vectors.generation_strings,
101
+ dummy_test_vectors.generation_tokens)):
102
gen_str = gen_strings[f"request-{idx}"]
103
gen_toks = gen_tokens[f"request-{idx}"]
104
109
assert not output_processor.has_unfinished_requests()
110
111
112
+def _validate_logprobs(
113
+ gen_tokens: Dict[str, List[int]],
114
+ gen_logprobs: Dict[str, Optional[SampleLogprobs]],
115
+ gen_prompt_logprobs: Dict[str, Optional[PromptLogprobs]],
116
+ gen_cumulative_logprob: Dict[str, float],
117
+ dtv: DummyOutputProcessorTestVectors,
118
+ request_id_list: List[str],
119
+ num_sample_logprobs: Optional[int],
120
+ num_prompt_logprobs: Optional[int],
121
+) -> None:
122
+ for req_idx, req_id in enumerate(request_id_list):
123
+ new_tokens = gen_tokens[req_id]
124
+ logprobs = gen_logprobs[req_id]
125
+ prompt_logprobs = gen_prompt_logprobs[req_id]
126
+ cumulative_logprob = gen_cumulative_logprob[req_id]
127
+ prompt_token_ids = dtv.prompt_tokens[req_idx]
128
+ ref_logprobs = dtv.generation_logprobs[req_idx]
129
+ ref_prompt_logprobs = dtv.prompt_logprobs[req_idx]
130
+ if num_sample_logprobs is not None:
131
+ # Validate sample logprobs
132
+ assert logprobs is not None, (f"Request {req_id} requires sample"
133
+ " logprobs but sample logprobs are"
134
+ " None.")
135
+ # Require num sampled tokens to match num
136
+ # sampled logprobs - especially important
137
+ # to check since the detokenizer can cause
138
+ # a request to finish early due to a stop
139
+ # string being hit
140
+ num_new_tokens = len(new_tokens)
141
+ len_sample_logprobs = len(logprobs)
142
+ assert num_new_tokens == len_sample_logprobs, (
143
+ f"Request {req_id} has {num_new_tokens}"
144
+ " completion tokens but has"
145
+ f" {len_sample_logprobs} sample logprobs.")
146
+ ref_cumulative_logprob = 0.0
147
+ for idx, (sampled_token,
148
+ pos_logprob_dict) in enumerate(zip(new_tokens,
149
+ logprobs)):
150
+ # Break out the reference log probability value &
151
+ # logprob token id tensors associated with this
152
+ # position in the completion. Also break out the
153
+ # sampled token ranks
154
+ (ref_pos_logprob_toks, ref_pos_logprob_vals,
155
+ ref_sampled_token_rank) = ref_logprobs[idx]
156
+ # For each position in the completion sequence,
157
+ # ensure the actual sampled token is among the
158
+ # logprobs
159
+ assert sampled_token in pos_logprob_dict, (
160
+ f"Sampled token {sampled_token} not"
161
+ f" present in logprob at index {idx}")
162
+
163
+ # Validate number of sample logprobs
164
+ num_lp_toks = len(pos_logprob_dict)
165
+ assert (num_lp_toks == num_sample_logprobs
166
+ or num_lp_toks == num_sample_logprobs +
167
+ 1), ("Valid numbers of sample logprobs are"
168
+ f" {num_sample_logprobs} or"
169
+ f" {num_sample_logprobs+1} but"
170
+ f" {num_lp_toks} logprobs found at"
171
+ f" position {idx}. Logprobs dict:"
172
+ f" {pos_logprob_dict}")
173
+
174
+ # Validate sampled token logprob rank
175
+ smp_lp = pos_logprob_dict[sampled_token]
176
+ smp_lp_rank = smp_lp.rank
177
+ assert (ref_sampled_token_rank == smp_lp_rank), (
178
+ "Sampled token logprob rank"
179
+ f" {smp_lp_rank} does not match"
180
+ " correct value"
181
+ f" {ref_sampled_token_rank}"
182
+ f" in Logprob {smp_lp}")
183
+
184
+ # Validate that the logprob processor yields
185
+ # the correct log probabilities and valid
186
+ # rankings
187
+ rank_one_appears = False
188
+ for jdx in range(1, len(ref_pos_logprob_toks)):
189
+ # Iterate over the (logprob val,logprob tok id)
190
+ # pairs expected by the test fixture at this
191
+ # position in the completion.
192
+ ref_lp_val = ref_pos_logprob_vals[jdx]
193
+ ref_tok_id = ref_pos_logprob_toks[jdx]
194
+ assert ref_tok_id in pos_logprob_dict, (
195
+ f"Expected token {ref_tok_id} to be"
196
+ f" in logprob dict but it is not.")
197
+
198
+ # Extract actually-generated logprob
199
+ # info
200
+ lp = pos_logprob_dict[ref_tok_id]
201
+ lp_val = lp.logprob
202
+ lp_rank = lp.rank
203
+
204
+ # A "top" (rank 1) logprob must be
205
+ # present
206
+ rank_one_appears = (True
207
+ if lp_rank == 1 else rank_one_appears)
208
+
209
+ # Rank must be >= 1
210
+ assert lp_rank >= 1, (f"Logprob {lp} has invalid"
211
+ f" rank {lp_rank} < 1."
212
+ f" Logprob dict: {pos_logprob_dict}")
213
+
214
+ # Validate log probability
215
+ assert math.isclose(lp_val, ref_lp_val), (
216
+ f"Token id {ref_tok_id} appears in logprobs dict"
217
+ f" at position {idx} in completion with log"
218
+ f" probability {lp_val} but {ref_lp_val} was"
219
+ f" expected. Logprob: {lp}")
220
+
221
+ assert rank_one_appears, (f"No Logprob has rank 1"
222
+ " in the following Logprob"
223
+ f" dict: {pos_logprob_dict}")
224
+
225
+ # Validate logprobs detokenization
226
+ for lp_tok in pos_logprob_dict:
227
+ # Confirm that sample logprob decoded token matches
228
+ # the logprob token id at this sequence position
229
+ decoded_token = pos_logprob_dict[lp_tok].decoded_token
230
+ ref_decoded_token = _ref_convert_id_to_token(
231
+ dtv.tokenizer, lp_tok)
232
+ assert decoded_token == ref_decoded_token, (
233
+ f"Sampled logprob token id {lp_tok} decodes to"
234
+ f" {ref_decoded_token} but Logprob decoded"
235
+ f" token is {decoded_token} instead"
236
+ f" (at position {idx})")
237
+
238
+ ref_cumulative_logprob += pos_logprob_dict[
239
+ sampled_token].logprob
240
+ # Assert that cumulative logprobs are correct
241
+ assert math.isclose(cumulative_logprob, ref_cumulative_logprob)
242
+ else:
243
+ # Sample logprobs disabled for this request
244
+ assert logprobs is None
245
+ assert cumulative_logprob is None
246
+
247
+ if num_prompt_logprobs is not None:
248
+ # Validate prompt logprobs
249
+ assert prompt_logprobs is not None, (
250
+ f"Request {req_id} requires prompt"
251
+ " logprobs but prompt logprobs are"
252
+ " None.")
253
+ # Require num prompt tokens to match num
254
+ # prompt logprobs
255
+ num_prompt_tokens = len(prompt_token_ids)
256
+ len_prompt_logprobs = len(prompt_logprobs)
257
+ assert num_prompt_tokens == len_prompt_logprobs, (
258
+ f"Request {req_id} has {num_prompt_tokens}"
259
+ " prompt tokens but has"
260
+ f" {len_prompt_logprobs} prompt logprobs.")
261
+ # First prompt logprob is None
262
+ first_plp_dict = prompt_logprobs[0]
263
+ assert first_plp_dict is None, (
264
+ f"Request {req_id} first prompt logprob"
265
+ f" should be None but has following value"
266
+ f" instead: {first_plp_dict}")
267
+ # Break out the reference prompt log prob value &
268
+ # logprob token id matrices for the whole prompt.
269
+ # Also break out the prompt token rank vector
270
+ (ref_prompt_logprob_toks, ref_prompt_logprob_vals,
271
+ ref_prompt_token_ranks) = ref_prompt_logprobs
272
+ for idx, (prompt_token, pos_logprob_dict) in enumerate(
273
+ zip(prompt_token_ids[1:], prompt_logprobs[1:])):
274
+
275
+ # Break out the reference prompt log prob value
276
+ # vector, prompt logprob token id vector, and
277
+ # prompt token rank at the current position.
278
+ (ref_pos_prompt_logprob_toks, ref_pos_prompt_logprob_vals,
279
+ ref_pos_prompt_token_rank) = (ref_prompt_logprob_toks[idx, :],
280
+ ref_prompt_logprob_vals[idx, :],
281
+ ref_prompt_token_ranks[idx])
282
+
283
+ # For each position in the prompt sequence,
284
+ # ensure the actual prompt token is among the
285
+ # logprobs
286
+ assert prompt_token in pos_logprob_dict, (
287
+ f"Prompt token {prompt_token} not"
288
+ f" present in logprob at index {idx}")
289
+ # Validate number of prompt logprobs
290
+ num_plp_toks = len(pos_logprob_dict)
291
+ assert (num_plp_toks == num_prompt_logprobs
292
+ or num_plp_toks == num_prompt_logprobs +
293
+ 1), ("Valid numbers of prompt logprobs are"
294
+ f" {num_prompt_logprobs} or"
295
+ f" {num_prompt_logprobs+1} but"
296
+ f" {num_plp_toks} logprobs found at"
297
+ f" position {idx}. Logprobs dict:"
298
+ f" {pos_logprob_dict}")
299
+
300
+ # Validate prompt token logprob rank
301
+ prmpt_tok_lp = pos_logprob_dict[prompt_token]
302
+ prmpt_tok_lp_rank = prmpt_tok_lp.rank
303
+ ref_prmpt_tok_lp_rank = ref_pos_prompt_token_rank
304
+ assert (ref_prmpt_tok_lp_rank == prmpt_tok_lp_rank), (
305
+ "Prompt token logprob rank"
306
+ f" {prmpt_tok_lp_rank} does not match"
307
+ " correct value"
308
+ f" {ref_prmpt_tok_lp_rank}"
309
+ f" in Logprob {prmpt_tok_lp}")
310
+
311
+ # Validate that the logprob processor yields
312
+ # the correct prompt log probs and valid
313
+ # rankings
314
+ rank_one_appears = False
315
+ for jdx in range(1, len(ref_pos_prompt_logprob_toks)):
316
+ # Iterate over the (logprob val,logprob tok id)
317
+ # pairs expected by the test fixture at this
318
+ # position in the completion.
319
+ ref_plp_val = float(ref_pos_prompt_logprob_vals[jdx])
320
+ ref_tok_id = int(ref_pos_prompt_logprob_toks[jdx])
321
+ assert ref_tok_id in pos_logprob_dict, (
322
+ f"Expected token {ref_tok_id} to be"
323
+ f" in logprob dict but it is not.")
324
+
325
+ # Extract actually-generated logprob
326
+ # info
327
+ plp = pos_logprob_dict[ref_tok_id]
328
+ plp_val = plp.logprob
329
+ plp_rank = plp.rank
330
+
331
+ # A "top" (rank 1) logprob must be
332
+ # present
333
+ rank_one_appears = (True
334
+ if plp_rank == 1 else rank_one_appears)
335
+
336
+ # Rank must be >= 1
337
+ assert plp_rank >= 1, (
338
+ f"Logprob {plp} has invalid"
339
+ f" rank {plp_rank} < 1."
340
+ f" Logprob dict: {pos_logprob_dict}")
341
+
342
+ # Validate log probability
343
+ assert math.isclose(plp_val, ref_plp_val), (
344
+ f"Token id {ref_tok_id} appears in logprobs dict"
345
+ f" at position {idx} in completion with log"
346
+ f" probability {plp_val} but {ref_plp_val} was"
347
+ f" expected. Logprob: {plp}")
348
+
349
+ assert rank_one_appears, (f"No Logprob has rank 1"
350
+ " in the following Logprob"
351
+ f" dict: {pos_logprob_dict}")
352
+
353
+ # Validate prompt logprob detokenization
354
+ for plp_tok in pos_logprob_dict:
355
+ # Confirm that prompt logprob decoded token matches
356
+ # the logprob token id at this sequence position
357
+ decoded_token = pos_logprob_dict[plp_tok].decoded_token
358
+ ref_decoded_token = _ref_convert_id_to_token(
359
+ dtv.tokenizer, plp_tok)
360
+ assert decoded_token == ref_decoded_token, (
361
+ f"Prompt logprob token id {plp_tok} decodes to"
362
+ f" {ref_decoded_token} but Logprob decoded"
363
+ f" token is {decoded_token} instead"
364
+ f" (at position {idx})")
365
+ else:
366
+ # Prompt logprobs disabled for this request
367
+ assert prompt_logprobs is None
368
+
369
+
370
+@pytest.mark.parametrize(
371
+ "request_output_kind",
372
+ [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
373
+@pytest.mark.parametrize("num_sample_logprobs",
374
+ [None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
375
+@pytest.mark.parametrize("num_prompt_logprobs",
376
+ [None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
377
+def test_logprobs_processor(request_output_kind: RequestOutputKind,
378
+ num_sample_logprobs: Optional[int],
379
+ num_prompt_logprobs: Optional[int],
380
+ dummy_test_vectors):
381
+ output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
382
+ log_stats=False)
383
+ engine_core = MockEngineCore(
384
+ tokens_list=dummy_test_vectors.generation_tokens,
385
+ generated_logprobs_raw=None if num_sample_logprobs is None else
386
+ dummy_test_vectors.generation_logprobs,
387
+ prompt_logprobs_raw=None
388
+ if num_prompt_logprobs is None else dummy_test_vectors.prompt_logprobs)
389
+
390
+ # Make N requests.
391
+ request_id_list = [
392
+ f"request-{idx}"
393
+ for idx in range(len(dummy_test_vectors.prompt_strings))
394
+ ]
395
+ requests = [
396
+ EngineCoreRequest(request_id=request_id_list[idx],
397
+ prompt=prompt,
398
+ prompt_token_ids=prompt_tokens,
399
+ arrival_time=0,
400
+ mm_inputs=None,
401
+ mm_hashes=None,
402
+ mm_placeholders=None,
403
+ eos_token_id=None,
404
+ lora_request=None,
405
+ sampling_params=SamplingParams(
406
+ skip_special_tokens=False,
407
+ spaces_between_special_tokens=False,
408
+ output_kind=request_output_kind,
409
+ stop=[],
410
+ include_stop_str_in_output=False,
411
+ logprobs=num_sample_logprobs,
412
+ prompt_logprobs=num_prompt_logprobs,
413
+ )) for idx, (prompt, prompt_tokens) in enumerate(
414
+ zip(dummy_test_vectors.prompt_strings,
415
+ dummy_test_vectors.prompt_tokens))
416
+ ]
417
+
418
+ # Add requests to the detokenizer.
419
+ for request in requests:
420
+ output_processor.add_request(request)
421
+
422
+ gen_tokens = {}
423
+ gen_logprobs = {}
424
+ gen_prompt_logprobs = {}
425
+ gen_cumulative_logprobs = {}
426
+ while True:
427
+ # Mock output from the EngineCore.
428
+ outputs = engine_core.get_outputs()
429
+ if len(outputs) == 0:
430
+ break
431
+
432
+ # Step the logprobs processor.
433
+ processed_outputs = output_processor.process_outputs(outputs)
434
+ request_outputs = processed_outputs.request_outputs
435
+ requests_to_abort = processed_outputs.reqs_to_abort
436
+ assert len(requests_to_abort) == 0
437
+
438
+ # Update tracking.
439
+ for request_output in request_outputs:
440
+ request_id = request_output.request_id
441
+ new_tokens = request_output.outputs[0].token_ids
442
+ prompt_logprobs = request_output.prompt_logprobs
443
+ logprobs = request_output.outputs[0].logprobs
444
+ gen_cumulative_logprobs[request_id] = request_output.outputs[
445
+ 0].cumulative_logprob
446
+ if request_id not in gen_logprobs:
447
+ # Start tracking sample and prompt logprobs for this request
448
+ gen_tokens[request_id] = new_tokens
449
+ gen_logprobs[request_id] = logprobs
450
+ gen_prompt_logprobs[request_id] = prompt_logprobs
451
+ else:
452
+ # Extend logprobs tracker
453
+ gen_tokens[request_id].extend(new_tokens)
454
+ lp = gen_logprobs[request_id]
455
+ plp = gen_prompt_logprobs[request_id]
456
+ if lp:
457
+ lp.extend(logprobs)
458
+ if plp:
459
+ plp.extend(prompt_logprobs)
460
+
461
+ # Confirmed tracked logprobs match what we expect
462
+ _validate_logprobs(gen_tokens, gen_logprobs, gen_prompt_logprobs,
463
+ gen_cumulative_logprobs, dummy_test_vectors,
464
+ request_id_list, num_sample_logprobs,
465
+ num_prompt_logprobs)
466
+
467
+ assert output_processor.get_num_unfinished_requests() == 0
468
+ assert not output_processor.has_unfinished_requests()
469
+
470
+
471
@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
147
-def test_stop_string(include_stop_str_in_output: bool):
148
- output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
149
- engine_core = MockEngineCore(GENERATION_TOKENS)
472
+@pytest.mark.parametrize("num_sample_logprobs",
473
+ [None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
474
+@pytest.mark.parametrize("num_prompt_logprobs",
475
+ [None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
476
+def test_stop_string(include_stop_str_in_output: bool,
477
+ num_sample_logprobs: Optional[int],
478
+ num_prompt_logprobs: Optional[int], dummy_test_vectors):
479
+ output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
480
+ log_stats=False)
481
+ engine_core = MockEngineCore(
482
+ tokens_list=dummy_test_vectors.generation_tokens,
483
+ generated_logprobs_raw=dummy_test_vectors.generation_logprobs
484
+ if num_sample_logprobs else None,
485
+ prompt_logprobs_raw=dummy_test_vectors.prompt_logprobs
486
+ if num_prompt_logprobs else None)
487
488
# Make N requests.
489
+ request_id_list = [
490
+ f"request-{idx}"
491
+ for idx in range(len(dummy_test_vectors.prompt_strings))
492
+ ]
493
requests = [
494
EngineCoreRequest(
154
- request_id=f"request-{idx}",
495
+ request_id=request_id_list[idx],
496
prompt=prompt,
497
prompt_token_ids=prompt_tokens,
498
arrival_time=0,
507
output_kind=RequestOutputKind.DELTA,
508
stop=STOP_STRINGS,
509
include_stop_str_in_output=include_stop_str_in_output,
169
- )) for idx, (
170
- prompt,
171
- prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
510
+ logprobs=num_sample_logprobs,
511
+ prompt_logprobs=num_prompt_logprobs,
512
+ )) for idx, (prompt, prompt_tokens) in enumerate(
513
+ zip(dummy_test_vectors.prompt_strings,
514
+ dummy_test_vectors.prompt_tokens))
515
]
516
517
# Add requests to the detokenizer.
519
output_processor.add_request(request)
520
521
gen_strings = {}
522
+ gen_tokens = {}
523
+ gen_logprobs = {}
524
+ gen_prompt_logprobs = {}
525
+ gen_cumulative_logprobs = {}
526
aborted = []
527
while True:
528
# Mock output from the EngineCore.
546
547
request_id = request_output.request_id
548
new_text = request_output.outputs[0].text
549
+ new_tokens = request_output.outputs[0].token_ids
550
+ prompt_logprobs = request_output.prompt_logprobs
551
+ logprobs = request_output.outputs[0].logprobs
552
+ gen_cumulative_logprobs[request_id] = request_output.outputs[
553
+ 0].cumulative_logprob
554
if request_id not in gen_strings:
555
gen_strings[request_id] = new_text
556
+ gen_tokens[request_id] = new_tokens
557
+ gen_logprobs[request_id] = logprobs
558
+ gen_prompt_logprobs[request_id] = prompt_logprobs
559
else:
560
gen_strings[request_id] += new_text
561
+ gen_tokens[request_id].extend(new_tokens)
562
+ lp = gen_logprobs[request_id]
563
+ plp = gen_prompt_logprobs[request_id]
564
+ if lp:
565
+ lp.extend(logprobs)
566
+ if plp:
567
+ plp.extend(prompt_logprobs)
568
569
# Confirmed tracked values matches what we expected.
208
- for idx, (ref_gen_str,
209
- stop_str) in enumerate(zip(GENERATION_STRINGS, STOP_STRINGS)):
570
+ for idx, (ref_gen_str, stop_str) in enumerate(
571
+ zip(dummy_test_vectors.generation_strings, STOP_STRINGS)):
572
573
# Request should be aborted.
574
request_id = f"request-{idx}"
589
assert gen_str == ref_str_exc_stop, (
590
f"{gen_str=}, {ref_str_exc_stop=}")
591
592
+ # Confirmed tracked logprobs match what we expect
593
+ _validate_logprobs(gen_tokens, gen_logprobs, gen_prompt_logprobs,
594
+ gen_cumulative_logprobs, dummy_test_vectors,
595
+ request_id_list, num_sample_logprobs,
596
+ num_prompt_logprobs)
597
+
598
assert output_processor.get_num_unfinished_requests() == 0
599
assert not output_processor.has_unfinished_requests()
600
601
234
-def test_iteration_stats():
235
- output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=True)
236
- engine_core = MockEngineCore(GENERATION_TOKENS)
602
+def test_iteration_stats(dummy_test_vectors):
603
+ output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
604
+ log_stats=True)
605
+ engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
606
607
# Make N requests.
608
requests = [
617
eos_token_id=None,
618
lora_request=None,
619
sampling_params=SamplingParams(),
251
- ) for idx, (
252
- prompt,
253
- prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
620
+ ) for idx, (prompt, prompt_tokens) in enumerate(
621
+ zip(dummy_test_vectors.prompt_strings,
622
+ dummy_test_vectors.prompt_tokens))
623
]
624
625
# Add all requests except one to the OutputProcessor.
257
- num_active = len(GENERATION_TOKENS) - 1
626
+ num_active = len(dummy_test_vectors.generation_tokens) - 1
627
for request in requests[:num_active]:
628
output_processor.add_request(request)
629
inactive_request = requests[num_active]
632
outputs = engine_core.get_outputs()[:num_active]
633
processed_outputs = output_processor.process_outputs(outputs)
634
iteration_stats = processed_outputs.iteration_stats
266
- total_prompt_tokens = sum(
267
- [len(prompt_tokens) for prompt_tokens in PROMPT_TOKENS[:num_active]])
635
+ total_prompt_tokens = sum([
636
+ len(prompt_tokens)
637
+ for prompt_tokens in dummy_test_vectors.prompt_tokens[:num_active]
638
+ ])
639
640
assert iteration_stats.num_prompt_tokens == total_prompt_tokens
641
assert iteration_stats.num_generation_tokens == num_active
654
outputs = engine_core.get_outputs()[:num_active]
655
processed_outputs = output_processor.process_outputs(outputs)
656
iteration_stats = processed_outputs.iteration_stats
286
- total_prompt_tokens = len(PROMPT_TOKENS[num_active - 1])
657
+ total_prompt_tokens = len(dummy_test_vectors.prompt_tokens[num_active - 1])
658
659
assert iteration_stats.num_prompt_tokens == total_prompt_tokens
660
assert iteration_stats.num_generation_tokens == num_active