@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc0eaf12b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] spec decode handle None entries in topk args in create_sequence_group_output (#7232)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Aug 22, 2024 at 07:33 UTC cc0eaf12b1a94bc2fd8d497f6615202699fcf7da
2 files changed +84 -7
tests/spec_decode/e2e/test_logprobs.py
+75
@@ -343,3 +343,78 @@ def run_greedy_logprobs_correctness_test(baseline_llm_generator,
343 b=baseline_rank_to_logprob[rank],
344 abs_tol=1e-1,
345 )
346 +
347 +
348 +@pytest.mark.parametrize(
349 + "common_llm_kwargs",
350 + [{
351 + "model": "JackFram/llama-160m",
352 + # Skip cuda graph recording for fast test.
353 + "enforce_eager": True,
354 + # Required for spec decode.
355 + "use_v2_block_manager": True,
356 + "max_logprobs": 6,
357 + }])
358 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
359 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
360 +@pytest.mark.parametrize("test_llm_kwargs",
361 + [{
362 + "speculative_model": "JackFram/llama-68m",
363 + "num_speculative_tokens": 3,
364 + "disable_logprobs_during_spec_decoding": True,
365 + }])
366 +@pytest.mark.parametrize("seed", [1])
367 +def test_logprobs_disabled(baseline_llm_generator, test_llm_generator):
368 + """Check the behavior when logprobs are disabled.
369 + Token choices should match with the base model.
370 + """
371 + prompts = [
372 + "Hello, my name is",
373 + "The president of the United States is",
374 + "The capital of France is",
375 + "The future of AI is",
376 + "San Francisco is know for its",
377 + "Facebook was created in 2004 by",
378 + "Curious George is a",
379 + "Python 3.11 brings improvements to its",
380 + ]
381 +
382 + prompts = [prompt for prompt, _ in zip(cycle(prompts), range(4))]
383 +
384 + sampling_params = SamplingParams(
385 + # Use smaller output len for fast test
386 + max_tokens=7,
387 + ignore_eos=True,
388 + temperature=0.0,
389 + logprobs=2,
390 + )
391 +
392 + spec_batch_logprobs = get_logprobs_from_llm_generator(
393 + test_llm_generator, prompts, sampling_params)
394 + baseline_batch_logprobs = get_logprobs_from_llm_generator(
395 + baseline_llm_generator, prompts, sampling_params)
396 +
397 + assert len(baseline_batch_logprobs) == len(prompts)
398 + assert len(spec_batch_logprobs) == len(prompts)
399 +
400 + # For each sequence in the batch.
401 + for _, (baseline_logprobs, spec_logprobs) in enumerate(
402 + zip(baseline_batch_logprobs, spec_batch_logprobs)):
403 + assert len(spec_logprobs) == len(baseline_logprobs)
404 +
405 + # For each generated position of the sequence.
406 + for _, (spec_pos_logprobs, baseline_pos_logprobs) in enumerate(
407 + zip(spec_logprobs, baseline_logprobs)):
408 +
409 + assert len(spec_pos_logprobs) == 1
410 + spec_top_token_id = list(spec_pos_logprobs)[0]
411 +
412 + spec_top_logprob = spec_pos_logprobs[spec_top_token_id]
413 + assert spec_top_logprob.logprob == 0.0
414 + assert spec_top_logprob.rank == -1
415 +
416 + # check that the chosen token matches the base model
417 + baseline_logprob = baseline_pos_logprobs[spec_top_token_id]
418 + assert baseline_logprob.rank == 1
419 + assert spec_top_logprob.decoded_token \
420 + == baseline_logprob.decoded_token
vllm/spec_decode/util.py
+9 -7
@@ -64,23 +64,25 @@ def create_sequence_group_output(
64 token_id_logprob_rank (int): The logprob rank of the sampled token.
65 token_id_logprob (float): The logprob value of the sampled token.
66 seq_id (int): The sequence id.
67 - topk_token_ids (List[int]): The list of top-k token ids.
68 - topk_logprobs (List[float]): The list of top-k logprobs.
67 + topk_token_ids (List[Optional[int]]): The list of top-k token ids.
68 + topk_logprobs (List[Optional[float]]): The list of top-k logprobs.
69 """
70 # vLLM logprobs always include the sampled token. In addition, the user may
71 # request topk-logprobs (where top-k varies per user up to max_logprobs).
72 - logprobs: Dict[Optional[int], Logprob] = {
72 + logprobs: Dict[int, Logprob] = {
73 token_id: Logprob(
74 logprob=token_id_logprob,
75 rank=token_id_logprob_rank,
76 ),
77 }
78 logprobs.update({
79 - topk_token_ids[topk_logprob_index]: Logprob(
80 - logprob=topk_logprobs[topk_logprob_index],
81 - rank=topk_logprob_index + 1,
79 + topk_token_id: Logprob(
80 + logprob=topk_logprob if topk_logprob is not None else 0.0,
81 + rank=topk_index + 1,
82 )
83 - for topk_logprob_index, _ in enumerate(topk_token_ids)
83 + for topk_index, (topk_token_id, topk_logprob) \
84 + in enumerate(zip(topk_token_ids, topk_logprobs)) \
85 + if topk_token_id is not None
86 })
87
88 return CompletionSequenceGroupOutput(