@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 819924e749
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Core] Adding token ranks along with logprobs (#3516)

Co-authored-by: Swapnil Parekh <swapnilp@ibm.com>

Swapnil Parekh committed Mar 25, 2024 at 13:13 UTC 819924e74947ba2ac4f2035df88bb4dac79ec6cf
3 files changed +98 -12
tests/samplers/test_ranks.py new
+49
@@ -0,0 +1,49 @@
1 +import pytest
2 +from vllm import SamplingParams
3 +
4 +MODELS = ["facebook/opt-125m"]
5 +
6 +
7 +@pytest.mark.parametrize("model", MODELS)
8 +@pytest.mark.parametrize("dtype", ["half"])
9 +def test_ranks(
10 + vllm_runner,
11 + model,
12 + dtype,
13 + example_prompts,
14 +):
15 + max_tokens = 5
16 + num_top_logprobs = 5
17 + num_prompt_logprobs = 5
18 +
19 + vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
20 +
21 + ## Test greedy logprobs ranks
22 + vllm_sampling_params = SamplingParams(temperature=0.0,
23 + top_p=1.0,
24 + max_tokens=max_tokens,
25 + logprobs=num_top_logprobs,
26 + prompt_logprobs=num_prompt_logprobs)
27 + vllm_results = vllm_model.generate_w_logprobs(example_prompts,
28 + vllm_sampling_params)
29 + for result in vllm_results:
30 + assert result[2] is not None
31 + assert len(result[2]) == len(result[0])
32 + # check whether all chosen tokens have ranks = 1
33 + for token, logprobs in zip(result[0], result[2]):
34 + assert token in logprobs
35 + assert logprobs[token].rank == 1
36 +
37 + ## Test non-greedy logprobs ranks
38 + sampling_params = SamplingParams(temperature=1.0,
39 + top_p=1.0,
40 + max_tokens=max_tokens,
41 + logprobs=num_top_logprobs,
42 + prompt_logprobs=num_prompt_logprobs)
43 + res = vllm_model.generate_w_logprobs(example_prompts, sampling_params)
44 + for result in res:
45 + assert result[2] is not None
46 + assert len(result[2]) == len(result[0])
47 + # check whether all chosen tokens have ranks
48 + for token, logprobs in zip(result[0], result[2]):
49 + assert logprobs[token].rank >= 1
vllm/model_executor/layers/sampler.py
+40 -10
@@ -465,6 +465,24 @@ def _sample(
465 # sampling_tensors)
466
467
468 +def _get_ranks(x: torch.Tensor, indices: List[int]) -> torch.Tensor:
469 + """
470 + This function calculates the ranks of the chosen tokens in a logprob tensor.
471 +
472 + Args:
473 + x (torch.Tensor): 2D logprob tensor of shape (N, M)
474 + where N is the no. of tokens and M is the vocab dim.
475 + indices (List[int]): List of chosen token indices.
476 +
477 + Returns:
478 + torch.Tensor: 1D tensor of shape (N,) where N is the no. of tokens.
479 + Each element in the returned tensor represents the rank
480 + of the chosen token in the input logprob tensor.
481 + """
482 + vals = x[range(len(x)), indices]
483 + return (x > vals[:, None]).long().sum(1) + 1
484 +
485 +
486 def _get_logprobs(
487 logprobs: torch.Tensor,
488 sampling_metadata: SamplingMetadata,
@@ -520,6 +538,10 @@ def _get_logprobs(
538
539 batched_logprobs_query_result = batched_logprobs_query_result.cpu()
540
541 + batched_ranks_query_result = _get_ranks(
542 + logprobs[batched_logprobs_query_seq_indices],
543 + batched_logprobs_query_token_indices)
544 +
545 # Gather results
546 result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
547 result_sample_logprobs: List[SampleLogprobs] = []
@@ -540,15 +562,20 @@ def _get_logprobs(
562 for token_id in prompt_tokens[1:]:
563 prompt_logprobs_dict = {
564 token_id:
543 - batched_logprobs_query_result[query_result_idx].item()
565 + (batched_logprobs_query_result[query_result_idx].item(),
566 + batched_ranks_query_result[query_result_idx].item())
567 }
568 if num_logprobs > 0:
569 prompt_logprobs_dict.update(
547 - zip(top_token_ids[sample_idx, :num_logprobs].tolist(),
548 - top_logprobs[sample_idx, :num_logprobs].tolist()))
570 + zip(
571 + top_token_ids[sample_idx, :num_logprobs].tolist(),
572 + zip(
573 + top_logprobs[
574 + sample_idx, :num_logprobs].tolist(),
575 + range(1, num_logprobs + 1))))
576 group_prompt_logprobs.append({
550 - token_id: Logprob(logprob)
551 - for token_id, logprob in prompt_logprobs_dict.items()
577 + token_id: Logprob(*logprob_rank)
578 + for token_id, logprob_rank in prompt_logprobs_dict.items()
579 })
580 sample_idx += 1
581 query_result_idx += 1
@@ -564,7 +591,8 @@ def _get_logprobs(
591 for next_token_id, parent_id in zip(next_token_ids, parent_ids):
592 sample_logprobs_dict = {
593 next_token_id:
567 - batched_logprobs_query_result[query_result_idx].item()
594 + (batched_logprobs_query_result[query_result_idx].item(),
595 + batched_ranks_query_result[query_result_idx].item())
596 }
597 query_result_idx += 1
598 if num_logprobs > 0:
@@ -572,11 +600,13 @@ def _get_logprobs(
600 zip(
601 top_token_ids[sample_idx +
602 parent_id, :num_logprobs].tolist(),
575 - top_logprobs[sample_idx +
576 - parent_id, :num_logprobs].tolist()))
603 + zip(
604 + top_logprobs[sample_idx +
605 + parent_id, :num_logprobs].tolist(),
606 + range(1, num_logprobs + 1))))
607 group_sample_logprobs.append({
578 - token_id: Logprob(logprob)
579 - for token_id, logprob in sample_logprobs_dict.items()
608 + token_id: Logprob(*logprob_rank)
609 + for token_id, logprob_rank in sample_logprobs_dict.items()
610 })
611 result_sample_logprobs.append(group_sample_logprobs)
612 sample_idx += len(seq_ids)
vllm/sequence.py
+9 -2
@@ -16,8 +16,15 @@ if TYPE_CHECKING:
16
17 @dataclass
18 class Logprob:
19 - """Infos for supporting OpenAI compatible logprobs."""
19 + """Infos for supporting OpenAI compatible logprobs and token ranks.
20 +
21 + Attributes:
22 + logprob: The logprob of chosen token
23 + rank: The vocab rank of chosen token (>=1)
24 + decoded_token: The decoded chosen token index
25 + """
26 logprob: float
27 + rank: Optional[int] = None
28 decoded_token: Optional[str] = None
29
30
@@ -66,7 +73,7 @@ class SequenceStatus(enum.Enum):
73 class RequestMetrics:
74 """Metrics associated with a request.
75
69 - Args:
76 + Attributes:
77 arrival_time: The time when the request arrived.
78 first_scheduled_time: The time when the request was first scheduled.
79 first_token_time: The time when the first token was generated.