A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Core] Adding token ranks along with logprobs (#3516)
Co-authored-by: Swapnil Parekh <swapnilp@ibm.com>
Swapnil Parekh committed
Mar 25, 2024 at 13:13 UTC
819924e74947ba2ac4f2035df88bb4dac79ec6cf
3 files changed
+98
-12
tests/samplers/test_ranks.py
new
+49
@@ -0,0 +1,49 @@
1
+import pytest
2
+from vllm import SamplingParams
3
+
4
+MODELS = ["facebook/opt-125m"]
5
+
6
+
7
+@pytest.mark.parametrize("model", MODELS)
8
+@pytest.mark.parametrize("dtype", ["half"])
9
+def test_ranks(
10
+ vllm_runner,
11
+ model,
12
+ dtype,
13
+ example_prompts,
14
+):
15
+ max_tokens = 5
16
+ num_top_logprobs = 5
17
+ num_prompt_logprobs = 5
18
+
19
+ vllm_model = vllm_runner(model, dtype=dtype, max_logprobs=num_top_logprobs)
20
+
21
+ ## Test greedy logprobs ranks
22
+ vllm_sampling_params = SamplingParams(temperature=0.0,
23
+ top_p=1.0,
24
+ max_tokens=max_tokens,
25
+ logprobs=num_top_logprobs,
26
+ prompt_logprobs=num_prompt_logprobs)
27
+ vllm_results = vllm_model.generate_w_logprobs(example_prompts,
28
+ vllm_sampling_params)
29
+ for result in vllm_results:
30
+ assert result[2] is not None
31
+ assert len(result[2]) == len(result[0])
32
+ # check whether all chosen tokens have ranks = 1
33
+ for token, logprobs in zip(result[0], result[2]):
34
+ assert token in logprobs
35
+ assert logprobs[token].rank == 1
36
+
37
+ ## Test non-greedy logprobs ranks
38
+ sampling_params = SamplingParams(temperature=1.0,
39
+ top_p=1.0,
40
+ max_tokens=max_tokens,
41
+ logprobs=num_top_logprobs,
42
+ prompt_logprobs=num_prompt_logprobs)
43
+ res = vllm_model.generate_w_logprobs(example_prompts, sampling_params)
44
+ for result in res:
45
+ assert result[2] is not None
46
+ assert len(result[2]) == len(result[0])
47
+ # check whether all chosen tokens have ranks
48
+ for token, logprobs in zip(result[0], result[2]):
49
+ assert logprobs[token].rank >= 1
vllm/model_executor/layers/sampler.py
+40
-10
@@ -465,6 +465,24 @@ def _sample(
465
# sampling_tensors)
466
467
468
+def _get_ranks(x: torch.Tensor, indices: List[int]) -> torch.Tensor:
469
+ """
470
+ This function calculates the ranks of the chosen tokens in a logprob tensor.
471
+
472
+ Args:
473
+ x (torch.Tensor): 2D logprob tensor of shape (N, M)
474
+ where N is the no. of tokens and M is the vocab dim.
475
+ indices (List[int]): List of chosen token indices.
476
+
477
+ Returns:
478
+ torch.Tensor: 1D tensor of shape (N,) where N is the no. of tokens.
479
+ Each element in the returned tensor represents the rank
480
+ of the chosen token in the input logprob tensor.
481
+ """
482
+ vals = x[range(len(x)), indices]
483
+ return (x > vals[:, None]).long().sum(1) + 1
484
+
485
+
486
def _get_logprobs(
487
logprobs: torch.Tensor,
488
sampling_metadata: SamplingMetadata,
@@ -520,6 +538,10 @@ def _get_logprobs(
538
539
batched_logprobs_query_result = batched_logprobs_query_result.cpu()
540
541
+ batched_ranks_query_result = _get_ranks(
542
+ logprobs[batched_logprobs_query_seq_indices],
543
+ batched_logprobs_query_token_indices)
544
+
545
# Gather results
546
result_prompt_logprobs: List[Optional[PromptLogprobs]] = []
547
result_sample_logprobs: List[SampleLogprobs] = []
@@ -540,15 +562,20 @@ def _get_logprobs(
562
for token_id in prompt_tokens[1:]:
563
prompt_logprobs_dict = {
564
token_id:
543
- batched_logprobs_query_result[query_result_idx].item()
565
+ (batched_logprobs_query_result[query_result_idx].item(),
566
+ batched_ranks_query_result[query_result_idx].item())
567
}
568
if num_logprobs > 0:
569
prompt_logprobs_dict.update(
547
- zip(top_token_ids[sample_idx, :num_logprobs].tolist(),
548
- top_logprobs[sample_idx, :num_logprobs].tolist()))
570
+ zip(
571
+ top_token_ids[sample_idx, :num_logprobs].tolist(),
572
+ zip(
573
+ top_logprobs[
574
+ sample_idx, :num_logprobs].tolist(),
575
+ range(1, num_logprobs + 1))))
576
group_prompt_logprobs.append({
550
- token_id: Logprob(logprob)
551
- for token_id, logprob in prompt_logprobs_dict.items()
577
+ token_id: Logprob(*logprob_rank)
578
+ for token_id, logprob_rank in prompt_logprobs_dict.items()
579
})
580
sample_idx += 1
581
query_result_idx += 1
@@ -564,7 +591,8 @@ def _get_logprobs(
591
for next_token_id, parent_id in zip(next_token_ids, parent_ids):
592
sample_logprobs_dict = {
593
next_token_id:
567
- batched_logprobs_query_result[query_result_idx].item()
594
+ (batched_logprobs_query_result[query_result_idx].item(),
595
+ batched_ranks_query_result[query_result_idx].item())
596
}
597
query_result_idx += 1
598
if num_logprobs > 0:
@@ -572,11 +600,13 @@ def _get_logprobs(
600
zip(
601
top_token_ids[sample_idx +
602
parent_id, :num_logprobs].tolist(),
575
- top_logprobs[sample_idx +
576
- parent_id, :num_logprobs].tolist()))
603
+ zip(
604
+ top_logprobs[sample_idx +
605
+ parent_id, :num_logprobs].tolist(),
606
+ range(1, num_logprobs + 1))))
607
group_sample_logprobs.append({
578
- token_id: Logprob(logprob)
579
- for token_id, logprob in sample_logprobs_dict.items()
608
+ token_id: Logprob(*logprob_rank)
609
+ for token_id, logprob_rank in sample_logprobs_dict.items()
610
})
611
result_sample_logprobs.append(group_sample_logprobs)
612
sample_idx += len(seq_ids)
vllm/sequence.py
+9
-2
@@ -16,8 +16,15 @@ if TYPE_CHECKING:
16
17
@dataclass
18
class Logprob:
19
- """Infos for supporting OpenAI compatible logprobs."""
19
+ """Infos for supporting OpenAI compatible logprobs and token ranks.
20
+
21
+ Attributes:
22
+ logprob: The logprob of chosen token
23
+ rank: The vocab rank of chosen token (>=1)
24
+ decoded_token: The decoded chosen token index
25
+ """
26
logprob: float
27
+ rank: Optional[int] = None
28
decoded_token: Optional[str] = None
29
30
@@ -66,7 +73,7 @@ class SequenceStatus(enum.Enum):
73
class RequestMetrics:
74
"""Metrics associated with a request.
75
69
- Args:
76
+ Attributes:
77
arrival_time: The time when the request arrived.
78
first_scheduled_time: The time when the request was first scheduled.
79
first_token_time: The time when the first token was generated.