@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0630d4537a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1] Logprobs and prompt logprobs support (#9880)

This PR is adding support for sample logprobs & prompt logprobs to vLLM v1. New behavior: - During model execution, model runner computes sample logprobs (if user-provided logprobs setting is not None) and prompt logprobs (if user-provided prompt_logprobs setting is not None). For both sample and prompt logprobs, the engine core returns 3 vectors: token ids, token logprob values, token ranks. Ranks reflect tokens' 1-indexed positions in the vocabulary vector after sorting the vocabulary by log probability in descending order. - In scheduler.update_from_output(), sample and prompt logprobs are incorporated into the EngineCoreOutput data structure which is transferred to the engine client. If multiprocessing is enabled, then sample and prompt logprobs will be (de)serialized when the EngineCoreOutput data structure is (de)serialized. - During output processing, the LogprobsProcessor transforms the triplet of token ids, token logprobs values, and token ranks into the OpenAI-compatible List[Dict[token id,Logprob]] format (for sample and prompt logprobs respectively.) - Each Logprob instance (whether sample- or prompt-) consists of a token's log-probability, rank, and detokenized string representation. Note that logprob detokenization is handled by the LogprobsProcessor not the detokenizer. Signed-off-by: Andrew Feldman <afeldman@neuralmagic.com> Signed-off-by: Nick Hill <nhill@redhat.com> Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Nick Hill <nhill@redhat.com>

afeldman-nm committed Feb 7, 2025 at 10:26 UTC 0630d4537a5fbab80cb1109a26170101cffb7f84
30 files changed +2869 -287
tests/v1/core/test_scheduler.py
+2 -2
@@ -195,8 +195,8 @@ def test_schedule_partial_requests():
195 req_ids=[request.request_id for request in requests],
196 req_id_to_index=req_to_index,
197 sampled_token_ids=[0] * len(requests),
198 - logprob_token_ids_cpu=None,
199 - logprobs_cpu=None,
198 + logprobs=None,
199 + prompt_logprobs_dict={},
200 )
201 scheduler.update_from_output(output, model_runner_output)
202
tests/v1/engine/conftest.py new
+90
@@ -0,0 +1,90 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from typing import List, Tuple
4 +
5 +import pytest
6 +import torch
7 +from transformers import AutoTokenizer
8 +
9 +from tests.v1.engine.utils import (NUM_PROMPT_LOGPROBS_UNDER_TEST,
10 + NUM_SAMPLE_LOGPROBS_UNDER_TEST, PROMPT_LEN,
11 + TOKENIZER_NAME,
12 + DummyOutputProcessorTestVectors,
13 + generate_dummy_prompt_logprobs_tensors,
14 + generate_dummy_sample_logprobs)
15 +from vllm.engine.arg_utils import EngineArgs
16 +from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
17 +
18 +from tests.v1.engine.utils import FULL_STRINGS # isort: skip
19 +
20 +EngineCoreSampleLogprobsType = List[Tuple[torch.Tensor, torch.Tensor]]
21 +EngineCorePromptLogprobsType = Tuple[torch.Tensor, torch.Tensor]
22 +
23 +
24 +def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
25 + """Generate output processor dummy test vectors, without logprobs
26 +
27 + Returns:
28 + DummyOutputProcessorTestVectors instance with no logprobs
29 + """
30 +
31 + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
32 + vllm_config = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
33 + # Tokenize prompts under test & create dummy generated tokens
34 + prompt_tokens = [
35 + tokenizer(text).input_ids[:PROMPT_LEN] for text in FULL_STRINGS
36 + ]
37 + generation_tokens = [
38 + tokenizer(text).input_ids[PROMPT_LEN:] for text in FULL_STRINGS
39 + ]
40 + # Generate prompt strings
41 + prompt_strings = [
42 + tokenizer.decode(prompt_tokens, skip_special_tokens=True)
43 + for prompt_tokens in prompt_tokens
44 + ]
45 + prompt_strings_len = [
46 + len(prompt_string) for prompt_string in prompt_strings
47 + ]
48 + return DummyOutputProcessorTestVectors(
49 + tokenizer=tokenizer,
50 + tokenizer_group=init_tokenizer_from_configs(
51 + vllm_config.model_config, vllm_config.scheduler_config,
52 + vllm_config.parallel_config, vllm_config.lora_config),
53 + vllm_config=vllm_config,
54 + full_tokens=[tokenizer(text).input_ids for text in FULL_STRINGS],
55 + prompt_tokens=prompt_tokens,
56 + generation_tokens=generation_tokens,
57 + prompt_strings=prompt_strings,
58 + prompt_strings_len=prompt_strings_len,
59 + generation_strings=[
60 + text[prompt_len:]
61 + for text, prompt_len in zip(FULL_STRINGS, prompt_strings_len)
62 + ],
63 + prompt_logprobs=[],
64 + generation_logprobs=[])
65 +
66 +
67 +@pytest.fixture
68 +def dummy_test_vectors() -> DummyOutputProcessorTestVectors:
69 + """Generate output processor dummy test vectors, with logprobs
70 +
71 + Returns:
72 + DummyOutputProcessorTestVectors instance with logprobs
73 + """
74 + # Build dummy test vectors without logprobs
75 + dtv = _build_test_vectors_no_logprobs()
76 + # Inject logprobs into dummy test vectors
77 + # data structure
78 + dtv.generation_logprobs = [
79 + generate_dummy_sample_logprobs(
80 + sampled_tokens_list=tokens_list,
81 + num_logprobs=NUM_SAMPLE_LOGPROBS_UNDER_TEST,
82 + tokenizer=dtv.tokenizer) for tokens_list in dtv.generation_tokens
83 + ]
84 + dtv.prompt_logprobs = [
85 + generate_dummy_prompt_logprobs_tensors(
86 + prompt_tokens_list=tokens_list,
87 + num_logprobs=NUM_PROMPT_LOGPROBS_UNDER_TEST,
88 + tokenizer=dtv.tokenizer) for tokens_list in dtv.prompt_tokens
89 + ]
90 + return dtv
tests/v1/engine/test_async_llm.py
+45 -4
@@ -2,10 +2,11 @@
2
3 import asyncio
4 from contextlib import ExitStack
5 -from typing import List, Tuple
5 +from typing import List, Optional, Tuple
6
7 import pytest
8
9 +from tests.v1.engine.utils import PLP_APC_UNSUPPORTED_MSG
10 from vllm import SamplingParams
11 from vllm.engine.arg_utils import AsyncEngineArgs
12 from vllm.platforms import current_platform
@@ -21,13 +22,19 @@ ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
22 disable_log_requests=True)
23
24
24 -async def generate(engine: AsyncLLM, request_id: str,
25 +async def generate(engine: AsyncLLM,
26 + request_id: str,
27 output_kind: RequestOutputKind,
26 - max_tokens: int) -> Tuple[int, str]:
28 + max_tokens: int,
29 + prompt_logprobs: Optional[int] = None) -> Tuple[int, str]:
30 + # Ensure generate doesn't complete too fast for cancellation test.
31 + await asyncio.sleep(0.2)
32 +
33 count = 0
34 sampling_params = SamplingParams(max_tokens=max_tokens,
35 output_kind=output_kind,
30 - temperature=0)
36 + temperature=0,
37 + prompt_logprobs=prompt_logprobs)
38 async for out in engine.generate(request_id=request_id,
39 prompt="Hello my name is Robert and",
40 sampling_params=sampling_params):
@@ -43,6 +50,40 @@ async def generate(engine: AsyncLLM, request_id: str,
50 return count, request_id
51
52
53 +@pytest.mark.parametrize(
54 + "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
55 +@pytest.mark.asyncio
56 +async def test_async_llm_refuses_prompt_logprobs_with_apc(
57 + monkeypatch, output_kind: RequestOutputKind):
58 + """Test passes if AsyncLLM raises an exception when it is configured
59 + for automatic prefix caching and it receives a request with
60 + prompt_logprobs enabled, which is incompatible."""
61 + # TODO(rickyx): Remove monkeypatch VLLM_USE_V1 setting once we have a
62 + # better way to test V1 so that in the future when we switch, we don't
63 + # have to change all the tests.
64 + monkeypatch.setenv("VLLM_USE_V1", "1")
65 + # Create AsyncLLM engine with APC
66 + apc_engine_args = AsyncEngineArgs(model="facebook/opt-125m",
67 + enable_prefix_caching=True,
68 + gpu_memory_utilization=0.8,
69 + disable_log_requests=True)
70 + engine = AsyncLLM.from_engine_args(apc_engine_args)
71 + try:
72 + with pytest.raises(ValueError) as excinfo:
73 + # Issue a request with prompt logprobs enabled, which should fail
74 + await asyncio.create_task(
75 + generate(engine,
76 + "request-0",
77 + output_kind,
78 + 10,
79 + prompt_logprobs=5))
80 + # Validate exception string is correct
81 + assert str(excinfo.value) == PLP_APC_UNSUPPORTED_MSG
82 + finally:
83 + # Shut down engine
84 + engine.shutdown()
85 +
86 +
87 @pytest.mark.parametrize(
88 "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
89 @pytest.mark.asyncio
tests/v1/engine/test_llm_engine.py new
+23
@@ -0,0 +1,23 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import pytest
4 +
5 +from tests.v1.engine.utils import PLP_APC_UNSUPPORTED_MSG
6 +from vllm import LLM, SamplingParams
7 +
8 +
9 +def test_llm_engine_refuses_prompt_logprobs_with_apc(monkeypatch):
10 + """Test passes if LLMEngine raises an exception when it is configured
11 + for automatic prefix caching and it receives a request with
12 + prompt_logprobs enabled, which is incompatible."""
13 +
14 + monkeypatch.setenv("VLLM_USE_V1", "1")
15 + # TODO(nick): Single-proc to work around a ZMQ shutdown hang for now.
16 + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
17 + with pytest.raises(ValueError) as excinfo:
18 + LLM(model="facebook/opt-125m", enable_prefix_caching=True).generate(
19 + "Hello, my name is",
20 + SamplingParams(temperature=0.8, top_p=0.95, prompt_logprobs=5))
21 +
22 + # Validate exception string is correct
23 + assert str(excinfo.value) == PLP_APC_UNSUPPORTED_MSG
tests/v1/engine/test_output_processor.py
+462 -91
@@ -1,82 +1,47 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from typing import List
3 +import math
4 +from typing import Dict, List, Optional
5
6 import pytest
6 -from transformers import AutoTokenizer
7
8 -from vllm.engine.arg_utils import EngineArgs
8 +from tests.v1.engine.utils import (NUM_PROMPT_LOGPROBS_UNDER_TEST,
9 + NUM_SAMPLE_LOGPROBS_UNDER_TEST,
10 + STOP_STRINGS,
11 + DummyOutputProcessorTestVectors,
12 + MockEngineCore)
13 from vllm.sampling_params import RequestOutputKind, SamplingParams
10 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
11 -from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
14 +from vllm.sequence import PromptLogprobs, SampleLogprobs
15 +from vllm.transformers_utils.tokenizer import AnyTokenizer
16 +from vllm.v1.engine import EngineCoreRequest
17 from vllm.v1.engine.output_processor import OutputProcessor
18
14 -TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
15 -VLLM_CONFIG = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
16 -TOKENIZER_GROUP = init_tokenizer_from_configs(VLLM_CONFIG.model_config,
17 - VLLM_CONFIG.scheduler_config,
18 - VLLM_CONFIG.parallel_config,
19 - VLLM_CONFIG.lora_config)
20 -tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
21 -
22 -FULL_STRINGS = [
23 - "My name is Robert from Neural Magic and I love working on vLLM so much!",
24 - "Red Hat is the best open source company by far across Linux, K8s, and AI.",
25 - "Nick is the name of my brother in addition to my colleague from Red Hat.",
26 -]
27 -
28 -STOP_STRINGS = ["I love working on", "company by far", "brother in"]
29 -
30 -FULL_TOKENS = [tokenizer(text).input_ids for text in FULL_STRINGS]
31 -PROMPT_LEN = 5
32 -PROMPT_TOKENS = [
33 - tokenizer(text).input_ids[:PROMPT_LEN] for text in FULL_STRINGS
34 -]
35 -GENERATION_TOKENS = [
36 - tokenizer(text).input_ids[PROMPT_LEN:] for text in FULL_STRINGS
37 -]
38 -PROMPT_STRINGS = [
39 - tokenizer.decode(prompt_tokens, skip_special_tokens=True)
40 - for prompt_tokens in PROMPT_TOKENS
41 -]
42 -PROMPT_STRINGS_LEN = [len(prompt_string) for prompt_string in PROMPT_STRINGS]
43 -GENERATION_STRINGS = [
44 - text[prompt_len:]
45 - for text, prompt_len in zip(FULL_STRINGS, PROMPT_STRINGS_LEN)
46 -]
47 -
48 -
49 -class MockEngineCore:
50 - """Mock outputs form premade tokens lists."""
51 -
52 - def __init__(self, tokens_list: List[List[int]]):
53 - self.tokens_list = tokens_list
54 - self.current_idx = 0
55 -
56 - def get_outputs(self) -> List[EngineCoreOutput]:
57 - token_idx = self.current_idx
58 - self.current_idx += 1
59 -
60 - outputs = []
61 - for req_idx, token_ids in enumerate(self.tokens_list):
62 - if len(token_ids) > token_idx:
63 - output = EngineCoreOutput(request_id=f"request-{req_idx}",
64 - new_token_ids=[token_ids[token_idx]],
65 - finished=False)
66 - if token_idx == len(token_ids) - 1:
67 - output.finished = True
68 - output.finish_reason = "stopped"
69 - outputs.append(output)
70 -
71 - return outputs
19 +
20 +def _ref_convert_id_to_token(
21 + tokenizer: AnyTokenizer,
22 + token_id: int,
23 +) -> str:
24 + """Reference impl of logprobs detokenization.
25 +
26 + Args:
27 + tokenizer: tokenizer used by the model under test
28 + token_id: convert this token id
29 +
30 + Returns:
31 + String representation of input token id
32 + """
33 + return tokenizer.convert_ids_to_tokens(token_id) or ""
34
35
36 @pytest.mark.parametrize(
37 "request_output_kind",
38 [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
77 -def test_incremental_detokenization(request_output_kind: RequestOutputKind):
78 - output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
79 - engine_core = MockEngineCore(GENERATION_TOKENS)
39 +def test_incremental_detokenization(request_output_kind: RequestOutputKind,
40 + dummy_test_vectors):
41 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
42 + log_stats=False)
43 + engine_core = MockEngineCore(
44 + tokens_list=dummy_test_vectors.generation_tokens)
45
46 # Make N requests.
47 requests = [
@@ -94,10 +59,10 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
59 spaces_between_special_tokens=False,
60 output_kind=request_output_kind,
61 stop=[],
97 - include_stop_str_in_output=False))
98 - for idx, (
99 - prompt,
100 - prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
62 + include_stop_str_in_output=False,
63 + )) for idx, (prompt, prompt_tokens) in enumerate(
64 + zip(dummy_test_vectors.prompt_strings,
65 + dummy_test_vectors.prompt_tokens))
66 ]
67
68 # Add requests to the detokenizer.
@@ -113,7 +78,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
78 break
79
80 # Step the Detokenizer.
116 - processed_outputs = output_processor.process_outputs(outputs, )
81 + processed_outputs = output_processor.process_outputs(outputs)
82 request_outputs = processed_outputs.request_outputs
83 requests_to_abort = processed_outputs.reqs_to_abort
84 assert len(requests_to_abort) == 0
@@ -132,7 +97,8 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
97
98 # Confirmed tracked values matches what we expected.
99 for idx, (ref_gen_str, ref_gen_toks) in enumerate(
135 - zip(GENERATION_STRINGS, GENERATION_TOKENS)):
100 + zip(dummy_test_vectors.generation_strings,
101 + dummy_test_vectors.generation_tokens)):
102 gen_str = gen_strings[f"request-{idx}"]
103 gen_toks = gen_tokens[f"request-{idx}"]
104
@@ -143,15 +109,390 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
109 assert not output_processor.has_unfinished_requests()
110
111
112 +def _validate_logprobs(
113 + gen_tokens: Dict[str, List[int]],
114 + gen_logprobs: Dict[str, Optional[SampleLogprobs]],
115 + gen_prompt_logprobs: Dict[str, Optional[PromptLogprobs]],
116 + gen_cumulative_logprob: Dict[str, float],
117 + dtv: DummyOutputProcessorTestVectors,
118 + request_id_list: List[str],
119 + num_sample_logprobs: Optional[int],
120 + num_prompt_logprobs: Optional[int],
121 +) -> None:
122 + for req_idx, req_id in enumerate(request_id_list):
123 + new_tokens = gen_tokens[req_id]
124 + logprobs = gen_logprobs[req_id]
125 + prompt_logprobs = gen_prompt_logprobs[req_id]
126 + cumulative_logprob = gen_cumulative_logprob[req_id]
127 + prompt_token_ids = dtv.prompt_tokens[req_idx]
128 + ref_logprobs = dtv.generation_logprobs[req_idx]
129 + ref_prompt_logprobs = dtv.prompt_logprobs[req_idx]
130 + if num_sample_logprobs is not None:
131 + # Validate sample logprobs
132 + assert logprobs is not None, (f"Request {req_id} requires sample"
133 + " logprobs but sample logprobs are"
134 + " None.")
135 + # Require num sampled tokens to match num
136 + # sampled logprobs - especially important
137 + # to check since the detokenizer can cause
138 + # a request to finish early due to a stop
139 + # string being hit
140 + num_new_tokens = len(new_tokens)
141 + len_sample_logprobs = len(logprobs)
142 + assert num_new_tokens == len_sample_logprobs, (
143 + f"Request {req_id} has {num_new_tokens}"
144 + " completion tokens but has"
145 + f" {len_sample_logprobs} sample logprobs.")
146 + ref_cumulative_logprob = 0.0
147 + for idx, (sampled_token,
148 + pos_logprob_dict) in enumerate(zip(new_tokens,
149 + logprobs)):
150 + # Break out the reference log probability value &
151 + # logprob token id tensors associated with this
152 + # position in the completion. Also break out the
153 + # sampled token ranks
154 + (ref_pos_logprob_toks, ref_pos_logprob_vals,
155 + ref_sampled_token_rank) = ref_logprobs[idx]
156 + # For each position in the completion sequence,
157 + # ensure the actual sampled token is among the
158 + # logprobs
159 + assert sampled_token in pos_logprob_dict, (
160 + f"Sampled token {sampled_token} not"
161 + f" present in logprob at index {idx}")
162 +
163 + # Validate number of sample logprobs
164 + num_lp_toks = len(pos_logprob_dict)
165 + assert (num_lp_toks == num_sample_logprobs
166 + or num_lp_toks == num_sample_logprobs +
167 + 1), ("Valid numbers of sample logprobs are"
168 + f" {num_sample_logprobs} or"
169 + f" {num_sample_logprobs+1} but"
170 + f" {num_lp_toks} logprobs found at"
171 + f" position {idx}. Logprobs dict:"
172 + f" {pos_logprob_dict}")
173 +
174 + # Validate sampled token logprob rank
175 + smp_lp = pos_logprob_dict[sampled_token]
176 + smp_lp_rank = smp_lp.rank
177 + assert (ref_sampled_token_rank == smp_lp_rank), (
178 + "Sampled token logprob rank"
179 + f" {smp_lp_rank} does not match"
180 + " correct value"
181 + f" {ref_sampled_token_rank}"
182 + f" in Logprob {smp_lp}")
183 +
184 + # Validate that the logprob processor yields
185 + # the correct log probabilities and valid
186 + # rankings
187 + rank_one_appears = False
188 + for jdx in range(1, len(ref_pos_logprob_toks)):
189 + # Iterate over the (logprob val,logprob tok id)
190 + # pairs expected by the test fixture at this
191 + # position in the completion.
192 + ref_lp_val = ref_pos_logprob_vals[jdx]
193 + ref_tok_id = ref_pos_logprob_toks[jdx]
194 + assert ref_tok_id in pos_logprob_dict, (
195 + f"Expected token {ref_tok_id} to be"
196 + f" in logprob dict but it is not.")
197 +
198 + # Extract actually-generated logprob
199 + # info
200 + lp = pos_logprob_dict[ref_tok_id]
201 + lp_val = lp.logprob
202 + lp_rank = lp.rank
203 +
204 + # A "top" (rank 1) logprob must be
205 + # present
206 + rank_one_appears = (True
207 + if lp_rank == 1 else rank_one_appears)
208 +
209 + # Rank must be >= 1
210 + assert lp_rank >= 1, (f"Logprob {lp} has invalid"
211 + f" rank {lp_rank} < 1."
212 + f" Logprob dict: {pos_logprob_dict}")
213 +
214 + # Validate log probability
215 + assert math.isclose(lp_val, ref_lp_val), (
216 + f"Token id {ref_tok_id} appears in logprobs dict"
217 + f" at position {idx} in completion with log"
218 + f" probability {lp_val} but {ref_lp_val} was"
219 + f" expected. Logprob: {lp}")
220 +
221 + assert rank_one_appears, (f"No Logprob has rank 1"
222 + " in the following Logprob"
223 + f" dict: {pos_logprob_dict}")
224 +
225 + # Validate logprobs detokenization
226 + for lp_tok in pos_logprob_dict:
227 + # Confirm that sample logprob decoded token matches
228 + # the logprob token id at this sequence position
229 + decoded_token = pos_logprob_dict[lp_tok].decoded_token
230 + ref_decoded_token = _ref_convert_id_to_token(
231 + dtv.tokenizer, lp_tok)
232 + assert decoded_token == ref_decoded_token, (
233 + f"Sampled logprob token id {lp_tok} decodes to"
234 + f" {ref_decoded_token} but Logprob decoded"
235 + f" token is {decoded_token} instead"
236 + f" (at position {idx})")
237 +
238 + ref_cumulative_logprob += pos_logprob_dict[
239 + sampled_token].logprob
240 + # Assert that cumulative logprobs are correct
241 + assert math.isclose(cumulative_logprob, ref_cumulative_logprob)
242 + else:
243 + # Sample logprobs disabled for this request
244 + assert logprobs is None
245 + assert cumulative_logprob is None
246 +
247 + if num_prompt_logprobs is not None:
248 + # Validate prompt logprobs
249 + assert prompt_logprobs is not None, (
250 + f"Request {req_id} requires prompt"
251 + " logprobs but prompt logprobs are"
252 + " None.")
253 + # Require num prompt tokens to match num
254 + # prompt logprobs
255 + num_prompt_tokens = len(prompt_token_ids)
256 + len_prompt_logprobs = len(prompt_logprobs)
257 + assert num_prompt_tokens == len_prompt_logprobs, (
258 + f"Request {req_id} has {num_prompt_tokens}"
259 + " prompt tokens but has"
260 + f" {len_prompt_logprobs} prompt logprobs.")
261 + # First prompt logprob is None
262 + first_plp_dict = prompt_logprobs[0]
263 + assert first_plp_dict is None, (
264 + f"Request {req_id} first prompt logprob"
265 + f" should be None but has following value"
266 + f" instead: {first_plp_dict}")
267 + # Break out the reference prompt log prob value &
268 + # logprob token id matrices for the whole prompt.
269 + # Also break out the prompt token rank vector
270 + (ref_prompt_logprob_toks, ref_prompt_logprob_vals,
271 + ref_prompt_token_ranks) = ref_prompt_logprobs
272 + for idx, (prompt_token, pos_logprob_dict) in enumerate(
273 + zip(prompt_token_ids[1:], prompt_logprobs[1:])):
274 +
275 + # Break out the reference prompt log prob value
276 + # vector, prompt logprob token id vector, and
277 + # prompt token rank at the current position.
278 + (ref_pos_prompt_logprob_toks, ref_pos_prompt_logprob_vals,
279 + ref_pos_prompt_token_rank) = (ref_prompt_logprob_toks[idx, :],
280 + ref_prompt_logprob_vals[idx, :],
281 + ref_prompt_token_ranks[idx])
282 +
283 + # For each position in the prompt sequence,
284 + # ensure the actual prompt token is among the
285 + # logprobs
286 + assert prompt_token in pos_logprob_dict, (
287 + f"Prompt token {prompt_token} not"
288 + f" present in logprob at index {idx}")
289 + # Validate number of prompt logprobs
290 + num_plp_toks = len(pos_logprob_dict)
291 + assert (num_plp_toks == num_prompt_logprobs
292 + or num_plp_toks == num_prompt_logprobs +
293 + 1), ("Valid numbers of prompt logprobs are"
294 + f" {num_prompt_logprobs} or"
295 + f" {num_prompt_logprobs+1} but"
296 + f" {num_plp_toks} logprobs found at"
297 + f" position {idx}. Logprobs dict:"
298 + f" {pos_logprob_dict}")
299 +
300 + # Validate prompt token logprob rank
301 + prmpt_tok_lp = pos_logprob_dict[prompt_token]
302 + prmpt_tok_lp_rank = prmpt_tok_lp.rank
303 + ref_prmpt_tok_lp_rank = ref_pos_prompt_token_rank
304 + assert (ref_prmpt_tok_lp_rank == prmpt_tok_lp_rank), (
305 + "Prompt token logprob rank"
306 + f" {prmpt_tok_lp_rank} does not match"
307 + " correct value"
308 + f" {ref_prmpt_tok_lp_rank}"
309 + f" in Logprob {prmpt_tok_lp}")
310 +
311 + # Validate that the logprob processor yields
312 + # the correct prompt log probs and valid
313 + # rankings
314 + rank_one_appears = False
315 + for jdx in range(1, len(ref_pos_prompt_logprob_toks)):
316 + # Iterate over the (logprob val,logprob tok id)
317 + # pairs expected by the test fixture at this
318 + # position in the completion.
319 + ref_plp_val = float(ref_pos_prompt_logprob_vals[jdx])
320 + ref_tok_id = int(ref_pos_prompt_logprob_toks[jdx])
321 + assert ref_tok_id in pos_logprob_dict, (
322 + f"Expected token {ref_tok_id} to be"
323 + f" in logprob dict but it is not.")
324 +
325 + # Extract actually-generated logprob
326 + # info
327 + plp = pos_logprob_dict[ref_tok_id]
328 + plp_val = plp.logprob
329 + plp_rank = plp.rank
330 +
331 + # A "top" (rank 1) logprob must be
332 + # present
333 + rank_one_appears = (True
334 + if plp_rank == 1 else rank_one_appears)
335 +
336 + # Rank must be >= 1
337 + assert plp_rank >= 1, (
338 + f"Logprob {plp} has invalid"
339 + f" rank {plp_rank} < 1."
340 + f" Logprob dict: {pos_logprob_dict}")
341 +
342 + # Validate log probability
343 + assert math.isclose(plp_val, ref_plp_val), (
344 + f"Token id {ref_tok_id} appears in logprobs dict"
345 + f" at position {idx} in completion with log"
346 + f" probability {plp_val} but {ref_plp_val} was"
347 + f" expected. Logprob: {plp}")
348 +
349 + assert rank_one_appears, (f"No Logprob has rank 1"
350 + " in the following Logprob"
351 + f" dict: {pos_logprob_dict}")
352 +
353 + # Validate prompt logprob detokenization
354 + for plp_tok in pos_logprob_dict:
355 + # Confirm that prompt logprob decoded token matches
356 + # the logprob token id at this sequence position
357 + decoded_token = pos_logprob_dict[plp_tok].decoded_token
358 + ref_decoded_token = _ref_convert_id_to_token(
359 + dtv.tokenizer, plp_tok)
360 + assert decoded_token == ref_decoded_token, (
361 + f"Prompt logprob token id {plp_tok} decodes to"
362 + f" {ref_decoded_token} but Logprob decoded"
363 + f" token is {decoded_token} instead"
364 + f" (at position {idx})")
365 + else:
366 + # Prompt logprobs disabled for this request
367 + assert prompt_logprobs is None
368 +
369 +
370 +@pytest.mark.parametrize(
371 + "request_output_kind",
372 + [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
373 +@pytest.mark.parametrize("num_sample_logprobs",
374 + [None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
375 +@pytest.mark.parametrize("num_prompt_logprobs",
376 + [None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
377 +def test_logprobs_processor(request_output_kind: RequestOutputKind,
378 + num_sample_logprobs: Optional[int],
379 + num_prompt_logprobs: Optional[int],
380 + dummy_test_vectors):
381 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
382 + log_stats=False)
383 + engine_core = MockEngineCore(
384 + tokens_list=dummy_test_vectors.generation_tokens,
385 + generated_logprobs_raw=None if num_sample_logprobs is None else
386 + dummy_test_vectors.generation_logprobs,
387 + prompt_logprobs_raw=None
388 + if num_prompt_logprobs is None else dummy_test_vectors.prompt_logprobs)
389 +
390 + # Make N requests.
391 + request_id_list = [
392 + f"request-{idx}"
393 + for idx in range(len(dummy_test_vectors.prompt_strings))
394 + ]
395 + requests = [
396 + EngineCoreRequest(request_id=request_id_list[idx],
397 + prompt=prompt,
398 + prompt_token_ids=prompt_tokens,
399 + arrival_time=0,
400 + mm_inputs=None,
401 + mm_hashes=None,
402 + mm_placeholders=None,
403 + eos_token_id=None,
404 + lora_request=None,
405 + sampling_params=SamplingParams(
406 + skip_special_tokens=False,
407 + spaces_between_special_tokens=False,
408 + output_kind=request_output_kind,
409 + stop=[],
410 + include_stop_str_in_output=False,
411 + logprobs=num_sample_logprobs,
412 + prompt_logprobs=num_prompt_logprobs,
413 + )) for idx, (prompt, prompt_tokens) in enumerate(
414 + zip(dummy_test_vectors.prompt_strings,
415 + dummy_test_vectors.prompt_tokens))
416 + ]
417 +
418 + # Add requests to the detokenizer.
419 + for request in requests:
420 + output_processor.add_request(request)
421 +
422 + gen_tokens = {}
423 + gen_logprobs = {}
424 + gen_prompt_logprobs = {}
425 + gen_cumulative_logprobs = {}
426 + while True:
427 + # Mock output from the EngineCore.
428 + outputs = engine_core.get_outputs()
429 + if len(outputs) == 0:
430 + break
431 +
432 + # Step the logprobs processor.
433 + processed_outputs = output_processor.process_outputs(outputs)
434 + request_outputs = processed_outputs.request_outputs
435 + requests_to_abort = processed_outputs.reqs_to_abort
436 + assert len(requests_to_abort) == 0
437 +
438 + # Update tracking.
439 + for request_output in request_outputs:
440 + request_id = request_output.request_id
441 + new_tokens = request_output.outputs[0].token_ids
442 + prompt_logprobs = request_output.prompt_logprobs
443 + logprobs = request_output.outputs[0].logprobs
444 + gen_cumulative_logprobs[request_id] = request_output.outputs[
445 + 0].cumulative_logprob
446 + if request_id not in gen_logprobs:
447 + # Start tracking sample and prompt logprobs for this request
448 + gen_tokens[request_id] = new_tokens
449 + gen_logprobs[request_id] = logprobs
450 + gen_prompt_logprobs[request_id] = prompt_logprobs
451 + else:
452 + # Extend logprobs tracker
453 + gen_tokens[request_id].extend(new_tokens)
454 + lp = gen_logprobs[request_id]
455 + plp = gen_prompt_logprobs[request_id]
456 + if lp:
457 + lp.extend(logprobs)
458 + if plp:
459 + plp.extend(prompt_logprobs)
460 +
461 + # Confirmed tracked logprobs match what we expect
462 + _validate_logprobs(gen_tokens, gen_logprobs, gen_prompt_logprobs,
463 + gen_cumulative_logprobs, dummy_test_vectors,
464 + request_id_list, num_sample_logprobs,
465 + num_prompt_logprobs)
466 +
467 + assert output_processor.get_num_unfinished_requests() == 0
468 + assert not output_processor.has_unfinished_requests()
469 +
470 +
471 @pytest.mark.parametrize("include_stop_str_in_output", [True, False])
147 -def test_stop_string(include_stop_str_in_output: bool):
148 - output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
149 - engine_core = MockEngineCore(GENERATION_TOKENS)
472 +@pytest.mark.parametrize("num_sample_logprobs",
473 + [None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])
474 +@pytest.mark.parametrize("num_prompt_logprobs",
475 + [None, NUM_PROMPT_LOGPROBS_UNDER_TEST])
476 +def test_stop_string(include_stop_str_in_output: bool,
477 + num_sample_logprobs: Optional[int],
478 + num_prompt_logprobs: Optional[int], dummy_test_vectors):
479 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
480 + log_stats=False)
481 + engine_core = MockEngineCore(
482 + tokens_list=dummy_test_vectors.generation_tokens,
483 + generated_logprobs_raw=dummy_test_vectors.generation_logprobs
484 + if num_sample_logprobs else None,
485 + prompt_logprobs_raw=dummy_test_vectors.prompt_logprobs
486 + if num_prompt_logprobs else None)
487
488 # Make N requests.
489 + request_id_list = [
490 + f"request-{idx}"
491 + for idx in range(len(dummy_test_vectors.prompt_strings))
492 + ]
493 requests = [
494 EngineCoreRequest(
154 - request_id=f"request-{idx}",
495 + request_id=request_id_list[idx],
496 prompt=prompt,
497 prompt_token_ids=prompt_tokens,
498 arrival_time=0,
@@ -166,9 +507,11 @@ def test_stop_string(include_stop_str_in_output: bool):
507 output_kind=RequestOutputKind.DELTA,
508 stop=STOP_STRINGS,
509 include_stop_str_in_output=include_stop_str_in_output,
169 - )) for idx, (
170 - prompt,
171 - prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
510 + logprobs=num_sample_logprobs,
511 + prompt_logprobs=num_prompt_logprobs,
512 + )) for idx, (prompt, prompt_tokens) in enumerate(
513 + zip(dummy_test_vectors.prompt_strings,
514 + dummy_test_vectors.prompt_tokens))
515 ]
516
517 # Add requests to the detokenizer.
@@ -176,6 +519,10 @@ def test_stop_string(include_stop_str_in_output: bool):
519 output_processor.add_request(request)
520
521 gen_strings = {}
522 + gen_tokens = {}
523 + gen_logprobs = {}
524 + gen_prompt_logprobs = {}
525 + gen_cumulative_logprobs = {}
526 aborted = []
527 while True:
528 # Mock output from the EngineCore.
@@ -199,14 +546,29 @@ def test_stop_string(include_stop_str_in_output: bool):
546
547 request_id = request_output.request_id
548 new_text = request_output.outputs[0].text
549 + new_tokens = request_output.outputs[0].token_ids
550 + prompt_logprobs = request_output.prompt_logprobs
551 + logprobs = request_output.outputs[0].logprobs
552 + gen_cumulative_logprobs[request_id] = request_output.outputs[
553 + 0].cumulative_logprob
554 if request_id not in gen_strings:
555 gen_strings[request_id] = new_text
556 + gen_tokens[request_id] = new_tokens
557 + gen_logprobs[request_id] = logprobs
558 + gen_prompt_logprobs[request_id] = prompt_logprobs
559 else:
560 gen_strings[request_id] += new_text
561 + gen_tokens[request_id].extend(new_tokens)
562 + lp = gen_logprobs[request_id]
563 + plp = gen_prompt_logprobs[request_id]
564 + if lp:
565 + lp.extend(logprobs)
566 + if plp:
567 + plp.extend(prompt_logprobs)
568
569 # Confirmed tracked values matches what we expected.
208 - for idx, (ref_gen_str,
209 - stop_str) in enumerate(zip(GENERATION_STRINGS, STOP_STRINGS)):
570 + for idx, (ref_gen_str, stop_str) in enumerate(
571 + zip(dummy_test_vectors.generation_strings, STOP_STRINGS)):
572
573 # Request should be aborted.
574 request_id = f"request-{idx}"
@@ -227,13 +589,20 @@ def test_stop_string(include_stop_str_in_output: bool):
589 assert gen_str == ref_str_exc_stop, (
590 f"{gen_str=}, {ref_str_exc_stop=}")
591
592 + # Confirmed tracked logprobs match what we expect
593 + _validate_logprobs(gen_tokens, gen_logprobs, gen_prompt_logprobs,
594 + gen_cumulative_logprobs, dummy_test_vectors,
595 + request_id_list, num_sample_logprobs,
596 + num_prompt_logprobs)
597 +
598 assert output_processor.get_num_unfinished_requests() == 0
599 assert not output_processor.has_unfinished_requests()
600
601
234 -def test_iteration_stats():
235 - output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=True)
236 - engine_core = MockEngineCore(GENERATION_TOKENS)
602 +def test_iteration_stats(dummy_test_vectors):
603 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,
604 + log_stats=True)
605 + engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)
606
607 # Make N requests.
608 requests = [
@@ -248,13 +617,13 @@ def test_iteration_stats():
617 eos_token_id=None,
618 lora_request=None,
619 sampling_params=SamplingParams(),
251 - ) for idx, (
252 - prompt,
253 - prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
620 + ) for idx, (prompt, prompt_tokens) in enumerate(
621 + zip(dummy_test_vectors.prompt_strings,
622 + dummy_test_vectors.prompt_tokens))
623 ]
624
625 # Add all requests except one to the OutputProcessor.
257 - num_active = len(GENERATION_TOKENS) - 1
626 + num_active = len(dummy_test_vectors.generation_tokens) - 1
627 for request in requests[:num_active]:
628 output_processor.add_request(request)
629 inactive_request = requests[num_active]
@@ -263,8 +632,10 @@ def test_iteration_stats():
632 outputs = engine_core.get_outputs()[:num_active]
633 processed_outputs = output_processor.process_outputs(outputs)
634 iteration_stats = processed_outputs.iteration_stats
266 - total_prompt_tokens = sum(
267 - [len(prompt_tokens) for prompt_tokens in PROMPT_TOKENS[:num_active]])
635 + total_prompt_tokens = sum([
636 + len(prompt_tokens)
637 + for prompt_tokens in dummy_test_vectors.prompt_tokens[:num_active]
638 + ])
639
640 assert iteration_stats.num_prompt_tokens == total_prompt_tokens
641 assert iteration_stats.num_generation_tokens == num_active
@@ -283,7 +654,7 @@ def test_iteration_stats():
654 outputs = engine_core.get_outputs()[:num_active]
655 processed_outputs = output_processor.process_outputs(outputs)
656 iteration_stats = processed_outputs.iteration_stats
286 - total_prompt_tokens = len(PROMPT_TOKENS[num_active - 1])
657 + total_prompt_tokens = len(dummy_test_vectors.prompt_tokens[num_active - 1])
658
659 assert iteration_stats.num_prompt_tokens == total_prompt_tokens
660 assert iteration_stats.num_generation_tokens == num_active
tests/v1/engine/utils.py new
+382
@@ -0,0 +1,382 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import random
4 +from dataclasses import dataclass
5 +from typing import List, Optional, Tuple, Union
6 +
7 +import torch
8 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
9 +
10 +from vllm.engine.arg_utils import EngineArgs
11 +from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
12 + BaseTokenizerGroup)
13 +from vllm.v1.engine import EngineCoreOutput, FinishReason
14 +from vllm.v1.outputs import LogprobsLists, LogprobsTensors
15 +
16 +GeneralTokenizerType = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
17 +
18 +# Number of sample logprobs to request when testing sample logprobs
19 +NUM_SAMPLE_LOGPROBS_UNDER_TEST = 5
20 +# Number of prompt logprobs to request when testing prompt logprobs
21 +NUM_PROMPT_LOGPROBS_UNDER_TEST = 7
22 +
23 +TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
24 +
25 +FULL_STRINGS = [
26 + "My name is Robert from Neural Magic and I love working on vLLM so much!",
27 + "Red Hat is the best open source company by far across Linux, K8s, and AI.",
28 + "Nick is the name of my brother in addition to my colleague from Red Hat.",
29 +]
30 +STOP_STRINGS = ["I love working on", "company by far", "brother in"]
31 +PROMPT_LEN = 5
32 +
33 +PLP_APC_UNSUPPORTED_MSG = ("Prefix caching with prompt logprobs not yet "
34 + "supported on VLLM V1.")
35 +
36 +random.seed(42)
37 +
38 +
39 +def _create_random_top_logprob_test_vector(
40 + num_logprobs: int,
41 + lower: float,
42 + upper: float,
43 +) -> torch.Tensor:
44 + """Create a random vector of top logprob float values.
45 +
46 + Use to create fake sample logprobs for testing.
47 +
48 + Note that a real production scenario would require
49 + logprobs to be sorted in descending order, something
50 + which is omitted in this function.
51 +
52 + Args:
53 + num_logprobs: number of top logprobs
54 + lower: lower range of logprob float values
55 + upper: upper range of logprob float values
56 +
57 + Returns:
58 + 1D length-`num_logprobs` torch Tensor of float logprob values
59 + """
60 + return torch.rand(num_logprobs) * (upper - lower) + lower
61 +
62 +
63 +def _create_random_top_logprob_test_matrix(
64 + shape: Tuple,
65 + lower: float,
66 + upper: float,
67 +) -> torch.Tensor:
68 + """Create a random matrix of top logprob float values.
69 +
70 + Use to create fake prompt logprobs for testing.
71 +
72 + Note that a real production scenario would require
73 + logprobs to be sorted in descending order along rows,
74 + something which is omitted in this function.
75 +
76 + Args:
77 + shape: (num_tokens,num_logprobs) tuple representing
78 + matrix shape
79 + lower: lower range of logprob float values
80 + upper: upper range of logprob float values
81 +
82 + Returns:
83 + 2D num_tokens x num_logprobs torch Tensor of float logprob values
84 + """
85 + return torch.rand(*shape) * (upper - lower) + lower
86 +
87 +
88 +def _create_random_top_token_test_vector(
89 + num_logprobs: int,
90 + lower: int,
91 + upper: int,
92 + sampled_token_id: int,
93 + adjust_num_logprobs: bool = True) -> Tuple[torch.Tensor, int]:
94 + """Create a random vector of top logprob token indices
95 +
96 + Use to create fake sample logprobs for testing. The sampled token
97 + ID must always be one of the top logprobs, which this dummy test
98 + vector generator enforces. OpenAI API
99 + compatible engines must be able to return an additional sample
100 + logprob for the sampled token if the sampled token was not
101 + among the top sample logprobs; `adjust_num_logprobs` emulates
102 + this behavior by increasing the vector length by 1 if
103 + `adjust_num_logprobs` is set.
104 +
105 + Args:
106 + num_logprobs: number of top logprobs
107 + lower: lower range of token ids
108 + upper: upper range of token ids
109 + sampled_token_id: the token actually sampled
110 + adjust_num_logprobs: if True, emulate situation where sampled
111 + token logprob must be injected into top
112 + logprobs
113 +
114 + Returns:
115 + 1D length-x torch Tensor of token ids where x is
116 + `num_logprobs+1` if `adjust_num_logprobs` and
117 + `num_logprobs` otherwise
118 + sampled_token_rank: the rank of sampled_token_id in the vocab
119 + vector when sorted in descending order by
120 + logprob
121 + """
122 +
123 + # Calculate the final number of logprobs required
124 + total_logprobs = num_logprobs + 1 if adjust_num_logprobs else num_logprobs
125 +
126 + # Generate random indices using torch
127 + choice_tensor = torch.randperm(upper - lower)[:total_logprobs] + lower
128 +
129 + # Ensure the sampled token ID is included in the tensor
130 + choice_tensor[0] = sampled_token_id
131 +
132 + # Check if the sampled_token_id occurs in choice_tensor[1:]
133 + if sampled_token_id in choice_tensor[1:]:
134 + sampled_token_rank = (choice_tensor[1:] == sampled_token_id).nonzero(
135 + as_tuple=True)[0].item()
136 + else:
137 + # If not found, assign a random int between num_logprobs and 50700
138 + sampled_token_rank = random.randint(num_logprobs, 50700)
139 +
140 + return choice_tensor, sampled_token_rank
141 +
142 +
143 +def _create_random_top_token_test_matrix(
144 + shape: Tuple[int, int],
145 + lower: int,
146 + upper: int,
147 + tokens_list: List[int],
148 +) -> Tuple[torch.Tensor, torch.Tensor]:
149 + """Create a random matrix of top logprob token indices
150 +
151 + Use to create fake prompt logprobs for testing.
152 +
153 + Token ids are generated randomly and sampled without
154 + replacement.
155 +
156 + Args:
157 + shape: (num_tokens, num_logprobs) tuple representing
158 + matrix shape
159 + lower: lower range of token ids
160 + upper: upper range of token ids
161 +
162 + Returns:
163 + Tuple containing:
164 + - 2D num_tokens x num_logprobs+1 torch Tensor of token ids
165 + - 1D tensor of ranks of prompt tokens in their respective
166 + rows, or random values
167 + """
168 + num_elements = shape[0] * shape[1]
169 + choice_tensor = torch.randperm(upper - lower)[:num_elements] + lower
170 + matrix = torch.cat(
171 + (torch.tensor(tokens_list, dtype=torch.int).unsqueeze(-1),
172 + choice_tensor.view(shape)),
173 + dim=1)
174 +
175 + # Initialize the tensor for storing the ranks
176 + prompt_token_ranks = torch.empty(shape[0], dtype=torch.int)
177 +
178 + # Iterate over each row to check presence of
179 + # tokens_list[rdx] and determine its index
180 + for rdx in range(shape[0]):
181 + row = matrix[rdx,
182 + 1:] # Skip the first column as it contains the token list
183 + token_index = (row == tokens_list[rdx]).nonzero(as_tuple=True)[0]
184 + if token_index.numel() > 0:
185 + prompt_token_ranks[rdx] = token_index.item()
186 + else:
187 + prompt_token_ranks[rdx] = random.randint(shape[1], 50700)
188 +
189 + return matrix, prompt_token_ranks
190 +
191 +
192 +def decode_token(
193 + tok_id: int,
194 + tokenizer: PreTrainedTokenizer,
195 +) -> str:
196 + """Reproduce the process of detokenizing a token for testing purposes.
197 +
198 + Args:
199 + tok_id: token id to detokenize
200 + tokenizer: tokenizer to use for detokenization
201 +
202 + Returns:
203 + string representation of token
204 + """
205 + return tokenizer.convert_ids_to_tokens(tok_id)
206 +
207 +
208 +def generate_dummy_sample_logprobs(
209 + sampled_tokens_list: List,
210 + num_logprobs: int,
211 + tokenizer: PreTrainedTokenizer,
212 +) -> List[Tuple[List[int], List[float], int]]:
213 + """Generate dummy sample logprobs
214 +
215 + Generate a test data structure which imitates the list of sample logprobs
216 + which would be assembled in the engine core during decode phase.
217 +
218 + Args:
219 + sampled_tokens_list: list of sampled tokens
220 + num_logprobs: return `num_logprobs` or `num_logprobs+1` logprobs per token
221 + tokenizer: model tokenizer to use for detokenization
222 +
223 + Returns
224 + List of (top token ids vector, logprobs vector, sampled token rank)
225 + Python lists tuples; in each tuple the logprobs and top token ids
226 + vectors have the same length which is either `num_logprobs` or
227 + `num_logprobs+1`. Sampled token rank is the rank (index+1) of the
228 + sampled token within the vocab vector when sorted by logprob in
229 + descending order.
230 + """
231 + res = []
232 + for sampled_token_id in sampled_tokens_list:
233 + (
234 + token_vector,
235 + sampled_token_rank,
236 + ) = _create_random_top_token_test_vector(num_logprobs, 0,
237 + len(tokenizer.vocab) - 1,
238 + sampled_token_id)
239 +
240 + res.append(
241 + (token_vector,
242 + _create_random_top_logprob_test_vector(num_logprobs + 1, -100,
243 + 0), sampled_token_rank))
244 +
245 + # Convert tensors in the list tuples to Python lists
246 + res_list_format = [
247 + (log_probs_tensor.tolist(), token_ids_tensor.tolist(),
248 + sampled_token_rank)
249 + for log_probs_tensor, token_ids_tensor, sampled_token_rank in res
250 + ]
251 +
252 + return res_list_format
253 +
254 +
255 +def generate_dummy_prompt_logprobs_tensors(
256 + prompt_tokens_list: List,
257 + num_logprobs: int,
258 + tokenizer: PreTrainedTokenizer,
259 +) -> LogprobsTensors:
260 + """Generate dummy prompt logprobs tensors
261 +
262 + Generate a test data structure which imitates the torch Tensors of prompt
263 + logprobs which would be assembled in the engine core during chunked
264 + prefill.
265 +
266 + Args:
267 + prompt_tokens_list: list of prompt tokens
268 + num_logprobs: return `num_logprobs` logprobs per token
269 + tokenizer: model tokenizer to use for detokenization
270 +
271 + Returns
272 + Single Tuple of (logprobs matrix, top token ids matrix) torch Tensor,
273 + where both matrices have dimensions
274 + num_prompt_tokens x num_logprobs
275 + """
276 + # For now, assume the whole prompt is processed in one chunk; thus,
277 + # the number of non-`None` prompt logprobs is `len(prompt_tokens_list)-1`.
278 + # Prior to injecting `None` at the beginning of prompt logprobs (which
279 + # happens later in the detokenizer, not here), the prompt logprobs in
280 + # the ith position are predicting the probability distribution of the
281 + # prompt token in (i+1)st position. Thus, we concat
282 + # `prompt_tokens_list[1:]` to the dummy token ids, just as the engine
283 + # would.
284 + num_prompt_logprobs = len(prompt_tokens_list) - 1
285 + (
286 + token_vector,
287 + prompt_token_ranks,
288 + ) = _create_random_top_token_test_matrix(
289 + (num_prompt_logprobs, num_logprobs), 0,
290 + len(tokenizer.vocab) - 1, prompt_tokens_list[1:])
291 + return LogprobsTensors(
292 + token_vector,
293 + _create_random_top_logprob_test_matrix(
294 + (num_prompt_logprobs, num_logprobs + 1), -100, 0),
295 + prompt_token_ranks)
296 +
297 +
298 +@dataclass
299 +class DummyOutputProcessorTestVectors:
300 + """Dummy test vectors for output processor tests"""
301 + tokenizer: GeneralTokenizerType
302 + tokenizer_group: BaseTokenizerGroup
303 + vllm_config: EngineArgs
304 + full_tokens: List[List[int]] # Prompt + generated tokens
305 + prompt_tokens: List[List[int]]
306 + generation_tokens: List[List[int]]
307 + # Each request is associated with a tuple of
308 + # (top tokens, top logprobs, ranks) prompt logprobs tensors
309 + prompt_logprobs: List[LogprobsTensors]
310 + # Each request is associated with a sample logprobs; a request's
311 + # sample logprobs are a list of (top tokens, top logprobs, ranks)
312 + # sample logprobs tensors at each sequence position
313 + generation_logprobs: List[List[Tuple[List[int], List[float], int]]]
314 + prompt_strings: List[str]
315 + prompt_strings_len: List[int]
316 + generation_strings: List[str]
317 +
318 +
319 +class MockEngineCore:
320 + """Mock engine core outputs form premade tokens lists."""
321 +
322 + def __init__(
323 + self,
324 + tokens_list: List[List[int]],
325 + # For each request, for each sampled token offset,
326 + # a tuple of
327 + # (list of topk token ids, list of sample logprob vals, rank)
328 + generated_logprobs_raw: Optional[List[List[Tuple[List[int],
329 + List[float],
330 + int]]]] = None,
331 + # For each request, a tuple of
332 + # (prompt logprob val matrix, prompt logprob tok id matrix);
333 + # each matrix has dimensions
334 + # (num prompt toks) x (num prompt logprobs+1)
335 + prompt_logprobs_raw: Optional[List[LogprobsTensors]] = None,
336 + ) -> None:
337 + self.tokens_list = tokens_list
338 + self.current_idx = 0
339 + self.generated_logprobs_raw = generated_logprobs_raw
340 + self.do_logprobs = generated_logprobs_raw is not None
341 + self.prompt_logprobs_raw = prompt_logprobs_raw
342 + self.do_prompt_logprobs = prompt_logprobs_raw is not None
343 +
344 + def get_outputs(self) -> List[EngineCoreOutput]:
345 + do_logprobs = self.do_logprobs
346 + do_prompt_logprobs = self.do_prompt_logprobs
347 + token_idx = self.current_idx
348 +
349 + outputs = []
350 + for req_idx, token_ids in enumerate(self.tokens_list):
351 + if len(token_ids) > token_idx:
352 + if do_logprobs:
353 + assert self.generated_logprobs_raw is not None
354 + (logprobs_token_ids_, logprobs_, sampled_token_ranks_) = (
355 + self.generated_logprobs_raw[req_idx][token_idx])
356 + logprobs = LogprobsLists(
357 + [logprobs_token_ids_],
358 + [logprobs_],
359 + [sampled_token_ranks_],
360 + )
361 + else:
362 + logprobs = None
363 + if do_prompt_logprobs:
364 + if self.current_idx == 0:
365 + assert self.prompt_logprobs_raw is not None
366 + prompt_logprobs = self.prompt_logprobs_raw[req_idx]
367 + else:
368 + prompt_logprobs = None
369 + else:
370 + prompt_logprobs = None
371 + output = EngineCoreOutput(
372 + request_id=f"request-{req_idx}",
373 + new_token_ids=[token_ids[token_idx]],
374 + new_logprobs=logprobs,
375 + new_prompt_logprobs_tensors=prompt_logprobs,
376 + )
377 + if token_idx == len(token_ids) - 1:
378 + output.finish_reason = FinishReason.STOP
379 + outputs.append(output)
380 +
381 + self.current_idx += 1
382 + return outputs
tests/v1/entrypoints/__init__.py
tests/v1/entrypoints/conftest.py new
+161
@@ -0,0 +1,161 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import pytest
4 +
5 +
6 +@pytest.fixture
7 +def sample_prompts():
8 + return [
9 + "Hello, my name is",
10 + "The president of the United States is",
11 + "The capital of France is",
12 + "The future of AI is",
13 + ]
14 +
15 +
16 +@pytest.fixture
17 +def sample_token_ids():
18 + return [
19 + [0],
20 + [0, 1],
21 + [0, 2, 1],
22 + [0, 3, 1, 2],
23 + ]
24 +
25 +
26 +@pytest.fixture
27 +def sample_regex():
28 + return (r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
29 + r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)")
30 +
31 +
32 +@pytest.fixture
33 +def sample_json_schema():
34 + return {
35 + "type": "object",
36 + "properties": {
37 + "name": {
38 + "type": "string"
39 + },
40 + "age": {
41 + "type": "integer"
42 + },
43 + "skills": {
44 + "type": "array",
45 + "items": {
46 + "type": "string",
47 + "maxLength": 10
48 + },
49 + "minItems": 3
50 + },
51 + "work_history": {
52 + "type": "array",
53 + "items": {
54 + "type": "object",
55 + "properties": {
56 + "company": {
57 + "type": "string"
58 + },
59 + "duration": {
60 + "type": "number"
61 + },
62 + "position": {
63 + "type": "string"
64 + }
65 + },
66 + "required": ["company", "position"]
67 + }
68 + }
69 + },
70 + "required": ["name", "age", "skills", "work_history"]
71 + }
72 +
73 +
74 +@pytest.fixture
75 +def sample_complex_json_schema():
76 + return {
77 + "type": "object",
78 + "properties": {
79 + "score": {
80 + "type": "integer",
81 + "minimum": 0,
82 + "maximum": 100 # Numeric range
83 + },
84 + "grade": {
85 + "type": "string",
86 + "pattern": "^[A-D]$" # Regex pattern
87 + },
88 + "email": {
89 + "type": "string",
90 + "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$"
91 + },
92 + "tags": {
93 + "type": "array",
94 + "items": {
95 + "type": "string",
96 + "pattern":
97 + "^[a-z]{1,10}$" # Combining length and pattern restrictions
98 + }
99 + }
100 + },
101 + "required": ["score", "grade", "email", "tags"]
102 + }
103 +
104 +
105 +@pytest.fixture
106 +def sample_definition_json_schema():
107 + return {
108 + '$defs': {
109 + 'Step': {
110 + 'properties': {
111 + 'explanation': {
112 + 'title': 'Explanation',
113 + 'type': 'string'
114 + },
115 + 'output': {
116 + 'title': 'Output',
117 + 'type': 'string'
118 + }
119 + },
120 + 'required': ['explanation', 'output'],
121 + 'title': 'Step',
122 + 'type': 'object'
123 + }
124 + },
125 + 'properties': {
126 + 'steps': {
127 + 'items': {
128 + '$ref': '#/$defs/Step'
129 + },
130 + 'title': 'Steps',
131 + 'type': 'array'
132 + },
133 + 'final_answer': {
134 + 'title': 'Final Answer',
135 + 'type': 'string'
136 + }
137 + },
138 + 'required': ['steps', 'final_answer'],
139 + 'title': 'MathReasoning',
140 + 'type': 'object'
141 + }
142 +
143 +
144 +@pytest.fixture
145 +def sample_guided_choice():
146 + return [
147 + "Python", "Java", "JavaScript", "C++", "C#", "PHP", "TypeScript",
148 + "Ruby", "Swift", "Kotlin"
149 + ]
150 +
151 +
152 +@pytest.fixture
153 +def sample_sql_statements():
154 + return ("""
155 +start: select_statement
156 +select_statement: "SELECT" column "from" table "where" condition
157 +column: "col_1" | "col_2"
158 +table: "table_1" | "table_2"
159 +condition: column "=" number
160 +number: "1" | "2"
161 +""")
tests/v1/entrypoints/openai/test_completion.py new
+475
@@ -0,0 +1,475 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import re
4 +from typing import Dict, List, Optional
5 +
6 +import openai # use the official client for correctness check
7 +import pytest
8 +import pytest_asyncio
9 +from openai import BadRequestError
10 +
11 +from tests.utils import RemoteOpenAIServer
12 +from vllm.transformers_utils.tokenizer import get_tokenizer
13 +
14 +# any model with a chat template should work here
15 +MODEL_NAME = "facebook/opt-125m"
16 +
17 +
18 +@pytest.fixture(scope="module")
19 +def default_server_args():
20 + return [
21 + # use half precision for speed and memory savings in CI environment
22 + "--dtype",
23 + "bfloat16",
24 + "--max-model-len",
25 + "2048",
26 + "--max-num-seqs",
27 + "128",
28 + "--enforce-eager"
29 + ]
30 +
31 +
32 +@pytest.fixture(scope="module",
33 + params=[["--no-enable-prefix-caching"],
34 + [
35 + "--no-enable-prefix-caching",
36 + "--disable-frontend-multiprocessing"
37 + ]])
38 +def server(default_server_args, request):
39 + if request.param:
40 + default_server_args.extend(request.param)
41 + with RemoteOpenAIServer(MODEL_NAME, default_server_args) as remote_server:
42 + yield remote_server
43 +
44 +
45 +@pytest_asyncio.fixture
46 +async def client(server):
47 + async with server.get_async_client() as async_client:
48 + yield async_client
49 +
50 +
51 +@pytest.mark.asyncio
52 +@pytest.mark.parametrize(
53 + "model_name",
54 + [MODEL_NAME],
55 +)
56 +async def test_single_completion(client: openai.AsyncOpenAI,
57 + model_name: str) -> None:
58 + completion = await client.completions.create(model=model_name,
59 + prompt="Hello, my name is",
60 + max_tokens=5,
61 + temperature=0.0)
62 +
63 + assert completion.id is not None
64 + assert completion.choices is not None and len(completion.choices) == 1
65 +
66 + choice = completion.choices[0]
67 + assert len(choice.text) >= 5
68 + assert choice.finish_reason == "length"
69 + assert completion.usage == openai.types.CompletionUsage(
70 + completion_tokens=5, prompt_tokens=6, total_tokens=11)
71 +
72 + # test using token IDs
73 + completion = await client.completions.create(
74 + model=model_name,
75 + prompt=[0, 0, 0, 0, 0],
76 + max_tokens=5,
77 + temperature=0.0,
78 + )
79 + assert len(completion.choices[0].text) >= 1
80 + assert completion.choices[0].prompt_logprobs is None
81 +
82 +
83 +@pytest.mark.asyncio
84 +@pytest.mark.parametrize(
85 + "model_name",
86 + [MODEL_NAME],
87 +)
88 +async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):
89 + # test using token IDs
90 + completion = await client.completions.create(
91 + model=model_name,
92 + prompt=[0, 0, 0, 0, 0],
93 + max_tokens=5,
94 + temperature=0.0,
95 + logprobs=None,
96 + )
97 + choice = completion.choices[0]
98 + assert choice.logprobs is None
99 +
100 +
101 +@pytest.mark.asyncio
102 +@pytest.mark.parametrize(
103 + "model_name",
104 + [MODEL_NAME],
105 +)
106 +async def test_zero_logprobs(client: openai.AsyncOpenAI, model_name: str):
107 + # test using token IDs
108 + completion = await client.completions.create(
109 + model=model_name,
110 + prompt=[0, 0, 0, 0, 0],
111 + max_tokens=5,
112 + temperature=0.0,
113 + logprobs=0,
114 + )
115 + choice = completion.choices[0]
116 + assert choice.logprobs is not None
117 + assert choice.logprobs.token_logprobs is not None
118 + assert choice.logprobs.top_logprobs is not None
119 + assert len(choice.logprobs.top_logprobs[0]) == 1
120 +
121 +
122 +@pytest.mark.asyncio
123 +@pytest.mark.parametrize(
124 + "model_name",
125 + [MODEL_NAME],
126 +)
127 +async def test_some_logprobs(client: openai.AsyncOpenAI, model_name: str):
128 + # test using token IDs
129 + completion = await client.completions.create(
130 + model=model_name,
131 + prompt=[0, 0, 0, 0, 0],
132 + max_tokens=5,
133 + temperature=0.0,
134 + logprobs=5,
135 + )
136 + choice = completion.choices[0]
137 + assert choice.logprobs is not None
138 + assert choice.logprobs.token_logprobs is not None
139 + assert choice.logprobs.top_logprobs is not None
140 + assert 5 <= len(choice.logprobs.top_logprobs[0]) <= 6
141 +
142 +
143 +@pytest.mark.asyncio
144 +@pytest.mark.parametrize(
145 + "model_name",
146 + [MODEL_NAME],
147 +)
148 +async def test_too_many_completion_logprobs(client: openai.AsyncOpenAI,
149 + model_name: str) -> None:
150 +
151 + with pytest.raises(
152 + (openai.BadRequestError, openai.APIError)): # test using token IDs
153 + await client.completions.create(
154 + model=model_name,
155 + prompt=[0, 0, 0, 0, 0],
156 + max_tokens=5,
157 + temperature=0.0,
158 + # vLLM has higher default max_logprobs (20 instead of 5) to support
159 + # both Completion API and Chat Completion API
160 + logprobs=21,
161 + )
162 + ...
163 + with pytest.raises(
164 + (openai.BadRequestError, openai.APIError)): # test using token IDs
165 + stream = await client.completions.create(
166 + model=model_name,
167 + prompt=[0, 0, 0, 0, 0],
168 + max_tokens=5,
169 + temperature=0.0,
170 + # vLLM has higher default max_logprobs (20 instead of 5) to support
171 + # both Completion API and Chat Completion API
172 + logprobs=30,
173 + stream=True,
174 + )
175 + async for chunk in stream:
176 + ...
177 +
178 + # the server should still work afterwards
179 + completion = await client.completions.create(
180 + model=model_name,
181 + prompt=[0, 0, 0, 0, 0],
182 + max_tokens=5,
183 + temperature=0.0,
184 + )
185 + assert len(completion.choices[0].text) >= 0
186 +
187 +
188 +@pytest.mark.asyncio
189 +@pytest.mark.parametrize("model_name, prompt_logprobs", [(MODEL_NAME, -1),
190 + (MODEL_NAME, 0),
191 + (MODEL_NAME, 1),
192 + (MODEL_NAME, None)])
193 +async def test_prompt_logprobs_completion(client: openai.AsyncOpenAI,
194 + model_name: str,
195 + prompt_logprobs: Optional[int]):
196 + params: Dict = {
197 + "prompt": ["A robot may not injure another robot", "My name is"],
198 + "model": model_name,
199 + }
200 + if prompt_logprobs is not None:
201 + params["extra_body"] = {"prompt_logprobs": prompt_logprobs}
202 +
203 + if prompt_logprobs is not None and prompt_logprobs < 0:
204 + with pytest.raises(BadRequestError):
205 + await client.completions.create(**params)
206 + else:
207 + completion = await client.completions.create(**params)
208 + if prompt_logprobs is not None:
209 + assert completion.choices[0].prompt_logprobs is not None
210 + assert len(completion.choices[0].prompt_logprobs) > 0
211 +
212 + assert completion.choices[1].prompt_logprobs is not None
213 + assert len(completion.choices[1].prompt_logprobs) > 0
214 +
215 + else:
216 + assert completion.choices[0].prompt_logprobs is None
217 +
218 +
219 +@pytest.mark.asyncio
220 +@pytest.mark.parametrize(
221 + "model_name",
222 + [MODEL_NAME],
223 +)
224 +async def test_completion_streaming(client: openai.AsyncOpenAI,
225 + model_name: str) -> None:
226 + prompt = "What is an LLM?"
227 +
228 + single_completion = await client.completions.create(
229 + model=model_name,
230 + prompt=prompt,
231 + max_tokens=5,
232 + temperature=0.0,
233 + )
234 + single_output = single_completion.choices[0].text
235 + stream = await client.completions.create(model=model_name,
236 + prompt=prompt,
237 + max_tokens=5,
238 + temperature=0.0,
239 + stream=True)
240 + chunks: List[str] = []
241 + finish_reason_count = 0
242 + async for chunk in stream:
243 + chunks.append(chunk.choices[0].text)
244 + if chunk.choices[0].finish_reason is not None:
245 + finish_reason_count += 1
246 + # finish reason should only return in last block
247 + assert finish_reason_count == 1
248 + assert chunk.choices[0].finish_reason == "length"
249 + assert chunk.choices[0].text
250 + assert "".join(chunks) == single_output
251 +
252 +
253 +@pytest.mark.asyncio
254 +@pytest.mark.parametrize(
255 + "model_name",
256 + [MODEL_NAME],
257 +)
258 +async def test_completion_stream_options(client: openai.AsyncOpenAI,
259 + model_name: str):
260 + prompt = "What is the capital of France?"
261 +
262 + # Test stream=True, stream_options=
263 + # {"include_usage": False, "continuous_usage_stats": False}
264 + stream = await client.completions.create(model=model_name,
265 + prompt=prompt,
266 + max_tokens=5,
267 + temperature=0.0,
268 + stream=True,
269 + stream_options={
270 + "include_usage": False,
271 + "continuous_usage_stats":
272 + False,
273 + })
274 +
275 + async for chunk in stream:
276 + assert chunk.usage is None
277 +
278 + # Test stream=True, stream_options=
279 + # {"include_usage": False, "continuous_usage_stats": True}
280 + stream = await client.completions.create(model=model_name,
281 + prompt=prompt,
282 + max_tokens=5,
283 + temperature=0.0,
284 + stream=True,
285 + stream_options={
286 + "include_usage": False,
287 + "continuous_usage_stats":
288 + True,
289 + })
290 + async for chunk in stream:
291 + assert chunk.usage is None
292 +
293 + # Test stream=True, stream_options=
294 + # {"include_usage": True, "continuous_usage_stats": False}
295 + stream = await client.completions.create(model=model_name,
296 + prompt=prompt,
297 + max_tokens=5,
298 + temperature=0.0,
299 + stream=True,
300 + stream_options={
301 + "include_usage": True,
302 + "continuous_usage_stats":
303 + False,
304 + })
305 + async for chunk in stream:
306 + if chunk.choices[0].finish_reason is None:
307 + assert chunk.usage is None
308 + else:
309 + assert chunk.usage is None
310 + final_chunk = await stream.__anext__()
311 + assert final_chunk.usage is not None
312 + assert final_chunk.usage.prompt_tokens > 0
313 + assert final_chunk.usage.completion_tokens > 0
314 + assert final_chunk.usage.total_tokens == (
315 + final_chunk.usage.prompt_tokens +
316 + final_chunk.usage.completion_tokens)
317 + assert final_chunk.choices == []
318 +
319 + # Test stream=True, stream_options=
320 + # {"include_usage": True, "continuous_usage_stats": True}
321 + stream = await client.completions.create(model=model_name,
322 + prompt=prompt,
323 + max_tokens=5,
324 + temperature=0.0,
325 + stream=True,
326 + stream_options={
327 + "include_usage": True,
328 + "continuous_usage_stats":
329 + True,
330 + })
331 + async for chunk in stream:
332 + assert chunk.usage is not None
333 + assert chunk.usage.prompt_tokens > 0
334 + assert chunk.usage.completion_tokens > 0
335 + assert chunk.usage.total_tokens == (chunk.usage.prompt_tokens +
336 + chunk.usage.completion_tokens)
337 + if chunk.choices[0].finish_reason is not None:
338 + final_chunk = await stream.__anext__()
339 + assert final_chunk.usage is not None
340 + assert final_chunk.usage.prompt_tokens > 0
341 + assert final_chunk.usage.completion_tokens > 0
342 + assert final_chunk.usage.total_tokens == (
343 + final_chunk.usage.prompt_tokens +
344 + final_chunk.usage.completion_tokens)
345 + assert final_chunk.choices == []
346 +
347 + # Test stream=False, stream_options=
348 + # {"include_usage": None}
349 + with pytest.raises(BadRequestError):
350 + await client.completions.create(model=model_name,
351 + prompt=prompt,
352 + max_tokens=5,
353 + temperature=0.0,
354 + stream=False,
355 + stream_options={"include_usage": None})
356 +
357 + # Test stream=False, stream_options=
358 + # {"include_usage": True}
359 + with pytest.raises(BadRequestError):
360 + await client.completions.create(model=model_name,
361 + prompt=prompt,
362 + max_tokens=5,
363 + temperature=0.0,
364 + stream=False,
365 + stream_options={"include_usage": True})
366 +
367 + # Test stream=False, stream_options=
368 + # {"continuous_usage_stats": None}
369 + with pytest.raises(BadRequestError):
370 + await client.completions.create(
371 + model=model_name,
372 + prompt=prompt,
373 + max_tokens=5,
374 + temperature=0.0,
375 + stream=False,
376 + stream_options={"continuous_usage_stats": None})
377 +
378 + # Test stream=False, stream_options=
379 + # {"continuous_usage_stats": True}
380 + with pytest.raises(BadRequestError):
381 + await client.completions.create(
382 + model=model_name,
383 + prompt=prompt,
384 + max_tokens=5,
385 + temperature=0.0,
386 + stream=False,
387 + stream_options={"continuous_usage_stats": True})
388 +
389 +
390 +@pytest.mark.asyncio
391 +@pytest.mark.parametrize(
392 + "model_name",
393 + [MODEL_NAME],
394 +)
395 +async def test_batch_completions(client: openai.AsyncOpenAI, model_name: str):
396 + # test both text and token IDs
397 + for prompts in (["Hello, my name is"] * 2, [[0, 0, 0, 0, 0]] * 2):
398 + # test simple list
399 + batch = await client.completions.create(
400 + model=model_name,
401 + prompt=prompts,
402 + max_tokens=5,
403 + temperature=0.0,
404 + )
405 + assert len(batch.choices) == 2
406 + assert batch.choices[0].text == batch.choices[1].text
407 +
408 + # test n = 2
409 + batch = await client.completions.create(
410 + model=model_name,
411 + prompt=prompts,
412 + n=2,
413 + max_tokens=5,
414 + temperature=0.0,
415 + extra_body=dict(
416 + # NOTE: this has to be true for n > 1 in vLLM, but
417 + # not necessary for official client.
418 + use_beam_search=True),
419 + )
420 + assert len(batch.choices) == 4
421 + assert batch.choices[0].text != batch.choices[
422 + 1].text, "beam search should be different"
423 + assert batch.choices[0].text == batch.choices[
424 + 2].text, "two copies of the same prompt should be the same"
425 + assert batch.choices[1].text == batch.choices[
426 + 3].text, "two copies of the same prompt should be the same"
427 +
428 + # test streaming
429 + batch = await client.completions.create(
430 + model=model_name,
431 + prompt=prompts,
432 + max_tokens=5,
433 + temperature=0.0,
434 + stream=True,
435 + )
436 + texts = [""] * 2
437 + async for chunk in batch:
438 + assert len(chunk.choices) == 1
439 + choice = chunk.choices[0]
440 + texts[choice.index] += choice.text
441 + assert texts[0] == texts[1]
442 +
443 +
444 +@pytest.mark.asyncio
445 +@pytest.mark.parametrize(
446 + "model_name",
447 + [MODEL_NAME],
448 +)
449 +@pytest.mark.parametrize("logprobs_arg", [1, 0])
450 +async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
451 + model_name: str, logprobs_arg: int):
452 + tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME)
453 + # test using text and token IDs
454 + for prompt in ("Hello, my name is", [0, 0, 0, 0, 0]):
455 + completion = await client.completions.create(model=model_name,
456 + prompt=prompt,
457 + max_tokens=5,
458 + temperature=0.0,
459 + echo=True,
460 + logprobs=logprobs_arg)
461 +
462 + prompt_text = tokenizer.decode(prompt) if isinstance(prompt,
463 + list) else prompt
464 + assert re.search(r"^" + prompt_text, completion.choices[0].text)
465 + logprobs = completion.choices[0].logprobs
466 + assert logprobs is not None
467 + assert len(logprobs.text_offset) > 5
468 + assert (len(logprobs.token_logprobs) > 5
469 + and logprobs.token_logprobs[0] is None)
470 + assert (len(logprobs.top_logprobs) > 5
471 + and logprobs.top_logprobs[0] is None)
472 + for top_logprobs in logprobs.top_logprobs[1:]:
473 + assert max(logprobs_arg,
474 + 1) <= len(top_logprobs) <= logprobs_arg + 1
475 + assert len(logprobs.tokens) > 5
tests/v1/sample/test_logprobs.py new
+392
@@ -0,0 +1,392 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import itertools
4 +from typing import List, Tuple
5 +
6 +import pytest
7 +import torch
8 +
9 +from tests.kernels.utils import override_backend_env_variable
10 +from tests.v1.sample.utils import (
11 + assert_incr_detok_str_matches_non_incr_detok_str,
12 + compute_correct_cumulative_logprob, get_test_batch)
13 +from vllm import SamplingParams
14 +
15 +from ...conftest import VllmRunner
16 +
17 +MODEL = "meta-llama/Llama-3.2-1B"
18 +DTYPE = "half"
19 +
20 +
21 +@pytest.fixture(scope="module")
22 +def vllm_model(vllm_runner):
23 + with vllm_runner(
24 + MODEL,
25 + dtype=DTYPE,
26 + max_logprobs=7,
27 + # Very small number of batched tokens to ensure
28 + # that we test chunking.
29 + max_num_batched_tokens=16,
30 + max_num_seqs=16,
31 + max_model_len=128,
32 + enforce_eager=True,
33 + #TODO: enable this once we support it for
34 + # prompt logprobs.
35 + enable_prefix_caching=False,
36 + gpu_memory_utilization=0.5,
37 + ) as vllm_model:
38 + yield vllm_model
39 +
40 +
41 +@pytest.fixture(scope="module")
42 +def hf_model(hf_runner):
43 + with hf_runner(MODEL, dtype=DTYPE) as hf_model:
44 + yield hf_model
45 +
46 +
47 +def _repeat_logprob_config(
48 + test_prompts,
49 + logprob_prompt_logprob_list: List[Tuple],
50 +) -> List[Tuple]:
51 + """Ensure each test prompt has a logprob config.
52 +
53 + A logprob config specifies the optional (i.e.
54 + may-be-`None`) number of sample logprobs and
55 + the optional number of prompt logprobs.
56 +
57 + If more test prompts than logprob configs are
58 + provided, the provided logprob configs are
59 + tiled to match the number of test prompts.
60 +
61 + If fewer test prompts than logprob configs
62 + are provided, the list of logprob configs
63 + is truncated to match the number of test
64 + prompts.
65 +
66 + Otherwise, the list of logprob configs
67 + is returned as-is.
68 +
69 + Args:
70 + test_prompts: list of prompts under test
71 + logprob_prompt_logprob_list: list of
72 + (optional num sample logprob,
73 + optional num prompt logprob)
74 + tuples
75 +
76 + Returns:
77 + List of
78 + (optional num sample logprob,optional num prompt logprob)
79 + tuples which is either identical to
80 + `logprob_prompt_logprob_list`, or else repeats
81 + `logprob_prompt_logprob_list` enough times to match the
82 + number of `test_prompts`, or else is truncated to match
83 + the number of `test_prompts`
84 + """
85 + num_test_prompts = len(test_prompts)
86 + # Make sure there is a logprobs configuration for each test prompt
87 + logprob_prompt_logprob_list = list(
88 + itertools.islice(itertools.cycle(logprob_prompt_logprob_list),
89 + num_test_prompts))
90 + # Now the number of prompts should match the number of sample params combos
91 + assert num_test_prompts == len(logprob_prompt_logprob_list)
92 + return logprob_prompt_logprob_list
93 +
94 +
95 +def _test_case_get_logprobs_and_prompt_logprobs(
96 + hf_model,
97 + vllm_model,
98 + batch_logprobs_composition: str,
99 + temperature: float,
100 + example_prompts,
101 +) -> None:
102 + test_prompts = example_prompts
103 +
104 + max_tokens = 5
105 + hf_outputs = hf_model.generate_greedy(
106 + test_prompts,
107 + max_tokens=max_tokens,
108 + )
109 + hf_logprobs = hf_model.generate_greedy_logprobs(
110 + test_prompts,
111 + max_tokens=max_tokens,
112 + )
113 +
114 + # Batch has mixed sample params
115 + # (different logprobs/prompt logprobs combos)
116 + logprob_prompt_logprob_list = get_test_batch(batch_logprobs_composition)
117 +
118 + # Ensure that each test prompt has a logprob config for testing
119 + logprob_prompt_logprob_list = _repeat_logprob_config(
120 + test_prompts, logprob_prompt_logprob_list)
121 + # Generate SamplingParams
122 + vllm_sampling_params = [
123 + SamplingParams(max_tokens=max_tokens,
124 + logprobs=num_lp,
125 + prompt_logprobs=num_plp,
126 + temperature=temperature,
127 + seed=1984)
128 + for num_lp, num_plp in logprob_prompt_logprob_list
129 + ]
130 +
131 + vllm_results = vllm_model.model.generate(
132 + test_prompts, sampling_params=vllm_sampling_params)
133 +
134 + for vllm_result, hf_logprob, hf_output, logprob_prompt_logprob in zip(
135 + vllm_results, hf_logprobs, hf_outputs,
136 + logprob_prompt_logprob_list):
137 +
138 + # Extract request-level (prompt)logprobs config
139 + num_top_logprobs, num_top_prompt_logprobs = logprob_prompt_logprob
140 +
141 + # Test whether sampled token output is consistent between vLLM and HF
142 + # vLLM prompt+completion should match HF output
143 + if temperature == 0.0:
144 + assert (vllm_result.prompt_token_ids +
145 + vllm_result.outputs[0].token_ids == hf_output[0])
146 + else:
147 + # Sampled tokens won't match if not greedy
148 + assert (vllm_result.prompt_token_ids == hf_output[0]
149 + [:len(vllm_result.prompt_token_ids)])
150 +
151 + # Validate sample logprobs
152 + if num_top_logprobs is not None:
153 + assert num_top_logprobs is not None
154 + # Confirm that the structure of the sample logprobs in the result is
155 + # correct
156 + assert vllm_result.outputs[0].logprobs is not None
157 + assert len(vllm_result.outputs[0].logprobs) == max_tokens
158 + for logprobs, token_id in zip(vllm_result.outputs[0].logprobs,
159 + vllm_result.outputs[0].token_ids):
160 + assert logprobs is not None
161 +
162 + # Confirm that the output token appears among the logprobs
163 + assert token_id in logprobs
164 + token_in_topk = logprobs[token_id].rank <= num_top_logprobs
165 +
166 + # If the output token is not included in the top K
167 + # logprob, it can return 1 more data
168 + if token_in_topk and num_top_logprobs != 0:
169 + assert len(logprobs) == num_top_logprobs
170 + else:
171 + assert len(logprobs) == num_top_logprobs + 1
172 +
173 + if num_top_logprobs > 0:
174 + # We should have an entry for each of the topk ranks
175 + all_ranks = {lp.rank for lp in logprobs.values()}
176 + assert all(r in all_ranks
177 + for r in range(1, num_top_logprobs + 1))
178 +
179 + output_text = vllm_result.outputs[0].text
180 + output_string_from_most_likely_tokens_lst: List[str] = []
181 + for top_logprobs in vllm_result.outputs[0].logprobs:
182 + top_logprob = next(iter(top_logprobs.values()))
183 + output_string_from_most_likely_tokens_lst.append(
184 + top_logprob.decoded_token)
185 +
186 + output_string_from_most_likely_tokens = "".join(
187 + output_string_from_most_likely_tokens_lst)
188 + assert_incr_detok_str_matches_non_incr_detok_str(
189 + output_text, output_string_from_most_likely_tokens,
190 + "The output text from the top logprob for each token "
191 + "position should be the same as the output text in the "
192 + "result.")
193 +
194 + # Compare vLLM sample logprobs to HF
195 + vllm_sample_logprobs = vllm_result.outputs[0].logprobs
196 + for i, top_logprobs in enumerate(vllm_sample_logprobs):
197 + for token_id, sample_logprob in top_logprobs.items():
198 + if temperature == 0.0 or i == 0:
199 + logprob = sample_logprob.logprob
200 + torch.testing.assert_close(
201 + logprob,
202 + hf_logprob[i][-1][token_id].item(),
203 + atol=1e-2,
204 + rtol=1e-2)
205 + assert isinstance(
206 + sample_logprob.decoded_token,
207 + str), ("The token should be decoded by the time it is"
208 + " returned to the user.")
209 +
210 + # At this point we know the sample logprobs are correct for this
211 + # request. Validate that cumulative_logprob is actually the sum.
212 + # For each request, assert that the returned cumulative logprob
213 + # matches the correct value, which is computed below.
214 + torch.testing.assert_close(
215 + vllm_result.outputs[0].cumulative_logprob,
216 + compute_correct_cumulative_logprob(vllm_result.outputs[0]),
217 + atol=1e-6,
218 + rtol=1e-6)
219 + else:
220 + # Logprobs disabled for this request; should be None
221 + assert vllm_result.outputs[0].logprobs is None
222 +
223 + # Validate prompt logprobs
224 + if num_top_prompt_logprobs is not None:
225 + # Confirm that structure of prompt logprobs in result is correct
226 + assert vllm_result.prompt_logprobs is not None
227 + # - The first prompt logprob is always None
228 + assert vllm_result.prompt_logprobs[0] is None
229 + # - Prompt logprobs are returned for all indices in
230 + # the prompt
231 + assert len(vllm_result.prompt_logprobs) == len(
232 + vllm_result.prompt_token_ids)
233 + for prompt_logprobs, prompt_token_id in zip(
234 + vllm_result.prompt_logprobs[1:],
235 + vllm_result.prompt_token_ids[1:]):
236 + assert prompt_logprobs is not None
237 +
238 + # Confirm that the prompt token appears among the logprobs
239 + assert prompt_token_id in prompt_logprobs
240 + token_in_topk = prompt_logprobs[
241 + prompt_token_id].rank <= num_top_prompt_logprobs
242 +
243 + # If the prompt token is not included in the top K
244 + # logprob, it can return 1 more data
245 + if token_in_topk and num_top_prompt_logprobs != 0:
246 + assert len(prompt_logprobs) == num_top_prompt_logprobs
247 + else:
248 + assert len(prompt_logprobs) == num_top_prompt_logprobs + 1
249 +
250 + if num_top_prompt_logprobs > 0:
251 + # We should have an entry for each of the topk ranks
252 + all_ranks = {lp.rank for lp in prompt_logprobs.values()}
253 + assert all(r in all_ranks
254 + for r in range(1, num_top_prompt_logprobs + 1))
255 +
256 + # Compare prompt logprobs to HF
257 + # The first prompt logprob is always None, so we compare it from
258 + # 1:.
259 + vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:]
260 + for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs):
261 + for token_id, logprob in vllm_prompt_logprob_dict.items():
262 + torch.testing.assert_close(
263 + logprob.logprob,
264 + hf_logprob[0][i][token_id].item(),
265 + atol=2e-2,
266 + rtol=2e-2)
267 + else:
268 + assert vllm_result.prompt_logprobs is None
269 +
270 +
271 +#@pytest.mark.skip_global_cleanup
272 +@pytest.mark.parametrize("batch_logprobs_composition",
273 + ["NONE", "SAMPLE", "PROMPT", "SAMPLE_PROMPT"])
274 +@pytest.mark.parametrize("temperature", [0.0, 2.0])
275 +def test_get_logprobs_and_prompt_logprobs(
276 + hf_model,
277 + vllm_model,
278 + batch_logprobs_composition: str,
279 + temperature: float,
280 + example_prompts,
281 +) -> None:
282 + """Test V1 Engine logprobs & prompt logprobs
283 +
284 + Exercise a variety of combinations of `logprobs` and `prompt_logprobs`
285 + settings and validate that
286 + * The generated logprobs and prompt logprobs are consistent with the
287 + configuration settings, in terms of whether or not the logprobs
288 + (of either type) were requested and how many were requested
289 + * The generated logprobs are consistent with the generated tokens
290 + * The generated (prompt)logprobs are consistent with HuggingFace
291 + (prompt)logprobs, as a reference
292 +
293 + batch_logprobs_composition controls the logprobs configurations for
294 + requests in the batch under test.
295 +
296 + Args:
297 + hf_model
298 + vllm_model
299 + batch_logprobs_composition: logprobs configuration for test batch
300 + example_prompts
301 + monkeypatch
302 + """
303 + _test_case_get_logprobs_and_prompt_logprobs(
304 + hf_model=hf_model,
305 + vllm_model=vllm_model,
306 + batch_logprobs_composition=batch_logprobs_composition,
307 + temperature=temperature,
308 + example_prompts=example_prompts)
309 +
310 +
311 +def test_max_logprobs(monkeypatch):
312 + """vLLM v1 engine should fail a request with `logprobs > max_logprobs`
313 +
314 + Should also fail for `prompt_logprobs > max_logprobs`
315 +
316 + Args:
317 + monkeypatch
318 + """
319 + override_backend_env_variable(monkeypatch, "FLASH_ATTN")
320 +
321 + runner = VllmRunner("facebook/opt-125m",
322 + max_logprobs=1,
323 + enable_prefix_caching=False,
324 + max_model_len=256)
325 + vllm_sampling_params = SamplingParams(logprobs=1)
326 + # should pass
327 + runner.generate(["Hello world"], sampling_params=vllm_sampling_params)
328 +
329 + bad_sampling_params = SamplingParams(logprobs=2)
330 + with pytest.raises(ValueError):
331 + runner.generate(["Hello world"], sampling_params=bad_sampling_params)
332 +
333 +
334 +def test_none_logprobs(vllm_model, example_prompts, monkeypatch):
335 + """Engine should return `logprobs` and `prompt_logprobs` as `None`
336 +
337 + Args:
338 + vllm_model: vLLM model fixture
339 + example_prompts: list of example prompts (test fixture)
340 + monkeypatch: supports editing env vars and rolling back changes
341 + after the test
342 + """
343 + max_tokens = 5
344 +
345 + sampling_params_logprobs_none = SamplingParams(max_tokens=max_tokens,
346 + logprobs=None,
347 + prompt_logprobs=None,
348 + temperature=0.0)
349 + results_logprobs_none = vllm_model.model.generate(
350 + example_prompts, sampling_params=sampling_params_logprobs_none)
351 +
352 + for i in range(len(results_logprobs_none)):
353 + # Check sample logprobs are None
354 + assert results_logprobs_none[i].outputs[0].logprobs is None
355 + assert results_logprobs_none[i].outputs[0].cumulative_logprob is None
356 + # Check prompt logprobs are None
357 + assert results_logprobs_none[i].prompt_logprobs is None
358 +
359 +
360 +def test_zero_logprobs(vllm_model, example_prompts, monkeypatch):
361 + """Engine should return sampled token and prompt token logprobs
362 +
363 + Args:
364 + vllm_model: vLLM model fixture
365 + example_prompts: list of example prompts (test fixture)
366 + monkeypatch: supports editing env vars and rolling back changes
367 + after the test
368 + """
369 + max_tokens = 5
370 +
371 + sampling_params_logprobs_zero = SamplingParams(max_tokens=max_tokens,
372 + logprobs=0,
373 + prompt_logprobs=0,
374 + temperature=0.0)
375 + results_logprobs_zero = vllm_model.model.generate(
376 + example_prompts, sampling_params=sampling_params_logprobs_zero)
377 +
378 + for i in range(len(results_logprobs_zero)):
379 + # Check that there is one sample logprob dict for each
380 + # sample token
381 + logprobs = results_logprobs_zero[i].outputs[0].logprobs
382 + prompt_logprobs = results_logprobs_zero[i].prompt_logprobs
383 + sampled_token_ids = results_logprobs_zero[i].outputs[0].token_ids
384 + prompt_token_ids = results_logprobs_zero[i].prompt_token_ids
385 + assert logprobs is not None
386 + assert len(sampled_token_ids) == len(logprobs)
387 + assert results_logprobs_zero[i].outputs[
388 + 0].cumulative_logprob is not None
389 + # Check that there is one prompt logprob dict for each
390 + # prompt token
391 + assert prompt_logprobs is not None
392 + assert len(prompt_token_ids) == len(prompt_logprobs)
tests/v1/sample/test_logprobs_e2e.py new
+52
@@ -0,0 +1,52 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import lm_eval
4 +
5 +from ...utils import RemoteOpenAIServer
6 +
7 +# arc-easy uses prompt_logprobs=1, logprobs=1
8 +TASK = "arc_easy"
9 +FILTER = "acc_norm,none"
10 +RTOL = 0.03
11 +EXPECTED_VALUE = 0.62
12 +
13 +# FIXME(rob): enable prefix caching once supported.
14 +MODEL = "meta-llama/Llama-3.2-1B"
15 +MODEL_ARGS = f"pretrained={MODEL},enforce_eager=True,enable_prefix_caching=False" # noqa: E501
16 +SERVER_ARGS = [
17 + "--enforce_eager", "--no_enable_prefix_caching", "--disable-log-requests"
18 +]
19 +NUM_CONCURRENT = 100
20 +
21 +
22 +def test_prompt_logprobs_e2e():
23 + results = lm_eval.simple_evaluate(model="vllm",
24 + model_args=MODEL_ARGS,
25 + tasks=TASK,
26 + batch_size="auto")
27 +
28 + measured_value = results["results"][TASK][FILTER]
29 + assert (measured_value - RTOL < EXPECTED_VALUE
30 + and measured_value + RTOL > EXPECTED_VALUE
31 + ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
32 +
33 +
34 +def test_promt_logprobs_e2e_server():
35 + with RemoteOpenAIServer(MODEL, SERVER_ARGS) as remote_server:
36 + url = f"{remote_server.url_for('v1')}/completions"
37 +
38 + model_args = (
39 + f"model={MODEL},"
40 + f"base_url={url},"
41 + f"num_concurrent={NUM_CONCURRENT},tokenized_requests=False")
42 +
43 + results = lm_eval.simple_evaluate(
44 + model="local-completions",
45 + model_args=model_args,
46 + tasks=TASK,
47 + )
48 +
49 + measured_value = results["results"][TASK][FILTER]
50 + assert (measured_value - RTOL < EXPECTED_VALUE
51 + and measured_value + RTOL > EXPECTED_VALUE
52 + ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
tests/v1/sample/utils.py new
+120
@@ -0,0 +1,120 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import re
4 +from typing import List, Tuple
5 +
6 +from vllm import CompletionOutput
7 +
8 +
9 +def get_test_batch(batch_logprobs_composition: str) -> List[Tuple]:
10 + """Generate logprobs configs for a batch of requests
11 +
12 + A given request's logprobs configuration is (1) num_sample_logprobs and (2)
13 + num_prompt_logprobs. The batch logprobs configuration is the list of request
14 + logprobs configs.
15 +
16 + batch_logprobs_composition == "NONE" yields a batch with no sample or prompt
17 + logprobs
18 +
19 + batch_logprobs_composition == "SAMPLE" yields a batch with some requests
20 + configured for sample logprobs only, and others configured for no logprobs
21 +
22 + batch_logprobs_composition == "PROMPT" yields a batch with some requests
23 + configured for prompt logprobs only, and others configured for no logprobs
24 +
25 + batch_logprobs_composition == "SAMPLE_PROMPT" yields a batch with some
26 + requests configured for sample logprobs and prompt logprobs, some configured
27 + for only sample logprobs or only prompt logprobs, and some configured for
28 + no logprobs
29 +
30 + Args:
31 + batch_logprobs_composition: types of logprobs configs to include in batch
32 +
33 + Returns:
34 +
35 + List of (Optional[num_sample_logprobs], Optional[num_prompt_logprobs])
36 + tuples
37 + """
38 + if batch_logprobs_composition == "NONE":
39 + # No requests with sample or prompt logprobs
40 + return [(None, None)]
41 + elif batch_logprobs_composition == "SAMPLE":
42 + # Requests requiring sample logprobs or no logprobs
43 + return [
44 + (None, None),
45 + (0, None),
46 + (5, None),
47 + (3, None),
48 + ]
49 + elif batch_logprobs_composition == "PROMPT":
50 + # Requests requiring prompt logprobs or no logprobs
51 + return [
52 + (None, None),
53 + (None, 0),
54 + (None, 6),
55 + (None, 5),
56 + ]
57 + elif batch_logprobs_composition == "SAMPLE_PROMPT":
58 + # Requests requiring either no logprobs, just
59 + # sample logprobs, just prompt logprobs, or
60 + # both sample and prompt logprobs
61 + return [
62 + (None, None),
63 + (0, None),
64 + (5, None),
65 + (3, None),
66 + (0, 3),
67 + (6, 0),
68 + (6, 3),
69 + (None, 6),
70 + (None, 5),
71 + (None, 0),
72 + ]
73 + else:
74 + raise ValueError("Invalid logprobs batch configuration for test.")
75 +
76 +
77 +def assert_incr_detok_str_matches_non_incr_detok_str(
78 + incremental_detokenization_str: str,
79 + non_incremental_detokenization_str: str,
80 + msg: str,
81 +) -> None:
82 + """Compare incrementally detok. text to non-incrementally detok. text
83 +
84 + Fail if the strings mismatch after non-alphanumeric characters are stripped
85 + out.
86 +
87 + Rationale: incremental detokenization in the text generation process allows
88 + the tokenizer to adjust the next token text output based on the token's
89 + context in the string. However, logprobs detokenization detokenizes each
90 + token individually, and the resultant strings may include some
91 + non-alphanumeric placeholder characters where there could be i.e.
92 + whitespace. So, this function compares only the alphanumeric text
93 + between two strings and fails if there is a mismatch, which helps
94 + with validating logprobs detokenization.
95 +
96 + Args:
97 + incremental_detokenization_str: incrementally-detokenized generated text
98 + non_incremental_detokenization_str: non-incrementally-detokenized logprob
99 + tokens
100 + msg: error message if `assert` fails
101 + """
102 + rgx = r'[^a-zA-Z0-9]+'
103 + assert (re.sub(rgx, '', incremental_detokenization_str) == re.sub(
104 + rgx, '', non_incremental_detokenization_str)), (msg)
105 +
106 +
107 +def compute_correct_cumulative_logprob(
108 + completion_output: CompletionOutput) -> float:
109 + """Compute known-good value for evaluating cumulative logprob
110 +
111 + Args:
112 + completion_output: completion output from engine
113 +
114 + Returns:
115 + Known-good cumulative logprob value
116 + """
117 + token_ids = completion_output.token_ids
118 + logprobs = completion_output.logprobs
119 + assert logprobs is not None
120 + return sum([lp[tok_id].logprob for tok_id, lp in zip(token_ids, logprobs)])
vllm/outputs.py
+6 -4
@@ -142,6 +142,9 @@ class RequestOutput:
142 prompt_token_ids: Optional[List[int]],
143 text: str,
144 token_ids: List[int],
145 + logprobs: Optional[SampleLogprobs],
146 + prompt_logprobs: Optional[PromptLogprobs],
147 + cumulative_logprob: Optional[float],
148 finished: bool = False,
149 ) -> "RequestOutput":
150 """Initialize a new RequestOutput object."""
@@ -151,15 +154,14 @@ class RequestOutput:
154 index=0,
155 text=text,
156 token_ids=token_ids,
154 - cumulative_logprob=None,
155 - logprobs=None, # TODO
156 - )
157 + cumulative_logprob=cumulative_logprob,
158 + logprobs=logprobs)
159
160 return RequestOutput(
161 request_id=request_id,
162 prompt=prompt,
163 prompt_token_ids=prompt_token_ids,
162 - prompt_logprobs=None, # TODO
164 + prompt_logprobs=prompt_logprobs,
165 outputs=[completion_output],
166 finished=finished,
167 )
vllm/transformers_utils/detokenizer_utils.py
+19
@@ -74,6 +74,25 @@ def convert_prompt_ids_to_tokens(
74 return new_tokens, prefix_offset, read_offset
75
76
77 +def convert_ids_list_to_tokens(
78 + tokenizer: AnyTokenizer,
79 + token_ids: List[int],
80 +) -> List[str]:
81 + """Detokenize the input ids individually.
82 +
83 + Args:
84 + tokenizer: tokenizer used by model under test
85 + token_ids: convert these tokens (Python list form)
86 +
87 + Returns:
88 + Python list of token string representations
89 +
90 + """
91 + token_str_lst = tokenizer.convert_ids_to_tokens(token_ids)
92 + _replace_none_with_empty(token_str_lst) # type: ignore
93 + return token_str_lst
94 +
95 +
96 # Based on
97 # https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
98 # under Apache 2.0 license
vllm/v1/core/scheduler.py
+31 -12
@@ -437,6 +437,8 @@ class Scheduler:
437 ) -> EngineCoreOutputs:
438 # NOTE(woosuk): This method doesn't consider speculative decoding.
439 sampled_token_ids = model_runner_output.sampled_token_ids
440 + logprobs = model_runner_output.logprobs
441 + prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
442 num_scheduled_tokens = scheduler_output.num_scheduled_tokens
443 new_running: List[Request] = []
444 outputs: List[EngineCoreOutput] = []
@@ -471,6 +473,13 @@ class Scheduler:
473 self.encoder_cache_manager.free_encoder_input(
474 request, input_id)
475
476 + # Get prompt logprobs for this request.
477 + prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
478 +
479 + stopped = False
480 + new_logprobs = None
481 + new_token_ids = None
482 +
483 if request.num_computed_tokens == request.num_tokens:
484 req_index = model_runner_output.req_id_to_index[req_id]
485 # NOTE(woosuk): Currently, we assume that each request
@@ -486,20 +495,30 @@ class Scheduler:
495 if stopped:
496 self._free_request(request)
497
498 + # Extract sample logprobs if needed.
499 + if request.sampling_params.logprobs is not None:
500 + assert logprobs is not None
501 + # NOTE: once we support N tokens per step (spec decode),
502 + # the outer lists can be of length > 1.
503 + new_logprobs = logprobs.slice(req_index, req_index + 1)
504 +
505 + new_token_ids = request.output_token_ids[-num_new_tokens:]
506 +
507 + # Transmit partial if chunked prefill & prompt logprobs is enabled
508 + if new_token_ids or prompt_logprobs_tensors is not None:
509 # Add EngineCoreOutput for this Request.
490 - output = EngineCoreOutput(
491 - request_id=req_id,
492 - new_token_ids=request.output_token_ids[-num_new_tokens:],
493 - finished=request.is_finished(),
494 - finish_reason=request.get_finished_reason(),
495 - stop_reason=request.stop_reason)
496 - outputs.append(output)
497 -
498 - # Breakout of the loop.
499 - if stopped:
500 - continue
510 + outputs.append(
511 + EngineCoreOutput(
512 + request_id=req_id,
513 + new_token_ids=new_token_ids or [],
514 + finish_reason=request.get_finished_reason(),
515 + new_logprobs=new_logprobs,
516 + new_prompt_logprobs_tensors=prompt_logprobs_tensors,
517 + stop_reason=request.stop_reason))
518 +
519 + if not stopped:
520 + new_running.append(request)
521
502 - new_running.append(request)
522 self.running = new_running
523 return EngineCoreOutputs(
524 outputs=outputs,
vllm/v1/engine/__init__.py
+9 -1
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
7 import msgspec
8
9 from vllm.v1.metrics.stats import SchedulerStats
10 +from vllm.v1.outputs import LogprobsLists, LogprobsTensors
11
12 if TYPE_CHECKING:
13 from vllm.lora.request import LoRARequest
@@ -67,10 +68,17 @@ class EngineCoreOutput(
68
69 request_id: str
70 new_token_ids: List[int]
70 - finished: bool
71 +
72 + new_logprobs: Optional[LogprobsLists] = None
73 + new_prompt_logprobs_tensors: Optional[LogprobsTensors] = None
74 +
75 finish_reason: Optional[FinishReason] = None
76 stop_reason: Union[int, str, None] = None
77
78 + @property
79 + def finished(self) -> bool:
80 + return self.finish_reason is not None
81 +
82
83 class EngineCoreOutputs(
84 msgspec.Struct,
vllm/v1/engine/core.py
+2 -3
@@ -11,7 +11,6 @@ from typing import List, Tuple, Type
11 import psutil
12 import zmq
13 import zmq.asyncio
14 -from msgspec import msgpack
14
15 from vllm.config import VllmConfig
16 from vllm.logger import init_logger
@@ -26,7 +25,7 @@ from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
25 from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
26 from vllm.v1.executor.abstract import Executor
27 from vllm.v1.request import Request, RequestStatus
29 -from vllm.v1.serial_utils import PickleEncoder
28 +from vllm.v1.serial_utils import MsgpackEncoder, PickleEncoder
29 from vllm.version import __version__ as VLLM_VERSION
30
31 logger = init_logger(__name__)
@@ -292,7 +291,7 @@ class EngineCoreProc(EngineCore):
291 """Output socket IO thread."""
292
293 # Msgpack serialization encoding.
295 - encoder = msgpack.Encoder()
294 + encoder = MsgpackEncoder()
295 # Reuse send buffer.
296 buffer = bytearray()
297
vllm/v1/engine/core_client.py
+2 -3
@@ -7,7 +7,6 @@ import weakref
7 from abc import ABC, abstractmethod
8 from typing import List, Optional, Type
9
10 -import msgspec
10 import zmq
11 import zmq.asyncio
12
@@ -20,7 +19,7 @@ from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
19 EngineCoreRequestUnion, EngineCoreResetPrefixCache)
20 from vllm.v1.engine.core import EngineCore, EngineCoreProc
21 from vllm.v1.executor.abstract import Executor
23 -from vllm.v1.serial_utils import PickleEncoder
22 +from vllm.v1.serial_utils import MsgpackDecoder, PickleEncoder
23 from vllm.v1.utils import BackgroundProcHandle
24
25 logger = init_logger(__name__)
@@ -163,7 +162,7 @@ class MPClient(EngineCoreClient):
162
163 # Serialization setup.
164 self.encoder = PickleEncoder()
166 - self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
165 + self.decoder = MsgpackDecoder(EngineCoreOutputs)
166
167 # ZMQ setup.
168 self.ctx = (
vllm/v1/engine/detokenizer.py
+12 -42
@@ -1,27 +1,17 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from dataclasses import dataclass
4 -from typing import List, Optional, Union
4 +from typing import List, Optional
5
6 from vllm.engine.output_processor.stop_checker import StopChecker
7 from vllm.logger import init_logger
8 -from vllm.sampling_params import RequestOutputKind
8 from vllm.transformers_utils.detokenizer_utils import (
9 AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
11 -from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
10 +from vllm.v1.engine import EngineCoreRequest
11
12 logger = init_logger(__name__)
13
14
16 -@dataclass
17 -class DetokenizerOutput:
18 - output_text: str
19 - token_ids: List[int]
20 - finished: bool
21 - finish_reason: Optional[FinishReason] = None
22 - stop_reason: Union[int, str, None] = None
23 -
24 -
15 @dataclass
16 class IncrementalDetokenizer:
17
@@ -42,7 +32,6 @@ class IncrementalDetokenizer:
32 # Parameters for detokenization
33 skip_special_tokens: bool
34 spaces_between_special_tokens: bool
45 - output_kind: RequestOutputKind
35
36 # Tokenizer for this request
37 tokenizer: AnyTokenizer
@@ -90,25 +79,19 @@ class IncrementalDetokenizer:
79 skip_special_tokens=request.sampling_params.skip_special_tokens,
80 spaces_between_special_tokens=request.sampling_params.
81 spaces_between_special_tokens,
93 - output_kind=request.sampling_params.output_kind,
82 prompt_len=len(request.prompt_token_ids),
83 tokenizer=tokenizer,
84 stop_buffer_length=stop_buffer_length,
85 )
86
99 - def update_from_output(
100 - self,
101 - output: EngineCoreOutput,
102 - ) -> Optional[DetokenizerOutput]:
87 + def update(self, new_token_ids: List[int]) -> Optional[str]:
88 """
89 Update RequestState for the request_id by:
90 1) Detokenize the new token ids incrementally.
106 - 2) Update the RequestOutput with the new text.
107 - """
91 + 2) Evaluate stop criteria.
92
109 - new_token_ids = output.new_token_ids
110 - finish_reason = output.finish_reason
111 - stop_reason = output.stop_reason
93 + Return matched stop string or None.
94 + """
95
96 # 1) Detokenize the new token ids incrementally.
97 # TODO(woosuk): This method becomes very inefficient when the number of
@@ -131,11 +114,13 @@ class IncrementalDetokenizer:
114 self.tokens.extend(new_tokens)
115 self.prefix_offset = prefix_offset
116 self.read_offset = read_offset
134 - self.output_text += new_decoded_token_text
117
118 decoded_text += new_decoded_token_text
119
120 + self.output_text += decoded_text
121 +
122 # 2) Evaluate stop criteria.
123 + stop_string = None
124 if self.stop:
125 stop = StopChecker.check_stop_strings(
126 output_text=self.output_text,
@@ -144,28 +129,13 @@ class IncrementalDetokenizer:
129 include_in_output=self.include_stop_str_in_output,
130 )
131 if stop is not None:
147 - stop_str, truncate_to = stop
132 + stop_string, truncate_to = stop
133 if truncate_to != -1:
134 self.output_text = self.output_text[:truncate_to]
150 - finish_reason = FinishReason.STOP
151 - stop_reason = stop_str
152 -
153 - # TODO: handle stop_token_ids here too?
154 -
155 - # 3) Update the RequestOutput object with the new text.
156 - finished = finish_reason is not None
157 - if self.output_kind == RequestOutputKind.FINAL_ONLY \
158 - and not finished:
159 - return None
160 -
161 - delta = self.output_kind == RequestOutputKind.DELTA
162 - output_text = self._get_next_output_text(finished, delta)
163 - token_ids = new_token_ids if delta else self.output_token_ids
135
165 - return DetokenizerOutput(output_text, token_ids, finished,
166 - finish_reason, stop_reason)
136 + return stop_string
137
168 - def _get_next_output_text(self, finished: bool, delta: bool) -> str:
138 + def get_next_output_text(self, finished: bool, delta: bool) -> str:
139 """If delta is True, only new text since the last call to
140 this method is returned"""
141
vllm/v1/engine/llm_engine.py
+1
@@ -45,6 +45,7 @@ class LLMEngine:
45 multiprocess_mode: bool = False,
46 ) -> None:
47 self.model_config = vllm_config.model_config
48 + self.cache_config = vllm_config.cache_config
49
50 # Tokenizer (+ ensure liveness if running in another process).
51 self.tokenizer = init_tokenizer_from_configs(
vllm/v1/engine/logprobs.py new
+194
@@ -0,0 +1,194 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +import itertools
4 +from dataclasses import dataclass
5 +from typing import Dict, List, Optional
6 +
7 +from vllm.logger import init_logger
8 +from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9 +from vllm.transformers_utils.detokenizer_utils import (
10 + AnyTokenizer, convert_ids_list_to_tokens)
11 +from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
12 +from vllm.v1.outputs import LogprobsLists, LogprobsTensors
13 +
14 +logger = init_logger(__name__)
15 +
16 +
17 +@dataclass
18 +class LogprobsProcessor:
19 +
20 + # Tokenizer for this request
21 + tokenizer: AnyTokenizer
22 +
23 + # Logprobs for this request
24 + logprobs: Optional[SampleLogprobs]
25 + prompt_logprobs: Optional[PromptLogprobs]
26 + cumulative_logprob: Optional[float]
27 + num_logprobs: Optional[int]
28 + num_prompt_logprobs: Optional[int]
29 +
30 + @classmethod
31 + def from_new_request(
32 + cls,
33 + tokenizer: AnyTokenizer,
34 + request: EngineCoreRequest,
35 + ) -> "LogprobsProcessor":
36 + num_logprobs = request.sampling_params.logprobs
37 + num_prompt_logprobs = request.sampling_params.prompt_logprobs
38 + return cls(
39 + tokenizer=tokenizer,
40 + cumulative_logprob=(None if num_logprobs is None else 0.),
41 + logprobs=(None if num_logprobs is None else []),
42 + # NOTE: logprob of first prompt token is None.
43 + prompt_logprobs=(None if num_prompt_logprobs is None else [None]),
44 + num_prompt_logprobs=num_prompt_logprobs,
45 + num_logprobs=num_logprobs,
46 + )
47 +
48 + def _update_sample_logprobs(self, logprobs_lists: LogprobsLists) -> None:
49 + """Update with sample logprobs from EngineCore.
50 +
51 + Outer lists are only of len > 1 if EngineCore made
52 + >1 tokens in prior step (e.g. in spec decoding).
53 +
54 + Args:
55 + logprobs_lists: the lists of logprob tokens, logprobs, and ranks.
56 +
57 + """
58 +
59 + assert self.num_logprobs is not None
60 + assert self.logprobs is not None
61 + assert self.cumulative_logprob is not None
62 +
63 + token_ids_lst, logprobs_lst, ranks_lst = logprobs_lists
64 +
65 + for rank, logprobs, token_ids in zip(ranks_lst, logprobs_lst,
66 + token_ids_lst):
67 +
68 + # Detokenize (non-incrementally).
69 + decoded_tokens = convert_ids_list_to_tokens(
70 + self.tokenizer, token_ids)
71 +
72 + # Sampler puts the sampled logprob in first.
73 + sampled_token_logprob = logprobs[0]
74 + self.cumulative_logprob += sampled_token_logprob
75 +
76 + # Update with the Logprob dictionary for this pos.
77 + self.logprobs.append(
78 + self._make_logprob_dict(
79 + logprobs,
80 + token_ids,
81 + decoded_tokens,
82 + rank,
83 + self.num_logprobs,
84 + ))
85 +
86 + def _update_prompt_logprobs(
87 + self,
88 + prompt_logprobs_tensors: LogprobsTensors,
89 + ) -> None:
90 + """Update with prompt logprobs from EngineCore.
91 +
92 + Args:
93 + prompt_logprobs_tensors: tuple containing the prompt logprobs
94 + tensors.
95 +
96 + """
97 +
98 + # Prompt logprobs are enabled.
99 + assert self.num_prompt_logprobs is not None
100 + assert self.prompt_logprobs is not None
101 +
102 + token_ids, logprobs, ranks = prompt_logprobs_tensors
103 +
104 + # Detokenize non-incrementally.
105 + # Output is flat: [num_tok, num_lps] -> [num_tok * num_lps]
106 + decoded_tokens = convert_ids_list_to_tokens(
107 + self.tokenizer,
108 + token_ids.flatten().tolist())
109 +
110 + # Recover shapes.
111 + num_prompt_tokens, num_logprobs = logprobs.shape
112 +
113 + # Pythonize the torch tensors.
114 + # TODO(rob): experiment with doing this in EngineCore?
115 + prompt_token_ranks = ranks.tolist()
116 + prompt_logprobs = logprobs.tolist()
117 + token_ids = token_ids.tolist()
118 +
119 + # Make Logprob for each position.
120 + for pos in range(num_prompt_tokens):
121 + # Handle flattening.
122 + offset = pos * num_logprobs
123 + offset_end = offset + num_logprobs
124 + decoded_tokens_for_pos = decoded_tokens[offset:offset_end]
125 +
126 + # Update with the Logprob dictionary for this pos.
127 + self.prompt_logprobs.append(
128 + self._make_logprob_dict(prompt_logprobs[pos], token_ids[pos],
129 + decoded_tokens_for_pos,
130 + prompt_token_ranks[pos],
131 + self.num_prompt_logprobs))
132 +
133 + def pop_prompt_logprobs(self) -> Optional[PromptLogprobs]:
134 + """Pop and return all request prompt logprobs
135 +
136 + The logprobs processor aggregates prompt chunk logprobs
137 + over one or more prefill chunks. This method returns
138 + all prompt logprobs at once and then forgets them.
139 + Ensures correct RequestOutputKind.DELTA semantics
140 + wherein all prompt logprobs are returned at once at
141 + the end of prefill.
142 +
143 + Returns:
144 + None if prompt logprobs are disabled for this request.
145 + List of all prompt logprobs, otherwise.
146 + """
147 + plp = self.prompt_logprobs
148 + if plp:
149 + self.prompt_logprobs = []
150 + return plp
151 +
152 + @staticmethod
153 + def _make_logprob_dict(
154 + logprobs: List[float],
155 + logprob_token_ids: List[int],
156 + decoded_tokens: List[str],
157 + rank: int,
158 + num_logprobs: int,
159 + ) -> Dict[int, Logprob]:
160 + """Make a Logprob dictionary for a position.
161 +
162 + Args:
163 + logprobs: list of log probabilities
164 + logprob_token_ids: list of top token ids
165 + decoded_tokens: list of decoded top tokens
166 + rank: rank of the sampled token
167 + num_logprobs: number of logprobs requested
168 + by the user (in addition to sampled logprob)
169 +
170 + Returns:
171 + Dict[token id, Logprob]
172 + """
173 +
174 + # We do not need a special case for the sampled token
175 + # being in the topk, since inserting duplicated data
176 + # into a dictionary twice is the same as doing it once.
177 + topk_ranks = range(1, num_logprobs + 1)
178 + ranks = itertools.chain((rank, ), topk_ranks)
179 +
180 + return {
181 + token_id: Logprob(
182 + logprob=logprob,
183 + rank=rank,
184 + decoded_token=token,
185 + )
186 + for token_id, logprob, rank, token in zip(
187 + logprob_token_ids, logprobs, ranks, decoded_tokens)
188 + }
189 +
190 + def update_from_output(self, output: EngineCoreOutput) -> None:
191 + if output.new_logprobs is not None:
192 + self._update_sample_logprobs(output.new_logprobs)
193 + if output.new_prompt_logprobs_tensors is not None:
194 + self._update_prompt_logprobs(output.new_prompt_logprobs_tensors)
vllm/v1/engine/output_processor.py
+88 -38
@@ -5,11 +5,12 @@ from dataclasses import dataclass
5 from typing import Dict, List, Optional
6
7 from vllm.outputs import RequestOutput
8 -from vllm.transformers_utils.detokenizer_utils import AnyTokenizer
8 +from vllm.sampling_params import RequestOutputKind
9 +from vllm.transformers_utils.tokenizer import AnyTokenizer
10 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
10 -from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
11 -from vllm.v1.engine.detokenizer import (DetokenizerOutput,
12 - IncrementalDetokenizer)
11 +from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
12 +from vllm.v1.engine.detokenizer import IncrementalDetokenizer
13 +from vllm.v1.engine.logprobs import LogprobsProcessor
14 from vllm.v1.metrics.stats import IterationStats, RequestStateStats
15
16
@@ -26,16 +27,20 @@ class RequestState:
27 def __init__(
28 self,
29 request_id: str,
30 + output_kind: RequestOutputKind,
31 prompt: Optional[str],
32 prompt_token_ids: List[int],
33 + logprobs_processor: LogprobsProcessor,
34 detokenizer: IncrementalDetokenizer,
35 arrival_time: float,
36 queue: Optional[asyncio.Queue[RequestOutput]],
37 ):
38 self.request_id = request_id
39 + self.output_kind = output_kind
40 self.prompt = prompt
41 self.prompt_token_ids = prompt_token_ids
42 self.prompt_len = len(prompt_token_ids)
43 + self.logprobs_processor = logprobs_processor
44 self.detokenizer = detokenizer
45 self.is_prefilling = True
46 self.queue = queue
@@ -51,8 +56,13 @@ class RequestState:
56 ) -> "RequestState":
57 return cls(
58 request_id=request.request_id,
59 + output_kind=request.sampling_params.output_kind,
60 prompt=request.prompt,
61 prompt_token_ids=request.prompt_token_ids,
62 + logprobs_processor=LogprobsProcessor.from_new_request(
63 + tokenizer=tokenizer,
64 + request=request,
65 + ),
66 detokenizer=IncrementalDetokenizer.from_new_request(
67 tokenizer=tokenizer,
68 request=request,
@@ -127,13 +137,8 @@ class OutputProcessor:
137 batch to ensure system overheads are minimized. This is the
138 only function that should loop over EngineCoreOutputs.
139
130 - If you need to touch every element of the batch, implement a
131 - method called XXXClass.update_from_output() to be called
132 - within the loop below. For examples, see:
133 - * IterationStats.update_from_output()
134 - * Detokenizer.update_from_output()
135 -
136 - TODO(rob): add Protocol makes update_from_output explicit.
140 + If you need to touch every element of the batch, do it from
141 + within the loop below.
142
143 **********************************************************
144 """
@@ -154,17 +159,37 @@ class OutputProcessor:
159 req_state.is_prefilling,
160 req_state.prompt_len,
161 req_state.stats)
157 - req_state.is_prefilling = False
158 -
159 - # 2) Detokenize the token ids into text.
160 - detokenizer_output = req_state.detokenizer.update_from_output(
161 - engine_core_output)
162 -
163 - # 3) Create and handle RequestOutput objects.
164 - if detokenizer_output is not None:
165 - request_output = self._make_request_output(
166 - req_state, detokenizer_output)
162
163 + new_token_ids = engine_core_output.new_token_ids
164 + finish_reason = engine_core_output.finish_reason
165 +
166 + # TODO(andy): prompt logprobs + chunked prefill can
167 + # result in engine core returning an output for a
168 + # partial prefill (in order to send back partial
169 + # prompt logprobs.) This breaks the invariant that
170 + # process_outputs is only operating on engine core
171 + # outputs associated with non-partial completions.
172 + # Currently this is handled by having `is_prefilling`
173 + # check for new decoded tokens, indicating that
174 + # the completion is not partial.
175 + #
176 + # Follow up will aggregate partial prompt logprobs
177 + # in the EngineCore.
178 + req_state.is_prefilling = not new_token_ids
179 +
180 + # 2) Detokenize the token ids into text and check for stop
181 + # strings.
182 + stop_reason = req_state.detokenizer.update(new_token_ids)
183 + if stop_reason:
184 + finish_reason = FinishReason.STOP
185 +
186 + # 3) Compute sample and prompt logprobs for request,
187 + # if required.
188 + req_state.logprobs_processor.update_from_output(engine_core_output)
189 +
190 + # 4) Create and handle RequestOutput objects.
191 + if request_output := self._make_request_output(
192 + req_state, new_token_ids, finish_reason, stop_reason):
193 if req_state.queue is not None:
194 # AsyncLLM: put into queue for handling by generate().
195 req_state.queue.put_nowait(request_output)
@@ -174,18 +199,16 @@ class OutputProcessor:
199
200 # Free completed requests.
201 if request_output.finished:
177 - assert detokenizer_output.finish_reason is not None
178 -
202 self.request_states.pop(req_id)
203 if not engine_core_output.finished:
204 # If req not finished in EngineCore, but Detokenizer
205 # detected stop string, abort needed in EngineCore.
206 reqs_to_abort.append(req_id)
207
185 - # Track per-request stats
208 + # Track per-request stats.
209 + assert finish_reason is not None
210 iteration_stats.update_from_finished_request(
187 - detokenizer_output.finish_reason, request_output,
188 - req_state.stats)
211 + finish_reason, request_output, req_state.stats)
212
213 return OutputProcessorOutput(
214 request_outputs=request_outputs,
@@ -196,20 +219,47 @@ class OutputProcessor:
219 @staticmethod
220 def _make_request_output(
221 request_state: RequestState,
199 - detokenizer_output: DetokenizerOutput,
200 - ) -> RequestOutput:
222 + new_token_ids: List[int],
223 + finish_reason: Optional[FinishReason],
224 + stop_reason: Optional[str],
225 + ) -> Optional[RequestOutput]:
226 +
227 + finished = finish_reason is not None
228 + output_kind = request_state.output_kind
229 + # In follow up, we will switch to invariant where EngineCore
230 + # does not stream partial prefills.
231 + if not finished and (request_state.is_prefilling
232 + or output_kind == RequestOutputKind.FINAL_ONLY):
233 + # Only the final output is required in FINAL_ONLY mode.
234 + return None
235 +
236 + detokenizer = request_state.detokenizer
237 + logprobs_processor = request_state.logprobs_processor
238 +
239 + delta = output_kind == RequestOutputKind.DELTA
240 + logprobs = logprobs_processor.logprobs
241 + if delta:
242 + if logprobs:
243 + logprobs = logprobs[-len(new_token_ids):]
244 + # Side effect: logprobs processor forgets prompt logprobs
245 + prompt_logprobs = logprobs_processor.pop_prompt_logprobs()
246 + else:
247 + prompt_logprobs = logprobs_processor.prompt_logprobs
248 +
249 request_output = RequestOutput.new(
202 - request_state.request_id,
203 - request_state.prompt,
204 - request_state.prompt_token_ids,
205 - detokenizer_output.output_text,
206 - detokenizer_output.token_ids,
207 - detokenizer_output.finished,
250 + request_id=request_state.request_id,
251 + prompt=request_state.prompt,
252 + prompt_token_ids=request_state.prompt_token_ids,
253 + text=detokenizer.get_next_output_text(finished, delta),
254 + token_ids=new_token_ids if delta else detokenizer.output_token_ids,
255 + logprobs=logprobs,
256 + prompt_logprobs=prompt_logprobs,
257 + cumulative_logprob=logprobs_processor.cumulative_logprob,
258 + finished=finished,
259 )
209 - if detokenizer_output.finished:
260 + if finished:
261 completion_output = request_output.outputs[0]
211 - completion_output.finish_reason = str(
212 - detokenizer_output.finish_reason)
213 - completion_output.stop_reason = detokenizer_output.stop_reason
262 + completion_output.finish_reason = str(finish_reason)
263 + completion_output.stop_reason = stop_reason
264
265 return request_output
vllm/v1/engine/processor.py
+35 -4
@@ -33,6 +33,7 @@ class Processor:
33 ):
34
35 self.model_config = model_config
36 + self.cache_config = cache_config
37 self.lora_config = lora_config
38 self.tokenizer = tokenizer
39
@@ -51,6 +52,37 @@ class Processor:
52 self.use_hash = (not model_config.disable_mm_preprocessor_cache) or \
53 cache_config.enable_prefix_caching
54
55 + def _validate_logprobs(
56 + self,
57 + params: Union[SamplingParams, PoolingParams],
58 + ) -> None:
59 + if not isinstance(params, SamplingParams):
60 + return
61 +
62 + max_logprobs = self.model_config.max_logprobs
63 + # Validate sample logprobs.
64 + if params.logprobs and params.logprobs > max_logprobs:
65 + raise ValueError(
66 + f"Requested sample logprobs of {params.logprobs}, "
67 + f"which is greater than max allowed: {max_logprobs}")
68 +
69 + # Validate prompt logprobs.
70 + if params.prompt_logprobs and params.prompt_logprobs > max_logprobs:
71 + raise ValueError(
72 + f"Requested prompt logprobs of {params.prompt_logprobs}, "
73 + f"which is greater than max allowed: {max_logprobs}")
74 +
75 + # TODO(andy): enable this in follow up by recomputing.
76 + if (params.prompt_logprobs is not None
77 + and self.cache_config.enable_prefix_caching):
78 + raise ValueError("Prefix caching with prompt logprobs not yet "
79 + "supported on VLLM V1.")
80 +
81 + def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:
82 + if lora_request is not None and not self.lora_config:
83 + raise ValueError(f"Got lora_request {lora_request} but LoRA is "
84 + "not enabled!")
85 +
86 def process_inputs(
87 self,
88 request_id: str,
@@ -64,12 +96,11 @@ class Processor:
96 ) -> EngineCoreRequest:
97
98 # TODO(woosuk): Support pooling models.
67 - # TODO(woosuk): Check max_logprobs
99 # TODO(woosuk): Support encoder-decoder models.
100
70 - if lora_request is not None and not self.lora_config:
71 - raise ValueError(f"Got lora_request {lora_request} but LoRA is "
72 - "not enabled!")
101 + self._validate_logprobs(params)
102 + self._validate_lora(lora_request)
103 +
104 if arrival_time is None:
105 arrival_time = time.time()
106 assert priority == 0, "vLLM V1 does not support priority at the moment."
vllm/v1/metrics/stats.py
+11 -8
@@ -60,14 +60,17 @@ class IterationStats:
60
61 self.num_generation_tokens += num_new_generation_tokens
62 if is_prefilling:
63 - # This relies on the invariant that EngineCore does
64 - # not stream outputs for partially completed prefills
65 - # (scheduler.update_from_output makes EngineCoreOutput
66 - # iff num_computed_tokens == num_tokens).
67 - assert (num_new_generation_tokens > 0)
68 - self.num_prompt_tokens += prompt_len
69 -
70 - self.time_to_first_tokens_iter.append(last_token_latency)
63 + # TODO(andy): we used to assert that num_new_generation_tokens
64 + # > 0 with an invariant that EngineCore does not stream outputs
65 + # for partially completed prefills (scheduler.update_from_output
66 + # makes EngineCoreOutput iff num_computed_tokens == num_tokens).
67 + # When prompt logprobs are enabled, we currently stream out the
68 + # partially completed prompt.
69 + # This will be reverted in a follow up PR and we should re-enable
70 + # this assertion / invariant.
71 + if num_new_generation_tokens > 0:
72 + self.num_prompt_tokens += prompt_len
73 + self.time_to_first_tokens_iter.append(last_token_latency)
74 else:
75 self.time_per_output_tokens_iter.append(last_token_latency)
76
vllm/v1/outputs.py
+43 -11
@@ -1,25 +1,51 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from dataclasses import dataclass
4 -from typing import Dict, List, Optional
4 +from typing import Dict, List, NamedTuple, Optional
5
6 import torch
7
8
9 -@dataclass
10 -class SamplerOutput:
9 +class LogprobsLists(NamedTuple):
10
11 + # [num_reqs, max_num_logprobs + 1]
12 + logprob_token_ids: List[List[int]]
13 + # [num_reqs, max_num_logprobs + 1]
14 + logprobs: List[List[float]]
15 # [num_reqs]
13 - sampled_token_ids: torch.Tensor
16 + sampled_token_ranks: List[int]
17 +
18 + def slice(self, start: int, end: int):
19 + return LogprobsLists(
20 + self.logprob_token_ids[start:end],
21 + self.logprobs[start:end],
22 + self.sampled_token_ranks[start:end],
23 + )
24 +
25 +
26 +class LogprobsTensors(NamedTuple):
27
28 # [num_reqs, max_num_logprobs + 1]
16 - logprob_token_ids: Optional[torch.Tensor]
29 + logprob_token_ids: torch.Tensor
30 # [num_reqs, max_num_logprobs + 1]
18 - logprobs: Optional[torch.Tensor]
31 + logprobs: torch.Tensor
32 + # [num_reqs]
33 + selected_token_ranks: torch.Tensor
34
20 - # TODO: Support prompt logprobs.
21 - prompt_logprob_token_ids: Optional[torch.Tensor]
22 - prompt_logprobs: Optional[torch.Tensor]
35 + def tolists(self):
36 + return LogprobsLists(
37 + self.logprob_token_ids.tolist(),
38 + self.logprobs.tolist(),
39 + self.selected_token_ranks.tolist(),
40 + )
41 +
42 +
43 +@dataclass
44 +class SamplerOutput:
45 +
46 + # [num_reqs]
47 + sampled_token_ids: torch.Tensor
48 + logprobs_tensors: Optional[LogprobsTensors]
49
50
51 # ModelRunnerOutput is serialized and sent to the scheduler process.
@@ -36,6 +62,12 @@ class ModelRunnerOutput:
62 sampled_token_ids: List[int]
63
64 # [num_reqs, max_num_logprobs + 1]
39 - logprob_token_ids_cpu: Optional[torch.Tensor]
65 # [num_reqs, max_num_logprobs + 1]
41 - logprobs_cpu: Optional[torch.Tensor]
66 + # [num_reqs]
67 + logprobs: Optional[LogprobsLists]
68 +
69 + # req_id -> (token_ids, logprobs, ranks)
70 + # [prompt_len, num_prompt_logprobs]
71 + # [prompt_len, num_prompt_logprobs]
72 + # [prompt_len]
73 + prompt_logprobs_dict: Dict[str, LogprobsTensors]
vllm/v1/sample/metadata.py
+2 -1
@@ -20,7 +20,8 @@ class SamplingMetadata:
20
21 generators: Dict[int, torch.Generator]
22
23 - max_num_logprobs: int
23 + # None means no logprobs, 0 means sampled token logprobs only
24 + max_num_logprobs: Optional[int]
25
26 no_penalties: bool
27 prompt_token_ids: Optional[torch.Tensor]
vllm/v1/sample/sampler.py
+63 -31
@@ -1,11 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """A layer that samples the next tokens from the model's outputs."""
3 -from typing import Tuple
3
4 import torch
5 import torch.nn as nn
6
8 -from vllm.v1.outputs import SamplerOutput
7 +from vllm.v1.outputs import LogprobsTensors, SamplerOutput
8 from vllm.v1.sample.metadata import SamplingMetadata
9 from vllm.v1.sample.ops.penalties import (apply_all_penalties,
10 apply_min_token_penalties)
@@ -25,20 +24,16 @@ class Sampler(nn.Module):
24 logits: torch.Tensor,
25 sampling_metadata: SamplingMetadata,
26 ) -> SamplerOutput:
28 - needs_logprobs = sampling_metadata.max_num_logprobs > 0
29 - if needs_logprobs:
30 - # NOTE(woosuk): Use the original logits (before any penalties or
31 - # temperature scaling) for the top-k logprobs.
32 - # This is different from the V0 sampler, which uses the logits that
33 - # is used for sampling (after penalties and temperature scaling).
34 - # NOTE: We compute logprobs first because the below ops may
35 - # modify the logits tensor in-place (and we don't want to clone
36 - # the logits tensor for memory efficiency).
37 - topk_logprobs, topk_indices = self.get_topk_logprobs(
38 - logits, sampling_metadata)
39 - else:
40 - topk_logprobs = None
41 - topk_indices = None
27 +
28 + # NOTE(woosuk): Use the original logits (before any penalties or
29 + # temperature scaling) for the top-k logprobs.
30 + # This is different from the V0 sampler, which uses the logits that
31 + # is used for sampling (after penalties and temperature scaling).
32 + # TODO(rob): provide option for logprobs post sampling.
33 + # See https://vllm-dev.slack.com/archives/C07UUL8E61Z/p1735907856007919 # noqa: E501
34 + num_logprobs = sampling_metadata.max_num_logprobs
35 + if num_logprobs is not None:
36 + raw_logprobs = self.compute_logprobs(logits)
37
38 # Use float32 for the logits.
39 logits = logits.to(torch.float32)
@@ -48,15 +43,19 @@ class Sampler(nn.Module):
43 logits = self.apply_temperature(logits, sampling_metadata.temperature)
44 # Sample the next token.
45 sampled = self.sample(logits, sampling_metadata)
46 +
47 + # Gather the logprobs of the topk and sampled token (if requested).
48 + # Get logprobs and rank tensors (if requested)
49 + logprobs_tensors = None if num_logprobs is None else \
50 + self.gather_logprobs(raw_logprobs, num_logprobs, token_ids=sampled)
51 +
52 # Use int32 to reduce the tensor size.
53 sampled = sampled.to(torch.int32)
54
55 + # These are GPU tensors.
56 sampler_output = SamplerOutput(
57 sampled_token_ids=sampled,
56 - logprob_token_ids=topk_indices,
57 - logprobs=topk_logprobs,
58 - prompt_logprob_token_ids=None,
59 - prompt_logprobs=None,
58 + logprobs_tensors=logprobs_tensors,
59 )
60 return sampler_output
61
@@ -103,19 +102,52 @@ class Sampler(nn.Module):
102 )
103 return sampled
104
106 - def get_topk_logprobs(
105 + def compute_logprobs(self, logits: torch.Tensor) -> torch.Tensor:
106 + return logits.log_softmax(dim=-1, dtype=torch.float32)
107 +
108 + def gather_logprobs(
109 self,
108 - logits: torch.Tensor,
109 - sampling_metadata: SamplingMetadata,
110 - ) -> Tuple[torch.Tensor, torch.Tensor]:
111 - logprobs = logits.log_softmax(dim=-1, dtype=torch.float32)
112 - # FIXME: Mask the sampled token_id, get topk logprobs,
113 - # and concatenate the topk with the sampled token_id.
114 - topk_logprobs, topk_indices = torch.topk(
115 - logprobs, sampling_metadata.max_num_logprobs, dim=-1)
110 + logprobs: torch.Tensor,
111 + num_logprobs: int,
112 + token_ids: torch.Tensor,
113 + ) -> LogprobsTensors:
114 + """
115 + Gather logprobs for topk and sampled/prompt token.
116 +
117 + Args:
118 + logits: (num tokens) x (vocab) tensor
119 + num_logprobs: minimum number of logprobs to
120 + retain per token
121 + token_ids: prompt tokens (if prompt logprobs)
122 + or sampled tokens (if sampled
123 + logprobs); 1D token ID tensor
124 + with (num tokens) elements
125 +
126 + Returns:
127 + Top-k int indices tensor, (num tokens) x (num_logprobs + 1)
128 + Top-k float logprobs tensor, (num tokens) x (num_logprobs + 1)
129 + Sampled token rank tensor, (num tokens)
130 + """
131 + # Find the topK values.
132 + topk_logprobs, topk_indices = torch.topk(logprobs,
133 + num_logprobs,
134 + dim=-1)
135 +
136 + # Get with the logprob of the prompt or sampled token.
137 + token_ids = token_ids.unsqueeze(-1)
138 + token_logprobs = logprobs.gather(-1, token_ids)
139 +
140 + # Compute the ranks of the actual token.
141 + token_ranks = (logprobs >= token_logprobs).sum(-1)
142 +
143 + # Concatenate together with the topk.
144 + indices = torch.cat((token_ids, topk_indices), dim=1)
145 + logprobs = torch.cat((token_logprobs, topk_logprobs), dim=1)
146 +
147 # Use int32 to reduce the tensor size.
117 - topk_indices = topk_indices.to(torch.int32)
118 - return topk_logprobs, topk_indices
148 + indices = indices.to(torch.int32)
149 +
150 + return LogprobsTensors(indices, logprobs, token_ranks)
151
152 def apply_penalties(
153 self,
vllm/v1/serial_utils.py
+48 -2
@@ -1,12 +1,58 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import pickle
4 +from typing import Any
5 +
6 +import torch
7 +from msgspec import msgpack
8 +
9 +CUSTOM_TYPE_CODE_PICKLE = 1
10
11
12 class PickleEncoder:
13
8 - def encode(self, obj):
14 + def encode(self, obj: Any):
15 return pickle.dumps(obj)
16
11 - def decode(self, data):
17 + def decode(self, data: Any):
18 return pickle.loads(data)
19 +
20 +
21 +class MsgpackEncoder:
22 + """Encoder with custom torch tensor serialization."""
23 +
24 + def __init__(self):
25 + self.encoder = msgpack.Encoder(enc_hook=custom_enc_hook)
26 +
27 + def encode(self, obj: Any) -> bytes:
28 + return self.encoder.encode(obj)
29 +
30 + def encode_into(self, obj: Any, buf: bytearray) -> None:
31 + self.encoder.encode_into(obj, buf)
32 +
33 +
34 +class MsgpackDecoder:
35 + """Decoder with custom torch tensor serialization."""
36 +
37 + def __init__(self, t: Any):
38 + self.decoder = msgpack.Decoder(t, ext_hook=custom_ext_hook)
39 +
40 + def decode(self, obj: Any):
41 + return self.decoder.decode(obj)
42 +
43 +
44 +def custom_enc_hook(obj: Any) -> Any:
45 + if isinstance(obj, torch.Tensor):
46 + # NOTE(rob): it is fastest to use numpy + pickle
47 + # when serializing torch tensors.
48 + # https://gist.github.com/tlrmchlsmth/8067f1b24a82b6e2f90450e7764fa103 # noqa: E501
49 + return msgpack.Ext(CUSTOM_TYPE_CODE_PICKLE, pickle.dumps(obj.numpy()))
50 +
51 + raise NotImplementedError(f"Objects of type {type(obj)} are not supported")
52 +
53 +
54 +def custom_ext_hook(code: int, data: memoryview) -> Any:
55 + if code == CUSTOM_TYPE_CODE_PICKLE:
56 + return torch.from_numpy(pickle.loads(data))
57 +
58 + raise NotImplementedError(f"Extension type code {code} is not supported")
vllm/v1/worker/gpu_input_batch.py
+12 -15
@@ -176,7 +176,9 @@ class InputBatch:
176 self.generators: Dict[int, torch.Generator] = {}
177
178 self.num_logprobs: Dict[str, int] = {}
179 - self.prompt_logprob_reqs: Set[str] = set()
179 + # NOTE(rob): num_prompt_logprobs only includes reqs
180 + # that are currently in the prefill phase.
181 + self.num_prompt_logprobs: Dict[str, int] = {}
182
183 def add_request(
184 self,
@@ -238,11 +240,10 @@ class InputBatch:
240 if request.generator is not None:
241 self.generators[req_index] = request.generator
242
241 - num_logprobs = sampling_params.logprobs
242 - if num_logprobs is not None and num_logprobs > 0:
243 - self.num_logprobs[req_id] = num_logprobs
244 - if sampling_params.prompt_logprobs:
245 - self.prompt_logprob_reqs.add(req_id)
243 + if sampling_params.logprobs is not None:
244 + self.num_logprobs[req_id] = sampling_params.logprobs
245 + if sampling_params.prompt_logprobs is not None:
246 + self.num_prompt_logprobs[req_id] = sampling_params.prompt_logprobs
247
248 # Add request lora ID
249 if request.lora_request:
@@ -272,7 +273,7 @@ class InputBatch:
273 self.repetition_penalties_reqs.discard(req_id)
274 self.generators.pop(req_index, None)
275 self.num_logprobs.pop(req_id, None)
275 - self.prompt_logprob_reqs.discard(req_id)
276 + self.num_prompt_logprobs.pop(req_id, None)
277
278 # LoRA
279 lora_id = self.request_lora_mapping[req_index]
@@ -297,7 +298,7 @@ class InputBatch:
298 self.repetition_penalties_reqs.clear()
299 self.generators.clear()
300 self.num_logprobs.clear()
300 - self.prompt_logprob_reqs.clear()
301 + self.num_prompt_logprobs.clear()
302 self.request_lora_mapping.fill(0)
303 self.lora_id_to_lora_request.clear()
304 self.lora_id_to_request_ids.clear()
@@ -489,13 +490,9 @@ class InputBatch:
490 and len(self.repetition_penalties_reqs) == 0)
491
492 @property
492 - def max_num_logprobs(self) -> int:
493 - return max(self.num_logprobs.values()) if self.num_logprobs else 0
494 -
495 - @property
496 - def no_logprob(self) -> bool:
497 - return len(self.num_logprobs) == 0
493 + def max_num_logprobs(self) -> Optional[int]:
494 + return max(self.num_logprobs.values()) if self.num_logprobs else None
495
496 @property
497 def no_prompt_logprob(self) -> bool:
501 - return len(self.prompt_logprob_reqs) == 0
498 + return not self.num_prompt_logprobs
vllm/v1/worker/gpu_model_runner.py
+87 -15
@@ -29,7 +29,7 @@ from vllm.v1.core.encoder_cache_manager import compute_encoder_budget
29 from vllm.v1.engine.mm_input_mapper import MMInputMapperClient
30 from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
31 KVCacheSpec)
32 -from vllm.v1.outputs import ModelRunnerOutput
32 +from vllm.v1.outputs import LogprobsTensors, ModelRunnerOutput
33 from vllm.v1.sample.metadata import SamplingMetadata
34 from vllm.v1.utils import bind_kv_cache
35 from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
@@ -804,8 +804,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
804 inputs_embeds=inputs_embeds,
805 )
806 hidden_states = hidden_states[:num_scheduled_tokens]
807 - hidden_states = hidden_states[logits_indices]
808 - logits = self.model.compute_logits(hidden_states, None)
807 + sample_hidden_states = hidden_states[logits_indices]
808 + logits = self.model.compute_logits(sample_hidden_states, None)
809
810 # Sample the next token and get logprobs if needed.
811 sampling_metadata = self._prepare_sampling(batch_changed)
@@ -818,7 +818,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
818 # the requests one by one. Optimize.
819 num_reqs = self.input_batch.num_reqs
820 request_seq_lens: List[Tuple[int, CachedRequestState, int]] = []
821 - for i, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
821 + for i, req_id in enumerate( # type: ignore[assignment]
822 + self.input_batch.req_ids[:num_reqs]):
823 assert req_id is not None
824 req_state = self.requests[req_id]
825 seq_len = (req_state.num_computed_tokens +
@@ -847,27 +848,28 @@ class GPUModelRunner(LoRAModelRunnerMixin):
848 # NOTE: GPU -> CPU Sync happens here.
849 # Move as many CPU operations as possible before this sync point.
850 sampled_token_ids = sampler_output.sampled_token_ids.tolist()
851 + logprobs_tensors = sampler_output.logprobs_tensors
852 + logprobs_lists = logprobs_tensors.tolists() \
853 + if logprobs_tensors is not None else None
854 +
855 + # Compute prompt logprobs if needed.
856 + prompt_logprobs_dict = self._get_prompt_logprobs_dict(
857 + hidden_states,
858 + scheduler_output,
859 + )
860 +
861 # Update with the actual token ids
862 for i, req_state, seq_len in request_seq_lens:
863 token_id = sampled_token_ids[i]
864 self.input_batch.token_ids_cpu[i, seq_len] = token_id
865 req_state.output_token_ids[-1] = token_id
866
856 - if sampler_output.logprob_token_ids is None:
857 - logprob_token_ids = None
858 - else:
859 - logprob_token_ids = sampler_output.logprob_token_ids.cpu()
860 - if sampler_output.logprobs is None:
861 - logprobs = None
862 - else:
863 - logprobs = sampler_output.logprobs.cpu()
864 -
867 model_runner_output = ModelRunnerOutput(
868 req_ids=req_ids,
869 req_id_to_index=self.input_batch.req_id_to_index,
870 sampled_token_ids=sampled_token_ids,
869 - logprob_token_ids_cpu=logprob_token_ids,
870 - logprobs_cpu=logprobs,
871 + logprobs=logprobs_lists,
872 + prompt_logprobs_dict=prompt_logprobs_dict,
873 )
874 return model_runner_output
875
@@ -886,6 +888,76 @@ class GPUModelRunner(LoRAModelRunnerMixin):
888 logger.info("Loading model weights took %.4f GB",
889 self.model_memory_usage / float(2**30))
890
891 + def _get_prompt_logprobs_dict(
892 + self,
893 + hidden_states: torch.Tensor,
894 + scheduler_output: "SchedulerOutput",
895 + ) -> Dict[str, LogprobsTensors]:
896 + num_prompt_logprobs_dict = self.input_batch.num_prompt_logprobs
897 + if not num_prompt_logprobs_dict:
898 + return {}
899 +
900 + prompt_logprobs_dict: Dict[str, LogprobsTensors] = {}
901 +
902 + # Since prompt logprobs are a rare feature, prioritize simple,
903 + # maintainable loop over optimal performance.
904 + completed_prefill_reqs = []
905 + for req_id, num_prompt_logprobs in num_prompt_logprobs_dict.items():
906 +
907 + num_tokens = scheduler_output.num_scheduled_tokens[req_id]
908 +
909 + # Get metadata for this request.
910 + request = self.requests[req_id]
911 + num_prompt_tokens = len(request.prompt_token_ids)
912 + prompt_token_ids = torch.tensor(request.prompt_token_ids).to(
913 + self.device, non_blocking=True)
914 +
915 + # Determine number of logits to retrieve.
916 + start_tok = request.num_computed_tokens + 1
917 + num_remaining_tokens = num_prompt_tokens - start_tok
918 + if num_tokens < num_remaining_tokens:
919 + # This is a chunk, more tokens remain.
920 + num_logits = num_tokens
921 + else:
922 + # This is the last chunk of prompt tokens to return.
923 + num_logits = num_remaining_tokens
924 + completed_prefill_reqs.append(req_id)
925 +
926 + # Get the logits corresponding to this req's prompt tokens.
927 + # If this is a partial request (i.e. chunked prefill),
928 + # then there is prompt logprob generated for each index.
929 + req_idx = self.input_batch.req_id_to_index[req_id]
930 + offset = self.query_start_loc_np[req_idx].item()
931 + prompt_hidden_states = hidden_states[offset:offset + num_logits]
932 + logits = self.model.compute_logits(prompt_hidden_states, None)
933 +
934 + # Get the "target" tokens for each index. For prompt at index i,
935 + # the token at prompt index i+1 is the "sampled" token we want
936 + # to gather the logprob for.
937 + tgt_token_ids = prompt_token_ids[start_tok:start_tok + num_logits]
938 +
939 + # Compute prompt logprobs.
940 + logprobs = self.model.sampler.compute_logprobs(logits)
941 + token_ids, logprobs, ranks = self.model.sampler.gather_logprobs(
942 + logprobs, num_prompt_logprobs, tgt_token_ids)
943 +
944 + # Transfer GPU->CPU async.
945 + prompt_logprobs_dict[req_id] = LogprobsTensors(
946 + token_ids.to("cpu", non_blocking=True),
947 + logprobs.to("cpu", non_blocking=True),
948 + ranks.to("cpu", non_blocking=True),
949 + )
950 +
951 + # Remove requests that have completed prefill from the batch
952 + # num_prompt_logprobs_dict.
953 + for req_id in completed_prefill_reqs:
954 + del num_prompt_logprobs_dict[req_id]
955 +
956 + # Must synchronize the non-blocking GPU->CPU transfers.
957 + torch.cuda.synchronize()
958 +
959 + return prompt_logprobs_dict
960 +
961 @torch.inference_mode()
962 def _dummy_run(
963 self,