@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 428dd1445e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Logprobs support in Multi-step (#7652)

afeldman-nm committed Aug 29, 2024 at 22:19 UTC 428dd1445ee3750099967084725849c4920721a5
103 files changed +874 -378
tests/models/utils.py
+26 -17
@@ -1,7 +1,7 @@
1 import warnings
2 from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4 -from vllm.sequence import SampleLogprobs
4 +from vllm.sequence import Logprob, SampleLogprobs
5
6 TokensText = Tuple[List[int], str]
7
@@ -38,34 +38,39 @@ TokensTextLogprobs = Tuple[List[int], str, Optional[Union[List[Dict[int,
38 float]],
39 SampleLogprobs]]]
40
41 +# Allow for tokens to be represented as str's rather than IDs
42 +TextTextLogprobs = Tuple[List[str], str, Optional[Union[List[Dict[str, float]],
43 + List[Dict[str,
44 + Logprob]]]]]
45 +
46
47 def check_logprobs_close(
48 *,
44 - outputs_0_lst: Sequence[TokensTextLogprobs],
45 - outputs_1_lst: Sequence[TokensTextLogprobs],
49 + outputs_0_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
50 + outputs_1_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
51 name_0: str,
52 name_1: str,
53 num_outputs_0_skip_tokens: int = 0,
54 warn_on_mismatch: bool = True,
50 -):
51 - """
52 - Compare the logprobs of two sequences generated by different models,
55 + always_check_logprobs: bool = False,
56 +) -> None:
57 + """Compare the logprobs of two sequences generated by different models,
58 which should be similar but not necessarily equal.
59
55 - Arguments:
56 -
57 - * outputs_0_lst: First sequence to compare
58 - * outputs_0_lst: Second sequence to compare
59 - * name_0: sequence #0 name
60 - * name_1: sequence #1 name
61 - * num_outputs_0_skip_tokens: If > 0, specifies the number of initial
60 + Args:
61 + outputs_0_lst: First sequence to compare
62 + outputs_0_lst: Second sequence to compare
63 + name_0: sequence #0 name
64 + name_1: sequence #1 name
65 + num_outputs_0_skip_tokens: If > 0, specifies the number of initial
66 sequence #0 tokens & logprobs to discard
67 before comparison, i.e. all
68 of sequence #1 will be compared to
69 sequence #0 beginning at index
70 num_outputs_0_skip_tokens
67 - * warn_on_mismatch: Issue a warning if there is token-wise or text-wise
71 + warn_on_mismatch: Issue a warning if there is token-wise or text-wise
72 mismatch between the two sequences
73 + always_check_logprobs: If true, check logprobs even when tokens match
74 """
75 assert len(outputs_0_lst) == len(outputs_1_lst)
76
@@ -94,8 +99,12 @@ def check_logprobs_close(
99 for idx, (output_id_0,
100 output_id_1) in enumerate(zip(output_ids_0, output_ids_1)):
101
97 - # If generated tokens don't match, then
98 - if output_id_0 != output_id_1:
102 + is_tok_mismatch = output_id_0 != output_id_1
103 +
104 + # If generated tokens don't match
105 + # or it is desired to always check logprobs,
106 + # then
107 + if is_tok_mismatch or always_check_logprobs:
108 logprobs_elem_0 = logprobs_0[idx]
109 logprobs_elem_1 = logprobs_1[idx]
110
@@ -111,7 +120,7 @@ def check_logprobs_close(
120 assert output_id_0 in logprobs_elem_1, fail_msg
121 assert output_id_1 in logprobs_elem_0, fail_msg
122
114 - if warn_on_mismatch:
123 + if warn_on_mismatch and is_tok_mismatch:
124 with warnings.catch_warnings():
125 # This ensures that repeated warnings are shown
126 # in the output, not just the first occurrence
tests/multi_step/test_correctness_async_llm.py
+69 -30
@@ -1,10 +1,12 @@
1 # Test the AsyncLLMEngine with multi-step-decoding
2
3 -from typing import List
3 +from typing import List, Optional
4
5 import pytest
6
7 -from ..utils import RemoteOpenAIServer
7 +from ..models.utils import check_logprobs_close
8 +from ..utils import (completions_with_server_args, get_client_text_generations,
9 + get_client_text_logprob_generations)
10
11 MODELS = [
12 "JackFram/llama-160m",
@@ -23,22 +25,6 @@ DEFAULT_SERVER_ARGS: List[str] = [
25 ]
26
27
26 -async def completions_with_server_args(prompts: List[str], model_name: str,
27 - server_cli_args: List[str]):
28 -
29 - outputs = None
30 - with RemoteOpenAIServer(model_name, server_cli_args) as server:
31 - async with server.get_async_client() as client:
32 - outputs = await client.completions.create(model=model_name,
33 - prompt=prompts,
34 - temperature=0,
35 - stream=False,
36 - max_tokens=5)
37 - assert outputs is not None
38 -
39 - return outputs
40 -
41 -
28 @pytest.mark.parametrize("model", MODELS)
29 @pytest.mark.parametrize(("tp_size, pp_size"), [
30 (1, 1),
@@ -47,12 +33,43 @@ async def completions_with_server_args(prompts: List[str], model_name: str,
33 @pytest.mark.parametrize("eager_mode", [False, True])
34 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
35 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
36 +@pytest.mark.parametrize("num_logprobs", [None, 5])
37 @pytest.mark.parametrize("is_async", [False, True])
38 @pytest.mark.asyncio
52 -async def test_multi_step(example_prompts, model: str, tp_size: int,
53 - pp_size: int, eager_mode: int,
54 - num_scheduler_steps: int, num_prompts: int,
55 - is_async: bool):
39 +async def test_multi_step(
40 + example_prompts,
41 + model: str,
42 + tp_size: int,
43 + pp_size: int,
44 + eager_mode: int,
45 + num_scheduler_steps: int,
46 + num_prompts: int,
47 + is_async: bool,
48 + num_logprobs: Optional[int],
49 +) -> None:
50 + """Test vLLM engine with multi-step scheduling in an OpenAI-protocol
51 + client/server environment.
52 +
53 + Set up an engine with single-step scheduling as a ground-truth reference.
54 +
55 + Send a completions API request to both engines with the same prompts.
56 +
57 + Validate:
58 + * Generated tokens match
59 + * Generated logprobs are all very close
60 +
61 + Args:
62 + example_prompts: test fixture providing example prompts
63 + model: model under test (same for single- and multi-step engines)
64 + tp_size: degree of tensor-parallelism
65 + pp_size: degree of pipeline-parallelism
66 + eager_mode
67 + num_scheduler_steps: for multi-step scheduling, GPU-side steps per
68 + GPU -> CPU output transfer
69 + num_prompts: number of example prompts under test
70 + num_logprobs: corresponds to the `logprobs` argument to the OpenAI
71 + completions endpoint; `None` -> no logprobs
72 + """
73
74 prompts = example_prompts
75 if len(prompts) < num_prompts:
@@ -77,14 +94,36 @@ async def test_multi_step(example_prompts, model: str, tp_size: int,
94 str(pp_size),
95 ]
96
97 + # Spin up client/server & issue completion API requests.
98 + # Default `max_wait_seconds` is 240 but was empirically
99 + # was raised 3x to 720 *just for this test* due to
100 + # observed timeouts in GHA CI
101 ref_completions = await completions_with_server_args(
81 - prompts, model, server_args + distributed_args)
102 + prompts,
103 + model,
104 + server_args + distributed_args,
105 + num_logprobs,
106 + max_wait_seconds=3 * 240)
107 test_completions = await completions_with_server_args(
83 - prompts, model, ms_server_args + distributed_args)
84 -
85 - def get_text_generations(completions):
86 - return [x.text for x in completions.choices]
87 -
88 - ref_generations = get_text_generations(ref_completions)
89 - test_generations = get_text_generations(test_completions)
108 + prompts,
109 + model,
110 + ms_server_args + distributed_args,
111 + num_logprobs,
112 + max_wait_seconds=3 * 240)
113 +
114 + # Assert multi-step scheduling produces identical tokens
115 + # to single-step scheduling.
116 + ref_generations = get_client_text_generations(ref_completions)
117 + test_generations = get_client_text_generations(test_completions)
118 assert ref_generations == test_generations
119 +
120 + # Assert multi-step scheduling produces nearly-identical logprobs
121 + # to single-step scheduling.
122 + ref_text_logprobs = get_client_text_logprob_generations(ref_completions)
123 + test_text_logprobs = get_client_text_logprob_generations(test_completions)
124 + check_logprobs_close(
125 + outputs_0_lst=ref_text_logprobs,
126 + outputs_1_lst=test_text_logprobs,
127 + name_0="hf",
128 + name_1="vllm",
129 + )
tests/multi_step/test_correctness_llm.py
+74 -21
@@ -1,8 +1,10 @@
1 # Test the LLMEngine with multi-step-decoding
2
3 +from typing import Optional
4 +
5 import pytest
6
5 -from ..models.utils import check_outputs_equal
7 +from ..models.utils import check_logprobs_close, check_outputs_equal
8
9 MODELS = [
10 "JackFram/llama-160m",
@@ -18,10 +20,45 @@ NUM_PROMPTS = [10]
20 @pytest.mark.parametrize("enforce_eager", [True])
21 @pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
22 @pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
21 -def test_multi_step_llm(hf_runner, vllm_runner, example_prompts, model: str,
22 - dtype: str, tp_size: int, max_tokens: int,
23 - enforce_eager: int, num_scheduler_steps: int,
24 - num_prompts: int) -> None:
23 +@pytest.mark.parametrize("num_logprobs", [None, 5])
24 +def test_multi_step_llm(
25 + hf_runner,
26 + vllm_runner,
27 + example_prompts,
28 + model: str,
29 + dtype: str,
30 + tp_size: int,
31 + max_tokens: int,
32 + enforce_eager: int,
33 + num_scheduler_steps: int,
34 + num_prompts: int,
35 + num_logprobs: Optional[int],
36 +) -> None:
37 + """Test vLLM engine with multi-step scheduling via sync LLM Engine.
38 +
39 + Set up a HuggingFace (HF) transformers model as a ground-truth reference.
40 +
41 + Prompt them with the same example prompts.
42 +
43 + Validate:
44 + * Generated tokens match
45 + * Generated logprobs are all very close
46 +
47 + Args:
48 + hf_runner: HF transformers model runner fixture
49 + vllm_runner: vLLM model runner fixture
50 + example_prompts: test fixture providing example prompts
51 + model: model under test (same for single- and multi-step engines)
52 + dtype: tensor datatype for engine to utilize
53 + tp_size: degree of tensor-parallelism
54 + max_tokens: the maximum number of tokens to generate
55 + enforce_eager
56 + num_scheduler_steps: for multi-step scheduling, GPU-side steps per
57 + GPU -> CPU output transfer
58 + num_prompts: number of example prompts under test
59 + num_logprobs: corresponds to the `logprobs` argument to the OpenAI
60 + completions endpoint; `None` -> no logprobs
61 + """
62
63 prompts = example_prompts
64 if len(prompts) < num_prompts:
@@ -29,21 +66,37 @@ def test_multi_step_llm(hf_runner, vllm_runner, example_prompts, model: str,
66 prompts = prompts[:num_prompts]
67 assert len(prompts) == num_prompts
68
32 - with vllm_runner(model,
33 - dtype=dtype,
34 - enforce_eager=enforce_eager,
35 - gpu_memory_utilization=0.7,
36 - tensor_parallel_size=tp_size,
37 - use_v2_block_manager=True,
38 - num_scheduler_steps=num_scheduler_steps) as vllm_model:
39 - vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
69 + with vllm_runner(
70 + model,
71 + dtype=dtype,
72 + enforce_eager=enforce_eager,
73 + gpu_memory_utilization=0.7,
74 + tensor_parallel_size=tp_size,
75 + use_v2_block_manager=True,
76 + num_scheduler_steps=num_scheduler_steps,
77 + ) as vllm_model:
78 + vllm_outputs = (vllm_model.generate_greedy(prompts, max_tokens)
79 + if num_logprobs is None else
80 + vllm_model.generate_greedy_logprobs(
81 + prompts, max_tokens, num_logprobs))
82
83 with hf_runner(model, dtype=dtype) as hf_model:
42 - hf_outputs = hf_model.generate_greedy(prompts, max_tokens)
43 -
44 - check_outputs_equal(
45 - outputs_0_lst=hf_outputs,
46 - outputs_1_lst=vllm_outputs,
47 - name_0="hf",
48 - name_1="vllm",
49 - )
84 + hf_outputs = (hf_model.generate_greedy(prompts, max_tokens)
85 + if num_logprobs is None else
86 + hf_model.generate_greedy_logprobs_limit(
87 + prompts, max_tokens, num_logprobs))
88 +
89 + if num_logprobs is None:
90 + check_outputs_equal(
91 + outputs_0_lst=hf_outputs,
92 + outputs_1_lst=vllm_outputs,
93 + name_0="hf",
94 + name_1="vllm",
95 + )
96 + else:
97 + check_logprobs_close(
98 + outputs_0_lst=hf_outputs,
99 + outputs_1_lst=vllm_outputs,
100 + name_0="hf",
101 + name_1="vllm",
102 + )
tests/spec_decode/test_multi_step_worker.py
+2 -1
@@ -5,9 +5,10 @@ from unittest.mock import MagicMock
5 import pytest
6 import torch
7
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.model_executor.utils import set_random_seed
10 from vllm.sequence import (ExecuteModelRequest, HiddenStates, Logprob,
10 - SamplerOutput, get_all_seq_ids)
11 + get_all_seq_ids)
12 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
13 from vllm.spec_decode.multi_step_worker import MultiStepWorker
14 from vllm.spec_decode.top1_proposer import Top1Proposer
tests/spec_decode/test_spec_decode_worker.py
+2 -1
@@ -7,8 +7,9 @@ from unittest.mock import MagicMock
7 import pytest
8 import torch
9
10 +from vllm.model_executor.layers.sampler import SamplerOutput
11 from vllm.model_executor.utils import set_random_seed
11 -from vllm.sequence import ExecuteModelRequest, SamplerOutput, SequenceOutput
12 +from vllm.sequence import ExecuteModelRequest, SequenceOutput
13 from vllm.spec_decode.interfaces import SpeculativeProposals
14 from vllm.spec_decode.metrics import (AsyncMetricsCollector,
15 SpecDecodeWorkerMetrics)
tests/spec_decode/utils.py
+2 -2
@@ -8,12 +8,12 @@ from unittest.mock import MagicMock
8 import torch
9
10 from vllm.engine.arg_utils import EngineArgs
11 +from vllm.model_executor.layers.sampler import SamplerOutput
12 from vllm.model_executor.utils import set_random_seed
13 from vllm.sampling_params import SamplingParams
14 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
15 CompletionSequenceGroupOutput, Logprob,
15 - SamplerOutput, SequenceData, SequenceGroupMetadata,
16 - SequenceOutput)
16 + SequenceData, SequenceGroupMetadata, SequenceOutput)
17 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
18 from vllm.worker.cache_engine import CacheEngine
19 from vllm.worker.model_runner import ModelRunner
tests/test_sequence.py
+3 -2
@@ -2,9 +2,10 @@ from array import array
2
3 import pytest
4
5 +from vllm.model_executor.layers.sampler import SamplerOutput
6 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
6 - CompletionSequenceGroupOutput, SamplerOutput,
7 - SequenceData, SequenceOutput)
7 + CompletionSequenceGroupOutput, SequenceData,
8 + SequenceOutput)
9
10 from .core.utils import create_dummy_prompt
11
tests/utils.py
+60
@@ -11,9 +11,11 @@ from typing import Any, Callable, Dict, List, Optional
11
12 import openai
13 import requests
14 +from openai.types.completion import Completion
15 from transformers import AutoTokenizer
16 from typing_extensions import ParamSpec
17
18 +from tests.models.utils import TextTextLogprobs
19 from vllm.distributed import (ensure_model_parallel_initialized,
20 init_distributed_environment)
21 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -432,3 +434,61 @@ def fork_new_process_for_each_test(
434 f" args {args} and kwargs {kwargs}")
435
436 return wrapper
437 +
438 +
439 +async def completions_with_server_args(
440 + prompts: List[str],
441 + model_name: str,
442 + server_cli_args: List[str],
443 + num_logprobs: Optional[int],
444 + max_wait_seconds: int = 240,
445 +) -> Completion:
446 + '''Construct a remote OpenAI server, obtain an async client to the
447 + server & invoke the completions API to obtain completions.
448 +
449 + Args:
450 + prompts: test prompts
451 + model_name: model to spin up on the vLLM server
452 + server_cli_args: CLI args for starting the server
453 + num_logprobs: Number of logprobs to report (or `None`)
454 + max_wait_seconds: timeout interval for bringing up server.
455 + Default: 240sec
456 +
457 + Returns:
458 + OpenAI Completion instance
459 + '''
460 +
461 + outputs = None
462 + with RemoteOpenAIServer(model_name,
463 + server_cli_args,
464 + max_wait_seconds=max_wait_seconds) as server:
465 + client = server.get_async_client()
466 + outputs = await client.completions.create(model=model_name,
467 + prompt=prompts,
468 + temperature=0,
469 + stream=False,
470 + max_tokens=5,
471 + logprobs=num_logprobs)
472 + assert outputs is not None
473 +
474 + return outputs
475 +
476 +
477 +def get_client_text_generations(completions: Completion) -> List[str]:
478 + '''Extract generated tokens from the output of a
479 + request made to an Open-AI-protocol completions endpoint.
480 + '''
481 + return [x.text for x in completions.choices]
482 +
483 +
484 +def get_client_text_logprob_generations(
485 + completions: Completion) -> List[TextTextLogprobs]:
486 + '''Operates on the output of a request made to an Open-AI-protocol
487 + completions endpoint; obtains top-rank logprobs for each token in
488 + each :class:`SequenceGroup`
489 + '''
490 + text_generations = get_client_text_generations(completions)
491 + text = ''.join(text_generations)
492 + return [(text_generations, text,
493 + (None if x.logprobs is None else x.logprobs.top_logprobs))
494 + for x in completions.choices]
vllm/engine/async_llm_engine.py
+2 -1
@@ -22,11 +22,12 @@ from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
22 from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
23 from vllm.logger import init_logger
24 from vllm.lora.request import LoRARequest
25 +from vllm.model_executor.layers.sampler import SamplerOutput
26 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27 from vllm.pooling_params import PoolingParams
28 from vllm.prompt_adapter.request import PromptAdapterRequest
29 from vllm.sampling_params import SamplingParams
29 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
30 +from vllm.sequence import ExecuteModelRequest
31 from vllm.transformers_utils.tokenizer import AnyTokenizer
32 from vllm.usage.usage_lib import UsageContext
33 from vllm.utils import print_warning_once
vllm/engine/llm_engine.py
+3 -2
@@ -33,6 +33,7 @@ from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
33 from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
34 from vllm.logger import init_logger
35 from vllm.lora.request import LoRARequest
36 +from vllm.model_executor.layers.sampler import SamplerOutput
37 from vllm.multimodal import MultiModalDataDict
38 from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
39 RequestOutputFactory)
@@ -40,8 +41,8 @@ from vllm.pooling_params import PoolingParams
41 from vllm.prompt_adapter.request import PromptAdapterRequest
42 from vllm.sampling_params import SamplingParams
43 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
43 - SamplerOutput, Sequence, SequenceGroup,
44 - SequenceGroupMetadata, SequenceStatus)
44 + Sequence, SequenceGroup, SequenceGroupMetadata,
45 + SequenceStatus)
46 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
47 init_tracer)
48 from vllm.transformers_utils.config import try_get_generation_config
vllm/engine/output_processor/multi_step.py
+12 -3
@@ -4,6 +4,8 @@ from typing import Callable, List
4 from vllm.core.scheduler import Scheduler
5 from vllm.engine.output_processor.interfaces import (
6 SequenceGroupOutputProcessor)
7 +from vllm.engine.output_processor.single_step import (
8 + single_step_process_prompt_logprob)
9 from vllm.engine.output_processor.stop_checker import StopChecker
10 from vllm.logger import init_logger
11 from vllm.sampling_params import SamplingParams
@@ -46,9 +48,16 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
48
49 def process_prompt_logprob(self, seq_group: SequenceGroup,
50 outputs: List[SequenceGroupOutput]) -> None:
49 - # TODO(sang): Prompt logprob currently not implemented in multi step
50 - # workers.
51 - self._log_prompt_logprob_unsupported_warning_once()
51 + """Process prompt logprobs associated with each step of a multi-step-
52 + scheduled computation.
53 +
54 + Args:
55 + seq_group: the outputs are associated with this :class:`SequenceGroup`
56 + outputs: the :class:`SequenceGroupOutput`s for all scheduler steps
57 + """
58 + for output in outputs:
59 + # Concatenate single-step prompt logprob processing results.
60 + single_step_process_prompt_logprob(self, seq_group, output)
61
62 @staticmethod
63 @functools.lru_cache()
vllm/engine/output_processor/single_step.py
+46 -19
@@ -15,6 +15,44 @@ from vllm.utils import Counter
15 logger = init_logger(__name__)
16
17
18 +def single_step_process_prompt_logprob(
19 + sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
20 + output: SequenceGroupOutput) -> None:
21 + """Process prompt logprobs associated with the :class:`SequenceGroupOutput`
22 + for a given step.
23 +
24 + Do nothing if the output has no prompt logprobs.
25 +
26 + Account for the fact that transformers do not compute first-token logprobs.
27 +
28 + Args:
29 + sg_output_proc: :class:`SequenceGroupOutputProcessor` instance
30 + seq_group: the output is associated with this :class:`SequenceGroup`
31 + output: the :class:`SequenceGroupOutput` for a single scheduler step
32 + """
33 + prompt_logprobs = output.prompt_logprobs
34 +
35 + # If this is the first (or only) "chunk" of the prefill, we need
36 + # to prepend None to the list of prompt logprobs. The reason for this
37 + # is that for N prompt tokens, the Sampler will generate N-1 total
38 + # prompt logprobs during prefill since the token at idx 0 will not
39 + # have a logprob associated with it.
40 + if prompt_logprobs is not None:
41 + if not seq_group.prompt_logprobs:
42 + prompt_logprobs = [None] + prompt_logprobs
43 + seq_group.prompt_logprobs = []
44 +
45 + assert hasattr(sg_output_proc, 'detokenizer')
46 + if (seq_group.sampling_params.detokenize
47 + and sg_output_proc.detokenizer):
48 + sg_output_proc.detokenizer.decode_prompt_logprobs_inplace(
49 + seq_group,
50 + prompt_logprobs,
51 + position_offset=len(seq_group.prompt_logprobs))
52 +
53 + seq_group.prompt_logprobs.extend(prompt_logprobs)
54 +
55 +
56 class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
57 """SequenceGroupOutputProcessor which handles "output processing" logic,
58 which happens after the model returns generated token ids and before
@@ -60,27 +98,16 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
98
99 def process_prompt_logprob(self, seq_group: SequenceGroup,
100 outputs: List[SequenceGroupOutput]) -> None:
101 + """Process prompt logprobs associated with one step of a single-step-
102 + scheduled computation.
103 +
104 + Args:
105 + seq_group: the output is associated with this :class:`SequenceGroup`
106 + output: the :class:`SequenceGroupOutput` for a single scheduler step
107 + """
108 assert len(outputs) == 1, ("Single step should only has 1 output.")
109 output = outputs[0]
65 - prompt_logprobs = output.prompt_logprobs
66 -
67 - # If this is the first (or only) "chunk" of the prefill, we need
68 - # to prepend None to the list of prompt logprobs. The reason for this
69 - # is that for N prompt tokens, the Sampler will generate N-1 total
70 - # prompt logprobs during prefill since the token at idx 0 will not
71 - # have a logprob associated with it.
72 - if prompt_logprobs is not None:
73 - if not seq_group.prompt_logprobs:
74 - prompt_logprobs = [None] + prompt_logprobs
75 - seq_group.prompt_logprobs = []
76 -
77 - if seq_group.sampling_params.detokenize and self.detokenizer:
78 - self.detokenizer.decode_prompt_logprobs_inplace(
79 - seq_group,
80 - prompt_logprobs,
81 - position_offset=len(seq_group.prompt_logprobs))
82 -
83 - seq_group.prompt_logprobs.extend(prompt_logprobs)
110 + single_step_process_prompt_logprob(self, seq_group, output)
111
112 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
113 outputs: SequenceGroupOutput,
vllm/engine/output_processor/util.py
+2 -1
@@ -2,7 +2,8 @@ from typing import List
2 from typing import Sequence as GenericSequence
3 from typing import Union
4
5 -from vllm.sequence import PoolerOutput, SamplerOutput, SequenceGroupOutput
5 +from vllm.model_executor.layers.sampler import SamplerOutput
6 +from vllm.sequence import PoolerOutput, SequenceGroupOutput
7
8
9 def create_output_by_sequence_group(
vllm/engine/protocol.py
+1 -1
@@ -5,11 +5,11 @@ from vllm.config import DecodingConfig, ModelConfig
5 from vllm.core.scheduler import SchedulerOutputs
6 from vllm.inputs.data import PromptInputs
7 from vllm.lora.request import LoRARequest
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
10 from vllm.pooling_params import PoolingParams
11 from vllm.prompt_adapter.request import PromptAdapterRequest
12 from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import SamplerOutput
13 from vllm.transformers_utils.tokenizer import AnyTokenizer
14
15
vllm/executor/cpu_executor.py
+2 -1
@@ -11,8 +11,9 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
11 ResultHandler, WorkerMonitor)
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 +from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.prompt_adapter.request import PromptAdapterRequest
15 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
16 +from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import (GiB_bytes, get_distributed_init_method, get_open_port,
18 get_vllm_instance_id, make_async)
19 from vllm.worker.worker_base import WorkerWrapperBase
vllm/executor/distributed_gpu_executor.py
+2 -1
@@ -6,7 +6,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase
6 from vllm.executor.gpu_executor import GPUExecutor
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
9 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 +from vllm.model_executor.layers.sampler import SamplerOutput
10 +from vllm.sequence import ExecuteModelRequest
11
12 logger = init_logger(__name__)
13
vllm/executor/executor_base.py
+2 -1
@@ -6,8 +6,9 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 PromptAdapterConfig, SchedulerConfig,
7 SpeculativeConfig)
8 from vllm.lora.request import LoRARequest
9 +from vllm.model_executor.layers.sampler import SamplerOutput
10 from vllm.prompt_adapter.request import PromptAdapterRequest
10 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
11 +from vllm.sequence import ExecuteModelRequest
12
13
14 class ExecutorBase(ABC):
vllm/executor/gpu_executor.py
+2 -1
@@ -3,8 +3,9 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 +from vllm.model_executor.layers.sampler import SamplerOutput
7 from vllm.prompt_adapter.request import PromptAdapterRequest
7 -from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
8 +from vllm.sequence import ExecuteModelRequest, PoolerOutput
9 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10 make_async)
11 from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase
vllm/executor/multiproc_gpu_executor.py
+2 -1
@@ -14,7 +14,8 @@ from vllm.executor.gpu_executor import create_worker
14 from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
15 ResultHandler, WorkerMonitor)
16 from vllm.logger import init_logger
17 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
17 +from vllm.model_executor.layers.sampler import SamplerOutput
18 +from vllm.sequence import ExecuteModelRequest
19 from vllm.triton_utils import maybe_set_triton_cache_manager
20 from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
21 get_distributed_init_method, get_open_port,
vllm/executor/neuron_executor.py
+2 -1
@@ -3,7 +3,8 @@ from typing import List, Set, Tuple
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
6 +from vllm.model_executor.layers.sampler import SamplerOutput
7 +from vllm.sequence import ExecuteModelRequest
8 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9 make_async)
10
vllm/executor/openvino_executor.py
+2 -1
@@ -9,7 +9,8 @@ from vllm.config import CacheConfig, ModelConfig
9 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
12 +from vllm.model_executor.layers.sampler import SamplerOutput
13 +from vllm.sequence import ExecuteModelRequest
14 from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
15 get_open_port, make_async)
16
vllm/executor/ray_gpu_executor.py
+2 -1
@@ -12,7 +12,8 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
12 from vllm.executor.msgspec_utils import encode_hook
13 from vllm.executor.ray_utils import RayWorkerWrapper, ray
14 from vllm.logger import init_logger
15 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
15 +from vllm.model_executor.layers.sampler import SamplerOutput
16 +from vllm.sequence import ExecuteModelRequest
17 from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18 get_ip, get_open_port, get_vllm_instance_id,
19 make_async)
vllm/executor/ray_tpu_executor.py
+2 -1
@@ -10,7 +10,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase
10 from vllm.executor.ray_utils import RayWorkerWrapper, ray
11 from vllm.executor.tpu_executor import TPUExecutor
12 from vllm.logger import init_logger
13 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
13 +from vllm.model_executor.layers.sampler import SamplerOutput
14 +from vllm.sequence import ExecuteModelRequest
15 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16 get_vllm_instance_id, make_async)
17
vllm/executor/tpu_executor.py
+2 -1
@@ -5,7 +5,8 @@ import torch
5 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 +from vllm.sequence import ExecuteModelRequest
10 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11 make_async)
12
vllm/executor/xpu_executor.py
+2 -1
@@ -9,7 +9,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 from vllm.executor.executor_base import ExecutorAsyncBase
10 from vllm.executor.gpu_executor import GPUExecutor
11 from vllm.logger import init_logger
12 -from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
12 +from vllm.model_executor.layers.sampler import SamplerOutput
13 +from vllm.sequence import ExecuteModelRequest, PoolerOutput
14 from vllm.utils import make_async
15 from vllm.worker.worker_base import WorkerBase
16
vllm/model_executor/layers/sampler.py
+247 -43
@@ -1,13 +1,16 @@
1 """A layer that samples the next tokens from the model's outputs."""
2 import itertools
3 import warnings
4 +from dataclasses import dataclass
5 from importlib.util import find_spec
6 from math import inf
6 -from typing import Dict, List, Optional, Tuple
7 +from typing import Dict, List, Optional, Tuple, Union
8
9 +import msgspec
10 import torch
11 import torch.nn as nn
12
13 +from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
14 from vllm.triton_utils import HAS_TRITON
15
16 if HAS_TRITON:
@@ -19,8 +22,7 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
22 SequenceGroupToSample)
23 from vllm.sampling_params import SamplingType
24 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
22 - PromptLogprobs, SampleLogprobs, SamplerOutput,
23 - SequenceOutput)
25 + PromptLogprobs, SampleLogprobs, SequenceOutput)
26
27 if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
28 import flashinfer.sampling
@@ -35,6 +37,116 @@ else:
37 # (num_token_ids, num_parent_ids) per sequence group.
38 SampleResultType = List[Tuple[List[int], List[int]]]
39
40 +# Types of temporary data structures used for
41 +# computing sample_result
42 +SampleMetadataType = Dict[SamplingType, Tuple[List[int],
43 + List[SequenceGroupToSample]]]
44 +MultinomialSamplesType = Dict[SamplingType, torch.Tensor]
45 +SampleResultsDictType = Dict[int, Tuple[List[int], List[int]]]
46 +
47 +
48 +# Encapsulates temporary data structures for computing
49 +# sample_result.
50 +#
51 +# * For multi-step scheduling: must be returned
52 +# by `Sampler.forward()` and used later to compute the pythonized
53 +# sample_result
54 +#
55 +# * For single-step scheduling: consumed immediately
56 +# inside `Sampler.forward()` to compute pythonized sample_result.
57 +@dataclass
58 +class SampleResultArgsType:
59 + sample_metadata: SampleMetadataType
60 + multinomial_samples: MultinomialSamplesType
61 + sample_results_dict: SampleResultsDictType
62 + sampling_metadata: SamplingMetadata
63 + greedy_samples: Optional[torch.Tensor]
64 + beam_search_logprobs: Optional[torch.Tensor]
65 +
66 +
67 +# Union of non-deferred (single-step scheduling)
68 +# vs deferred (multi-step scheduling)
69 +# sample result types
70 +MaybeDeferredSampleResultType = Union[SampleResultType, SampleResultArgsType]
71 +
72 +# Abbreviation of the _sample() return type
73 +SampleReturnType = Tuple[MaybeDeferredSampleResultType, Optional[torch.Tensor]]
74 +
75 +
76 +class SamplerOutput(
77 + msgspec.Struct,
78 + omit_defaults=True, # type: ignore[call-arg]
79 + array_like=True): # type: ignore[call-arg]
80 + """For each sequence group, we generate a list of SequenceOutput object,
81 + each of which contains one possible candidate for the next token.
82 +
83 + This data structure implements methods, so it can be used like a list, but
84 + also has optional fields for device tensors.
85 + """
86 +
87 + outputs: List[CompletionSequenceGroupOutput]
88 +
89 + # On-device tensor containing probabilities of each token.
90 + sampled_token_probs: Optional[torch.Tensor] = None
91 +
92 + # On-device tensor containing the logprobs of each token.
93 + logprobs: Optional["torch.Tensor"] = None
94 +
95 + # Holds either (1) the pythonized sampler result (single-step scheduling)
96 + # or (2) what will be arguments for later deferred pythonization of the
97 + # sampler result (muliti-step scheduling)
98 + deferred_sample_results_args: Optional[SampleResultArgsType] = None
99 +
100 + # On-device tensor containing the sampled token ids.
101 + sampled_token_ids: Optional[torch.Tensor] = None
102 + # CPU tensor containing the sampled token ids. Used during multi-step to
103 + # return the sampled token ids from last rank to AsyncLLMEngine to be
104 + # 'broadcasted' to all other PP ranks for next step.
105 + sampled_token_ids_cpu: Optional[torch.Tensor] = None
106 +
107 + # Spec decode metrics populated by workers.
108 + spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
109 +
110 + # Optional last hidden states from the model.
111 + hidden_states: Optional[torch.Tensor] = None
112 +
113 + # Optional prefill hidden states from the model
114 + # (used for models like EAGLE).
115 + prefill_hidden_states: Optional[torch.Tensor] = None
116 +
117 + # Time taken in the forward pass for this across all workers
118 + model_forward_time: Optional[float] = None
119 +
120 + # Time taken in the model execute function. This will include model forward,
121 + # block/sync across workers, cpu-gpu sync time and sampling time.
122 + model_execute_time: Optional[float] = None
123 +
124 + def __getitem__(self, idx: int):
125 + return self.outputs[idx]
126 +
127 + def __setitem__(self, idx: int, value):
128 + self.outputs[idx] = value
129 +
130 + def __len__(self):
131 + return len(self.outputs)
132 +
133 + def __eq__(self, other: object):
134 + return isinstance(other,
135 + self.__class__) and self.outputs == other.outputs
136 +
137 + def __repr__(self) -> str:
138 + """Show the shape of a tensor instead of its values to reduce noise.
139 + """
140 + sampled_token_probs_repr = ("None" if self.sampled_token_probs is None
141 + else self.sampled_token_probs.shape)
142 + sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else
143 + self.sampled_token_ids.shape)
144 + return (
145 + f"SamplerOutput(outputs={self.outputs}, "
146 + f"sampled_token_probs={sampled_token_probs_repr}, "
147 + f"sampled_token_ids={sampled_token_ids_repr}, "
148 + f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
149 +
150
151 class Sampler(nn.Module):
152 """Samples the next tokens from the model's outputs.
@@ -98,6 +210,19 @@ class Sampler(nn.Module):
210 sampling_metadata: SamplingMetadata,
211 ) -> Optional[SamplerOutput]:
212 """
213 + Single-step scheduling:
214 + * Perform GPU-side sampling computation & compute
215 + GPU-side logprobs tensor
216 + * Pythonize sampling result & logprobs tensor
217 +
218 + Multi-step scheduling:
219 + * Perform GPU-side sampling computation & compute
220 + GPU-side logprobs tensor
221 + * Defer Pythonization of sampling result & logprobs
222 + tensor
223 + * Encapsulate arguments required for deferred Pythonization
224 + in the :class:`SamplerOutput` structure
225 +
226 Args:
227 logits: (num_tokens, vocab_size).
228 sampling_metadata: Metadata for sampling.
@@ -150,7 +275,7 @@ class Sampler(nn.Module):
275 logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
276
277 # Sample the next tokens.
153 - sample_results, maybe_sampled_tokens_tensor = _sample(
278 + maybe_deferred_sample_results, maybe_sampled_tokens_tensor = _sample(
279 probs,
280 logprobs,
281 sampling_metadata,
@@ -160,20 +285,28 @@ class Sampler(nn.Module):
285 )
286
287 if self.include_gpu_probs_tensor:
288 + # Since we will defer sampler result Pythonization,
289 + # preserve GPU-side tensors in support of later
290 + # deferred pythonization of logprobs
291 assert maybe_sampled_tokens_tensor is not None
292 on_device_tensors = (probs, logprobs, maybe_sampled_tokens_tensor)
293 else:
294 + # Since Pythonization has already happened, don't preserve
295 + # GPU-side tensors.
296 on_device_tensors = None
297
298 # Get the logprobs query results.
299 prompt_logprobs = None
300 sample_logprobs = None
301 if not sampling_metadata.skip_sampler_cpu_output:
172 - prompt_logprobs, sample_logprobs = _get_logprobs(
173 - logprobs, sampling_metadata, sample_results)
302 + # Pythonize logprobs now (GPU -> CPU); do not defer.
303 + assert not isinstance(maybe_deferred_sample_results,
304 + SampleResultArgsType)
305 + prompt_logprobs, sample_logprobs = get_logprobs(
306 + logprobs, sampling_metadata, maybe_deferred_sample_results)
307
308 return _build_sampler_output(
176 - sample_results,
309 + maybe_deferred_sample_results,
310 sampling_metadata,
311 prompt_logprobs,
312 sample_logprobs,
@@ -543,6 +676,60 @@ def _top_k_top_p_multinomial_with_flashinfer(
676 return batch_next_token_ids.view(-1, num_samples)
677
678
679 +def get_pythonized_sample_results(
680 + sample_result_args: SampleResultArgsType) -> SampleResultType:
681 + '''This function consumes GPU-side sampler results and computes
682 + Pythonized CPU-side sampler results (GPU -> CPU sync.)
683 +
684 + Single-step scheduling: this function is invoked at sampling-time
685 + for immediate Pythonization.
686 +
687 + Multi-step scheduling: Pythonization is deferred until after multiple
688 + GPU-side steps have been completed.
689 +
690 + Args:
691 + sample_result_args: GPU-side inputs to the Pythonization process
692 +
693 + Returns:
694 + Pythonized sampler results
695 + '''
696 +
697 + (
698 + sample_metadata,
699 + sampling_metadata,
700 + greedy_samples,
701 + multinomial_samples,
702 + beam_search_logprobs,
703 + sample_results_dict,
704 + ) = (
705 + sample_result_args.sample_metadata,
706 + sample_result_args.sampling_metadata,
707 + sample_result_args.greedy_samples,
708 + sample_result_args.multinomial_samples,
709 + sample_result_args.beam_search_logprobs,
710 + sample_result_args.sample_results_dict,
711 + )
712 +
713 + for sampling_type in SamplingType:
714 + if sampling_type not in sample_metadata:
715 + continue
716 + (seq_group_id, seq_groups) = sample_metadata[sampling_type]
717 + if sampling_type == SamplingType.GREEDY:
718 + sample_results = _greedy_sample(seq_groups, greedy_samples)
719 + elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
720 + sample_results = _random_sample(seq_groups,
721 + multinomial_samples[sampling_type])
722 + elif sampling_type == SamplingType.BEAM:
723 + sample_results = _beam_search_sample(seq_groups,
724 + beam_search_logprobs)
725 + sample_results_dict.update(zip(seq_group_id, sample_results))
726 +
727 + return [
728 + sample_results_dict.get(i, ([], []))
729 + for i in range(len(sampling_metadata.seq_groups))
730 + ]
731 +
732 +
733 def _sample_with_torch(
734 probs: torch.Tensor,
735 logprobs: torch.Tensor,
@@ -550,7 +737,19 @@ def _sample_with_torch(
737 sampling_tensors: SamplingTensors,
738 include_gpu_probs_tensor: bool,
739 modify_greedy_probs: bool,
553 -) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
740 +) -> SampleReturnType:
741 + '''Torch-oriented _sample() implementation.
742 +
743 + Single-step scheduling:
744 + * Perform GPU-side sampling computation
745 + * Immediately Pythonize sampling result
746 +
747 + Multi-step scheduling:
748 + * Perform GPU-side sampling computation
749 + * Defer Pythonization & preserve GPU-side
750 + tensors required for Pythonization
751 + '''
752 +
753 categorized_seq_group_ids: Dict[SamplingType,
754 List[int]] = {t: []
755 for t in SamplingType}
@@ -560,10 +759,11 @@ def _sample_with_torch(
759 sampling_type = sampling_params.sampling_type
760 categorized_seq_group_ids[sampling_type].append(i)
761
563 - sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
564 - sample_metadata: Dict[SamplingType,
565 - Tuple[List[int], List[SequenceGroupToSample]]] = {}
566 - multinomial_samples: Dict[SamplingType, torch.Tensor] = {}
762 + sample_results_dict: SampleResultsDictType = {}
763 + sample_metadata: SampleMetadataType = {}
764 + multinomial_samples: MultinomialSamplesType = {}
765 + greedy_samples: Optional[torch.Tensor] = None
766 + beam_search_logprobs: Optional[torch.Tensor] = None
767
768 # Create output tensor for sampled token ids.
769 if include_gpu_probs_tensor:
@@ -638,32 +838,29 @@ def _sample_with_torch(
838 else:
839 raise ValueError(f"Unsupported sampling type: {sampling_type}")
840
641 - # GPU<->CPU sync happens in the loop below.
642 - # This also converts the sample output to Python objects.
841 + # Encapsulate arguments for computing Pythonized sampler
842 + # results, whether deferred or otherwise.
843 + maybe_deferred_args = SampleResultArgsType(
844 + sampling_metadata=sampling_metadata,
845 + sample_metadata=sample_metadata,
846 + multinomial_samples=multinomial_samples,
847 + greedy_samples=greedy_samples,
848 + beam_search_logprobs=beam_search_logprobs,
849 + sample_results_dict=sample_results_dict)
850 +
851 if not sampling_metadata.skip_sampler_cpu_output:
644 - for sampling_type in SamplingType:
645 - if sampling_type not in sample_metadata:
646 - continue
647 - (seq_group_id, seq_groups) = sample_metadata[sampling_type]
648 - if sampling_type == SamplingType.GREEDY:
649 - sample_results = _greedy_sample(seq_groups, greedy_samples)
650 - elif sampling_type in (SamplingType.RANDOM,
651 - SamplingType.RANDOM_SEED):
652 - sample_results = _random_sample(
653 - seq_groups, multinomial_samples[sampling_type])
654 - elif sampling_type == SamplingType.BEAM:
655 - sample_results = _beam_search_sample(seq_groups,
656 - beam_search_logprobs)
657 - sample_results_dict.update(zip(seq_group_id, sample_results))
658 -
659 - sample_results = [
660 - sample_results_dict.get(i, ([], []))
661 - for i in range(len(sampling_metadata.seq_groups))
662 - ]
852 + # GPU<->CPU sync happens here.
853 + # This also converts the sampler output to a Python object.
854 + # Return Pythonized sampler result & sampled token ids
855 + return get_pythonized_sample_results(
856 + maybe_deferred_args), sampled_token_ids_tensor
857 else:
664 - sample_results = []
665 -
666 - return sample_results, sampled_token_ids_tensor
858 + # Defer sampler result Pythonization; return deferred
859 + # Pythonization args & sampled token ids
860 + return (
861 + maybe_deferred_args,
862 + sampled_token_ids_tensor,
863 + )
864
865
866 def _sample_with_triton_kernel(
@@ -755,7 +952,7 @@ def _sample(
952 sampling_tensors: SamplingTensors,
953 include_gpu_probs_tensor: bool,
954 modify_greedy_probs: bool,
758 -) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
955 +) -> SampleReturnType:
956 """
957 Args:
958 probs: (num_query_tokens_in_batch, num_vocab)
@@ -803,7 +1000,7 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
1000 return result.sum(1).add_(1)
1001
1002
806 -def _get_logprobs(
1003 +def get_logprobs(
1004 logprobs: torch.Tensor,
1005 sampling_metadata: SamplingMetadata,
1006 sample_results: SampleResultType,
@@ -1126,7 +1323,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
1323
1324
1325 def _build_sampler_output(
1129 - sample_results: SampleResultType,
1326 + maybe_deferred_sample_results: MaybeDeferredSampleResultType,
1327 sampling_metadata: SamplingMetadata,
1328 prompt_logprobs: Optional[List[Optional[PromptLogprobs]]],
1329 sample_logprobs: Optional[List[SampleLogprobs]],
@@ -1143,14 +1340,21 @@ def _build_sampler_output(
1340 speculative decoding rejection sampling.
1341 """
1342 sampler_output: List[CompletionSequenceGroupOutput] = []
1146 - if not skip_sampler_cpu_output:
1343 +
1344 + if skip_sampler_cpu_output:
1345 + assert isinstance(maybe_deferred_sample_results, SampleResultArgsType)
1346 + deferred_sample_results_args = maybe_deferred_sample_results
1347 + else:
1348 assert prompt_logprobs is not None
1349 assert sample_logprobs is not None
1350 + assert not isinstance(maybe_deferred_sample_results,
1351 + SampleResultArgsType)
1352 + deferred_sample_results_args = None
1353
1354 for (seq_group, sample_result, group_prompt_logprobs,
1355 group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1152 - sample_results, prompt_logprobs,
1153 - sample_logprobs):
1356 + maybe_deferred_sample_results,
1357 + prompt_logprobs, sample_logprobs):
1358 seq_ids = seq_group.seq_ids
1359 next_token_ids, parent_ids = sample_result
1360 seq_outputs: List[SequenceOutput] = []
@@ -1176,7 +1380,7 @@ def _build_sampler_output(
1380 sampled_token_probs=sampled_token_probs,
1381 sampled_token_ids=sampled_token_ids,
1382 logprobs=logprobs_tensor,
1179 - )
1383 + deferred_sample_results_args=deferred_sample_results_args)
1384
1385
1386 def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
vllm/model_executor/model_loader/neuron.py
+1 -2
@@ -10,9 +10,8 @@ from transformers import PretrainedConfig
10
11 from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 -from vllm.model_executor.layers.sampler import Sampler
13 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
15 -from vllm.sequence import SamplerOutput
15
16 TORCH_DTYPE_TO_NEURON_AMP = {
17 "auto": "f32",
vllm/model_executor/model_loader/openvino.py
+1 -2
@@ -15,9 +15,8 @@ from vllm.config import DeviceConfig, ModelConfig
15 from vllm.logger import init_logger
16 from vllm.model_executor.layers.logits_processor import (LogitsProcessor,
17 _prune_hidden_states)
18 -from vllm.model_executor.layers.sampler import Sampler
18 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 -from vllm.sequence import SamplerOutput
20
21 logger = init_logger(__name__)
22
vllm/model_executor/models/arctic.py
+2 -2
@@ -23,13 +23,13 @@ from vllm.model_executor.layers.quantization.base_config import (
23 from vllm.model_executor.layers.quantization.deepspeedfp import (
24 DeepSpeedFPConfig, DeepSpeedFPParameter)
25 from vllm.model_executor.layers.rotary_embedding import get_rope
26 -from vllm.model_executor.layers.sampler import Sampler
26 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
27 from vllm.model_executor.layers.vocab_parallel_embedding import (
28 ParallelLMHead, VocabParallelEmbedding)
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 from vllm.model_executor.sampling_metadata import SamplingMetadata
31 from vllm.model_executor.utils import set_weight_attrs
32 -from vllm.sequence import IntermediateTensors, SamplerOutput
32 +from vllm.sequence import IntermediateTensors
33 from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35 logger = init_logger(__name__)
vllm/model_executor/models/baichuan.py
+2 -2
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization.base_config import (
39 QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler
41 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import IntermediateTensors, SamplerOutput
46 +from vllm.sequence import IntermediateTensors
47
48 from .interfaces import SupportsLoRA
49
vllm/model_executor/models/bart.py
+2 -2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 -from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43
44 logger = logging.get_logger(__name__)
45
vllm/model_executor/models/blip2.py
+2 -2
@@ -13,13 +13,13 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.logits_processor import LogitsProcessor
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 -from vllm.model_executor.layers.sampler import Sampler
16 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 from vllm.model_executor.models.opt import OPTModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
22 - SamplerOutput, SequenceData)
22 + SequenceData)
23
24 from .blip import (BlipVisionModel, dummy_image_for_blip,
25 get_max_blip_image_tokens)
vllm/model_executor/models/bloom.py
+2 -2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 -from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43
44
45 def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
vllm/model_executor/models/chameleon.py
+2 -2
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22 from vllm.model_executor.layers.logits_processor import LogitsProcessor
23 from vllm.model_executor.layers.quantization import QuantizationConfig
24 from vllm.model_executor.layers.rotary_embedding import get_rope
25 -from vllm.model_executor.layers.sampler import Sampler
25 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26 from vllm.model_executor.layers.vocab_parallel_embedding import (
27 ParallelLMHead, VocabParallelEmbedding)
28 from vllm.model_executor.model_loader.weight_utils import (
@@ -33,7 +33,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.utils import (cached_get_tokenizer,
34 repeat_and_pad_placeholder_tokens)
35 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36 - SamplerOutput, SequenceData)
36 + SequenceData)
37 from vllm.utils import print_warning_once
38
39 from .interfaces import SupportsMultiModal
vllm/model_executor/models/chatglm.py
+2 -2
@@ -20,12 +20,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 from vllm.model_executor.layers.quantization.base_config import (
21 QuantizationConfig)
22 from vllm.model_executor.layers.rotary_embedding import get_rope
23 -from vllm.model_executor.layers.sampler import Sampler
23 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 ParallelLMHead, VocabParallelEmbedding)
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 -from vllm.sequence import IntermediateTensors, SamplerOutput
28 +from vllm.sequence import IntermediateTensors
29 from vllm.transformers_utils.configs import ChatGLMConfig
30
31 from .interfaces import SupportsLoRA
vllm/model_executor/models/commandr.py
+2 -2
@@ -38,14 +38,14 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization.base_config import (
39 QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler
41 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import (
45 default_weight_loader, row_parallel_weight_loader)
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.model_executor.utils import set_weight_attrs
48 -from vllm.sequence import IntermediateTensors, SamplerOutput
48 +from vllm.sequence import IntermediateTensors
49
50
51 @torch.compile
vllm/model_executor/models/dbrx.py
+2 -2
@@ -17,13 +17,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 from vllm.model_executor.layers.quantization.base_config import (
18 QuantizationConfig)
19 from vllm.model_executor.layers.rotary_embedding import get_rope
20 -from vllm.model_executor.layers.sampler import Sampler
20 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21 from vllm.model_executor.layers.vocab_parallel_embedding import (
22 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.model_executor.utils import set_weight_attrs
26 -from vllm.sequence import IntermediateTensors, SamplerOutput
26 +from vllm.sequence import IntermediateTensors
27 from vllm.transformers_utils.configs.dbrx import DbrxConfig
28
29
vllm/model_executor/models/deepseek.py
+2 -2
@@ -43,12 +43,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 from vllm.model_executor.layers.quantization.base_config import (
44 QuantizationConfig)
45 from vllm.model_executor.layers.rotary_embedding import get_rope
46 -from vllm.model_executor.layers.sampler import Sampler
46 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
48 ParallelLMHead, VocabParallelEmbedding)
49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
51 -from vllm.sequence import IntermediateTensors, SamplerOutput
51 +from vllm.sequence import IntermediateTensors
52
53
54 class DeepseekMLP(nn.Module):
vllm/model_executor/models/deepseek_v2.py
+2 -2
@@ -43,12 +43,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
43 from vllm.model_executor.layers.quantization.base_config import (
44 QuantizationConfig)
45 from vllm.model_executor.layers.rotary_embedding import get_rope
46 -from vllm.model_executor.layers.sampler import Sampler
46 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47 from vllm.model_executor.layers.vocab_parallel_embedding import (
48 ParallelLMHead, VocabParallelEmbedding)
49 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
51 -from vllm.sequence import IntermediateTensors, SamplerOutput
51 +from vllm.sequence import IntermediateTensors
52
53 from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
54
vllm/model_executor/models/eagle.py
+2 -1
@@ -5,12 +5,13 @@ import torch.nn as nn
5
6 from vllm.attention.backends.abstract import AttentionMetadata
7 from vllm.model_executor.layers.logits_processor import LogitsProcessor
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.model_executor.layers.vocab_parallel_embedding import (
10 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
11 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
12 from vllm.model_executor.models import ModelRegistry
13 from vllm.model_executor.sampling_metadata import SamplingMetadata
13 -from vllm.sequence import IntermediateTensors, SamplerOutput
14 +from vllm.sequence import IntermediateTensors
15 from vllm.transformers_utils.configs.eagle import EAGLEConfig
16
17
vllm/model_executor/models/falcon.py
+2 -2
@@ -39,12 +39,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 from vllm.model_executor.layers.quantization.base_config import (
40 QuantizationConfig)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 -from vllm.model_executor.layers.sampler import Sampler
42 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 -from vllm.sequence import IntermediateTensors, SamplerOutput
47 +from vllm.sequence import IntermediateTensors
48 from vllm.transformers_utils.configs import RWConfig
49
50 FalconConfig = Union[HF_FalconConfig, RWConfig]
vllm/model_executor/models/fuyu.py
+2 -1
@@ -31,6 +31,7 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31 from vllm.logger import init_logger
32 from vllm.model_executor.layers.linear import ColumnParallelLinear
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 +from vllm.model_executor.layers.sampler import SamplerOutput
35 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
36 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -39,7 +40,7 @@ from vllm.multimodal.base import MultiModalInputs
40 from vllm.multimodal.image import cached_get_image_processor
41 from vllm.multimodal.utils import cached_get_tokenizer
42 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 - SamplerOutput, SequenceData)
43 + SequenceData)
44
45 from .interfaces import SupportsMultiModal
46 from .utils import merge_multimodal_embeddings
vllm/model_executor/models/gemma.py
+2 -2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.quantization.base_config import (
35 QuantizationConfig)
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 -from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43
44 from .interfaces import SupportsLoRA
45
vllm/model_executor/models/gemma2.py
+2 -2
@@ -33,12 +33,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization.base_config import (
34 QuantizationConfig)
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler
36 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.sequence import IntermediateTensors, SamplerOutput
41 +from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsLoRA
44
vllm/model_executor/models/gpt2.py
+2 -2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 -from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43
44 from .utils import is_pp_missing_parameter, make_layers
45
vllm/model_executor/models/gpt_bigcode.py
+2 -2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 -from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43
44 from .interfaces import SupportsLoRA
45
vllm/model_executor/models/gpt_j.py
+2 -2
@@ -33,12 +33,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization.base_config import (
34 QuantizationConfig)
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler
36 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.sequence import IntermediateTensors, SamplerOutput
41 +from vllm.sequence import IntermediateTensors
42
43
44 class GPTJAttention(nn.Module):
vllm/model_executor/models/gpt_neox.py
+2 -2
@@ -33,12 +33,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization.base_config import (
34 QuantizationConfig)
35 from vllm.model_executor.layers.rotary_embedding import get_rope
36 -from vllm.model_executor.layers.sampler import Sampler
36 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 ParallelLMHead, VocabParallelEmbedding)
39 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.sampling_metadata import SamplingMetadata
41 -from vllm.sequence import IntermediateTensors, SamplerOutput
41 +from vllm.sequence import IntermediateTensors
42
43
44 class GPTNeoXAttention(nn.Module):
vllm/model_executor/models/internlm2.py
+2 -2
@@ -17,12 +17,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 from vllm.model_executor.layers.quantization.base_config import (
18 QuantizationConfig)
19 from vllm.model_executor.layers.rotary_embedding import get_rope
20 -from vllm.model_executor.layers.sampler import Sampler
20 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21 from vllm.model_executor.layers.vocab_parallel_embedding import (
22 ParallelLMHead, VocabParallelEmbedding)
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 -from vllm.sequence import IntermediateTensors, SamplerOutput
25 +from vllm.sequence import IntermediateTensors
26
27
28 class InternLM2MLP(nn.Module):
vllm/model_executor/models/internvl.py
+2 -1
@@ -18,13 +18,14 @@ from vllm.attention import AttentionMetadata
18 from vllm.config import CacheConfig, MultiModalConfig
19 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
20 from vllm.model_executor.layers.quantization import QuantizationConfig
21 +from vllm.model_executor.layers.sampler import SamplerOutput
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 from vllm.model_executor.models.intern_vit import InternVisionModel
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 from vllm.multimodal import MULTIMODAL_REGISTRY
26 from vllm.multimodal.base import MultiModalInputs
27 from vllm.multimodal.utils import cached_get_tokenizer
27 -from vllm.sequence import IntermediateTensors, SamplerOutput
28 +from vllm.sequence import IntermediateTensors
29
30 from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
31 get_clip_num_patches)
vllm/model_executor/models/jais.py
+2 -2
@@ -35,12 +35,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35 from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.quantization.base_config import (
37 QuantizationConfig)
38 -from vllm.model_executor.layers.sampler import Sampler
38 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 ParallelLMHead, VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 -from vllm.sequence import IntermediateTensors, SamplerOutput
43 +from vllm.sequence import IntermediateTensors
44 from vllm.transformers_utils.configs import JAISConfig
45
46 from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/jamba.py
+2 -2
@@ -27,14 +27,14 @@ from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
27 selective_scan_fn, selective_state_update)
28 from vllm.model_executor.layers.quantization.base_config import (
29 QuantizationConfig)
30 -from vllm.model_executor.layers.sampler import Sampler
30 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
31 from vllm.model_executor.layers.vocab_parallel_embedding import (
32 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
33 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34 from vllm.model_executor.models.interfaces import HasInnerState
35 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 from vllm.model_executor.utils import set_weight_attrs
37 -from vllm.sequence import IntermediateTensors, SamplerOutput
37 +from vllm.sequence import IntermediateTensors
38 from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
39 _get_graph_batch_size)
40
vllm/model_executor/models/llama.py
+2 -2
@@ -42,13 +42,13 @@ from vllm.model_executor.layers.quantization.base_config import (
42 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43 get_compressed_tensors_cache_scale)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler
45 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import (
49 default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
50 from vllm.model_executor.sampling_metadata import SamplingMetadata
51 -from vllm.sequence import IntermediateTensors, SamplerOutput
51 +from vllm.sequence import IntermediateTensors
52 from vllm.utils import is_hip
53
54 from .interfaces import SupportsLoRA
vllm/model_executor/models/llava.py
+2 -1
@@ -11,10 +11,11 @@ from vllm.config import CacheConfig, MultiModalConfig
11 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12 from vllm.model_executor.layers.activation import get_act_fn
13 from vllm.model_executor.layers.quantization import QuantizationConfig
14 +from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
17 -from vllm.sequence import IntermediateTensors, SamplerOutput
18 +from vllm.sequence import IntermediateTensors
19
20 from .clip import (CLIPVisionModel, dummy_image_for_clip,
21 dummy_seq_data_for_clip, get_max_clip_image_tokens,
vllm/model_executor/models/llava_next.py
+2 -1
@@ -15,10 +15,11 @@ from vllm.config import CacheConfig, MultiModalConfig
15 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
16 from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 +from vllm.model_executor.layers.sampler import SamplerOutput
19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 from vllm.model_executor.sampling_metadata import SamplingMetadata
21 from vllm.multimodal import MULTIMODAL_REGISTRY
21 -from vllm.sequence import IntermediateTensors, SamplerOutput
22 +from vllm.sequence import IntermediateTensors
23 from vllm.utils import is_list_of
24
25 from .clip import (CLIPVisionModel, dummy_image_for_clip,
vllm/model_executor/models/medusa.py
+1 -1
@@ -4,11 +4,11 @@ import torch
4 import torch.nn as nn
5
6 from vllm.model_executor.layers.logits_processor import LogitsProcessor
7 +from vllm.model_executor.layers.sampler import SamplerOutput
8 from vllm.model_executor.layers.vocab_parallel_embedding import (
9 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
10 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
11 from vllm.model_executor.sampling_metadata import SamplingMetadata
11 -from vllm.sequence import SamplerOutput
12 from vllm.transformers_utils.configs.medusa import MedusaConfig
13
14
vllm/model_executor/models/minicpm.py
+2 -2
@@ -44,13 +44,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
44 from vllm.model_executor.layers.quantization.base_config import (
45 QuantizationConfig)
46 from vllm.model_executor.layers.rotary_embedding import get_rope
47 -from vllm.model_executor.layers.sampler import Sampler
47 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
48 from vllm.model_executor.layers.vocab_parallel_embedding import (
49 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 from vllm.model_executor.utils import set_weight_attrs
53 -from vllm.sequence import IntermediateTensors, SamplerOutput
53 +from vllm.sequence import IntermediateTensors
54
55 from .interfaces import SupportsLoRA
56
vllm/model_executor/models/minicpmv.py
+2 -2
@@ -44,7 +44,7 @@ from vllm.logger import init_logger
44 from vllm.model_executor.layers.linear import ReplicatedLinear
45 from vllm.model_executor.layers.logits_processor import LogitsProcessor
46 from vllm.model_executor.layers.quantization import QuantizationConfig
47 -from vllm.model_executor.layers.sampler import Sampler
47 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
48 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
50 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -57,7 +57,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
57 from vllm.multimodal.image import cached_get_image_processor
58 from vllm.multimodal.utils import cached_get_tokenizer
59 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60 - SamplerOutput, SequenceData)
60 + SequenceData)
61
62 from .idefics2_vision_model import Idefics2VisionTransformer
63
vllm/model_executor/models/mixtral.py
+2 -2
@@ -39,13 +39,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39 from vllm.model_executor.layers.quantization.base_config import (
40 QuantizationConfig)
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 -from vllm.model_executor.layers.sampler import Sampler
42 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import (
46 default_weight_loader, maybe_remap_kv_scale_name)
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 -from vllm.sequence import IntermediateTensors, SamplerOutput
48 +from vllm.sequence import IntermediateTensors
49
50 from .interfaces import SupportsLoRA
51 from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/mixtral_quant.py
+2 -2
@@ -42,12 +42,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization.base_config import (
43 QuantizationConfig)
44 from vllm.model_executor.layers.rotary_embedding import get_rope
45 -from vllm.model_executor.layers.sampler import Sampler
45 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46 from vllm.model_executor.layers.vocab_parallel_embedding import (
47 ParallelLMHead, VocabParallelEmbedding)
48 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49 from vllm.model_executor.sampling_metadata import SamplingMetadata
50 -from vllm.sequence import IntermediateTensors, SamplerOutput
50 +from vllm.sequence import IntermediateTensors
51
52
53 class MixtralMLP(nn.Module):
vllm/model_executor/models/mlp_speculator.py
+1 -2
@@ -6,11 +6,10 @@ import torch.nn as nn
6
7 from vllm.model_executor import SamplingMetadata
8 from vllm.model_executor.layers.logits_processor import LogitsProcessor
9 -from vllm.model_executor.layers.sampler import Sampler
9 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
10 from vllm.model_executor.layers.vocab_parallel_embedding import (
11 ParallelLMHead, VocabParallelEmbedding)
12 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 -from vllm.sequence import SamplerOutput
13 from vllm.transformers_utils.configs import MLPSpeculatorConfig
14
15 SQRT2 = 2**0.5
vllm/model_executor/models/mpt.py
+2 -2
@@ -17,12 +17,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.quantization.base_config import (
19 QuantizationConfig)
20 -from vllm.model_executor.layers.sampler import Sampler
20 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21 from vllm.model_executor.layers.vocab_parallel_embedding import (
22 VocabParallelEmbedding)
23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24 from vllm.model_executor.sampling_metadata import SamplingMetadata
25 -from vllm.sequence import IntermediateTensors, SamplerOutput
25 +from vllm.sequence import IntermediateTensors
26 from vllm.transformers_utils.configs.mpt import MPTConfig
27
28
vllm/model_executor/models/nemotron.py
+2 -2
@@ -37,13 +37,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.quantization.base_config import (
38 QuantizationConfig)
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 -from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.model_loader.weight_utils import (
44 default_weight_loader, maybe_remap_kv_scale_name)
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import IntermediateTensors, SamplerOutput
46 +from vllm.sequence import IntermediateTensors
47 from vllm.transformers_utils.configs import NemotronConfig
48
49 from .interfaces import SupportsLoRA
vllm/model_executor/models/olmo.py
+2 -2
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization.base_config import (
39 QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler
41 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import IntermediateTensors, SamplerOutput
46 +from vllm.sequence import IntermediateTensors
47
48
49 class OlmoAttention(nn.Module):
vllm/model_executor/models/opt.py
+2 -2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 -from vllm.model_executor.layers.sampler import Sampler
37 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 ParallelLMHead, VocabParallelEmbedding)
40 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41 from vllm.model_executor.sampling_metadata import SamplingMetadata
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43
44
45 class OPTLearnedPositionalEmbedding(nn.Embedding):
vllm/model_executor/models/orion.py
+2 -2
@@ -21,12 +21,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 from vllm.model_executor.layers.quantization.base_config import (
22 QuantizationConfig)
23 from vllm.model_executor.layers.rotary_embedding import get_rope
24 -from vllm.model_executor.layers.sampler import Sampler
24 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25 from vllm.model_executor.layers.vocab_parallel_embedding import (
26 ParallelLMHead, VocabParallelEmbedding)
27 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 -from vllm.sequence import IntermediateTensors, SamplerOutput
29 +from vllm.sequence import IntermediateTensors
30
31
32 class OrionMLP(nn.Module):
vllm/model_executor/models/paligemma.py
+2 -2
@@ -11,13 +11,13 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 from vllm.logger import init_logger
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 from vllm.model_executor.layers.quantization import QuantizationConfig
14 -from vllm.model_executor.layers.sampler import Sampler
14 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
15 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16 from vllm.model_executor.models.gemma import GemmaModel
17 from vllm.model_executor.sampling_metadata import SamplingMetadata
18 from vllm.multimodal import MULTIMODAL_REGISTRY
19 from vllm.multimodal.utils import cached_get_tokenizer
20 -from vllm.sequence import IntermediateTensors, SamplerOutput
20 +from vllm.sequence import IntermediateTensors
21
22 from .interfaces import SupportsMultiModal
23 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
vllm/model_executor/models/persimmon.py
+2 -2
@@ -37,12 +37,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
37 from vllm.model_executor.layers.quantization.base_config import (
38 QuantizationConfig)
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 -from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
42 ParallelLMHead, VocabParallelEmbedding)
43 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 -from vllm.sequence import IntermediateTensors, SamplerOutput
45 +from vllm.sequence import IntermediateTensors
46
47
48 class PersimmonMLP(nn.Module):
vllm/model_executor/models/phi.py
+2 -2
@@ -52,12 +52,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
52 from vllm.model_executor.layers.quantization.base_config import (
53 QuantizationConfig)
54 from vllm.model_executor.layers.rotary_embedding import get_rope
55 -from vllm.model_executor.layers.sampler import Sampler
55 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
56 from vllm.model_executor.layers.vocab_parallel_embedding import (
57 ParallelLMHead, VocabParallelEmbedding)
58 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59 from vllm.model_executor.sampling_metadata import SamplingMetadata
60 -from vllm.sequence import IntermediateTensors, SamplerOutput
60 +from vllm.sequence import IntermediateTensors
61
62 from .interfaces import SupportsLoRA
63
vllm/model_executor/models/phi3_small.py
+2 -2
@@ -16,12 +16,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 from vllm.model_executor.layers.quantization.base_config import (
17 QuantizationConfig)
18 from vllm.model_executor.layers.rotary_embedding import get_rope
19 -from vllm.model_executor.layers.sampler import Sampler
19 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
20 from vllm.model_executor.layers.vocab_parallel_embedding import (
21 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 -from vllm.sequence import IntermediateTensors, SamplerOutput
24 +from vllm.sequence import IntermediateTensors
25
26
27 def load_column_parallel_weight(param: torch.nn.Parameter,
vllm/model_executor/models/phi3v.py
+2 -2
@@ -31,7 +31,7 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31 from vllm.logger import init_logger
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 -from vllm.model_executor.layers.sampler import Sampler
34 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
37 from vllm.model_executor.models.clip import CLIPVisionModel
@@ -39,7 +39,7 @@ from vllm.model_executor.models.llama import LlamaModel
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.multimodal import MULTIMODAL_REGISTRY
41 from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
42 -from vllm.sequence import IntermediateTensors, SamplerOutput
42 +from vllm.sequence import IntermediateTensors
43 from vllm.utils import is_list_of
44
45 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
vllm/model_executor/models/qwen.py
+2 -2
@@ -22,12 +22,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
22 from vllm.model_executor.layers.quantization.base_config import (
23 QuantizationConfig)
24 from vllm.model_executor.layers.rotary_embedding import get_rope
25 -from vllm.model_executor.layers.sampler import Sampler
25 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26 from vllm.model_executor.layers.vocab_parallel_embedding import (
27 ParallelLMHead, VocabParallelEmbedding)
28 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29 from vllm.model_executor.sampling_metadata import SamplingMetadata
30 -from vllm.sequence import IntermediateTensors, SamplerOutput
30 +from vllm.sequence import IntermediateTensors
31 from vllm.utils import print_warning_once
32
33 from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/qwen2.py
+2 -2
@@ -40,13 +40,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
40 from vllm.model_executor.layers.quantization.base_config import (
41 QuantizationConfig)
42 from vllm.model_executor.layers.rotary_embedding import get_rope
43 -from vllm.model_executor.layers.sampler import Sampler
43 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
44 from vllm.model_executor.layers.vocab_parallel_embedding import (
45 ParallelLMHead, VocabParallelEmbedding)
46 from vllm.model_executor.model_loader.weight_utils import (
47 default_weight_loader, maybe_remap_kv_scale_name)
48 from vllm.model_executor.sampling_metadata import SamplingMetadata
49 -from vllm.sequence import IntermediateTensors, SamplerOutput
49 +from vllm.sequence import IntermediateTensors
50
51 from .interfaces import SupportsLoRA
52 from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/qwen2_moe.py
+2 -2
@@ -45,12 +45,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
45 from vllm.model_executor.layers.quantization.base_config import (
46 QuantizationConfig)
47 from vllm.model_executor.layers.rotary_embedding import get_rope
48 -from vllm.model_executor.layers.sampler import Sampler
48 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
49 from vllm.model_executor.layers.vocab_parallel_embedding import (
50 ParallelLMHead, VocabParallelEmbedding)
51 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52 from vllm.model_executor.sampling_metadata import SamplingMetadata
53 -from vllm.sequence import IntermediateTensors, SamplerOutput
53 +from vllm.sequence import IntermediateTensors
54 from vllm.utils import print_warning_once
55
56 from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/stablelm.py
+2 -2
@@ -36,12 +36,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36 from vllm.model_executor.layers.quantization.base_config import (
37 QuantizationConfig)
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 -from vllm.model_executor.layers.sampler import Sampler
39 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 -from vllm.sequence import IntermediateTensors, SamplerOutput
44 +from vllm.sequence import IntermediateTensors
45
46
47 class StablelmMLP(nn.Module):
vllm/model_executor/models/starcoder2.py
+2 -2
@@ -35,12 +35,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 from vllm.model_executor.layers.rotary_embedding import get_rope
38 -from vllm.model_executor.layers.sampler import Sampler
38 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 from vllm.model_executor.layers.vocab_parallel_embedding import (
40 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 -from vllm.sequence import IntermediateTensors, SamplerOutput
43 +from vllm.sequence import IntermediateTensors
44
45
46 class Starcoder2Attention(nn.Module):
vllm/model_executor/models/ultravox.py
+2 -1
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
27 from vllm.model_executor.layers.layernorm import RMSNorm
28 from vllm.model_executor.layers.quantization.base_config import (
29 QuantizationConfig)
30 +from vllm.model_executor.layers.sampler import SamplerOutput
31 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32 from vllm.model_executor.models.interfaces import SupportsMultiModal
33 from vllm.model_executor.models.utils import (filter_weights,
@@ -37,7 +38,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
38 from vllm.multimodal.base import MultiModalInputs
39 from vllm.multimodal.utils import (cached_get_tokenizer,
40 repeat_and_pad_placeholder_tokens)
40 -from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SamplerOutput, SequenceData
41 +from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
42 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
43
44 _AUDIO_PLACEHOLDER_TOKEN = 128002
vllm/model_executor/models/xverse.py
+2 -2
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.quantization.base_config import (
39 QuantizationConfig)
40 from vllm.model_executor.layers.rotary_embedding import get_rope
41 -from vllm.model_executor.layers.sampler import Sampler
41 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42 from vllm.model_executor.layers.vocab_parallel_embedding import (
43 ParallelLMHead, VocabParallelEmbedding)
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.sampling_metadata import SamplingMetadata
46 -from vllm.sequence import IntermediateTensors, SamplerOutput
46 +from vllm.sequence import IntermediateTensors
47
48 from .interfaces import SupportsLoRA
49
vllm/sequence.py
-70
@@ -1060,76 +1060,6 @@ class IntermediateTensors(
1060 return f"IntermediateTensors(tensors={self.tensors})"
1061
1062
1063 -class SamplerOutput(
1064 - msgspec.Struct,
1065 - omit_defaults=True, # type: ignore[call-arg]
1066 - array_like=True): # type: ignore[call-arg]
1067 - """For each sequence group, we generate a list of SequenceOutput object,
1068 - each of which contains one possible candidate for the next token.
1069 -
1070 - This data structure implements methods, so it can be used like a list, but
1071 - also has optional fields for device tensors.
1072 - """
1073 -
1074 - outputs: List[CompletionSequenceGroupOutput]
1075 -
1076 - # On-device tensor containing probabilities of each token.
1077 - sampled_token_probs: Optional[torch.Tensor] = None
1078 -
1079 - # On-device tensor containing the logprobs of each token.
1080 - logprobs: Optional["torch.Tensor"] = None
1081 -
1082 - # On-device tensor containing the sampled token ids.
1083 - sampled_token_ids: Optional[torch.Tensor] = None
1084 - # CPU tensor containing the sampled token ids. Used during multi-step to
1085 - # return the sampled token ids from last rank to AsyncLLMEngine to be
1086 - # 'broadcasted' to all other PP ranks for next step.
1087 - sampled_token_ids_cpu: Optional[torch.Tensor] = None
1088 -
1089 - # Spec decode metrics populated by workers.
1090 - spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1091 -
1092 - # Optional last hidden states from the model.
1093 - hidden_states: Optional[torch.Tensor] = None
1094 -
1095 - # Optional prefill hidden states from the model
1096 - # (used for models like EAGLE).
1097 - prefill_hidden_states: Optional[torch.Tensor] = None
1098 -
1099 - # Time taken in the forward pass for this across all workers
1100 - model_forward_time: Optional[float] = None
1101 -
1102 - # Time taken in the model execute function. This will include model forward,
1103 - # block/sync across workers, cpu-gpu sync time and sampling time.
1104 - model_execute_time: Optional[float] = None
1105 -
1106 - def __getitem__(self, idx: int):
1107 - return self.outputs[idx]
1108 -
1109 - def __setitem__(self, idx: int, value):
1110 - self.outputs[idx] = value
1111 -
1112 - def __len__(self):
1113 - return len(self.outputs)
1114 -
1115 - def __eq__(self, other: object):
1116 - return isinstance(other,
1117 - self.__class__) and self.outputs == other.outputs
1118 -
1119 - def __repr__(self) -> str:
1120 - """Show the shape of a tensor instead of its values to reduce noise.
1121 - """
1122 - sampled_token_probs_repr = ("None" if self.sampled_token_probs is None
1123 - else self.sampled_token_probs.shape)
1124 - sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else
1125 - self.sampled_token_ids.shape)
1126 - return (
1127 - f"SamplerOutput(outputs={self.outputs}, "
1128 - f"sampled_token_probs={sampled_token_probs_repr}, "
1129 - f"sampled_token_ids={sampled_token_ids_repr}, "
1130 - f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
1131 -
1132 -
1063 class PoolerOutput(
1064 msgspec.Struct,
1065 omit_defaults=True, # type: ignore[call-arg]
vllm/spec_decode/batch_expansion.py
+2 -1
@@ -5,8 +5,9 @@ from typing import Iterator, List, Optional, Tuple
5 import torch
6
7 from vllm import SamplingParams
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
9 - SamplerOutput, SequenceData, SequenceGroupMetadata,
10 + SequenceData, SequenceGroupMetadata,
11 get_all_seq_ids)
12 from vllm.spec_decode.interfaces import (SpeculativeProposals,
13 SpeculativeScorer, SpeculativeScores)
vllm/spec_decode/draft_model_runner.py
+2 -2
@@ -3,6 +3,7 @@ from typing import List, Optional
3 import torch
4
5 from vllm import _custom_ops as ops
6 +from vllm.model_executor.layers.sampler import SamplerOutput
7
8 try:
9 from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -16,8 +17,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
17 PromptAdapterConfig, SchedulerConfig)
18 from vllm.logger import init_logger
19 from vllm.multimodal import MultiModalInputs
19 -from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
20 - SamplerOutput)
20 +from vllm.sequence import ExecuteModelRequest, IntermediateTensors
21 from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
22 ModelRunner)
23
vllm/spec_decode/medusa_worker.py
+2 -2
@@ -4,8 +4,8 @@ from typing import List, Optional, Set, Tuple
4 import torch
5
6 from vllm.model_executor import SamplingMetadata
7 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
8 - SequenceGroupMetadata)
7 +from vllm.model_executor.layers.sampler import SamplerOutput
8 +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
9 from vllm.spec_decode.interfaces import SpeculativeProposals
10 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11 from vllm.spec_decode.top1_proposer import Top1Proposer
vllm/spec_decode/mlp_speculator_worker.py
+2 -2
@@ -3,8 +3,8 @@ from typing import List, Optional, Set, Tuple
3 import torch
4
5 from vllm.model_executor import SamplingMetadata
6 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7 - SequenceGroupMetadata)
6 +from vllm.model_executor.layers.sampler import SamplerOutput
7 +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
8 from vllm.spec_decode.multi_step_worker import MultiStepWorker
9 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10
vllm/spec_decode/multi_step_worker.py
+3 -2
@@ -4,8 +4,9 @@ from typing import Dict, List, Set, Tuple
4
5 import torch
6
7 -from vllm.sequence import (ExecuteModelRequest, HiddenStates, SamplerOutput,
8 - SequenceData, SequenceGroupMetadata)
7 +from vllm.model_executor.layers.sampler import SamplerOutput
8 +from vllm.sequence import (ExecuteModelRequest, HiddenStates, SequenceData,
9 + SequenceGroupMetadata)
10 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
11 from vllm.spec_decode.interfaces import (SpeculativeProposals,
12 SpeculativeProposer)
vllm/spec_decode/ngram_worker.py
+2 -1
@@ -3,7 +3,8 @@ from typing import List, Optional, Set, Tuple
3
4 import torch
5
6 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
6 +from vllm.model_executor.layers.sampler import SamplerOutput
7 +from vllm.sequence import ExecuteModelRequest
8 from vllm.spec_decode.interfaces import SpeculativeProposals
9 from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10 from vllm.spec_decode.top1_proposer import Top1Proposer
vllm/spec_decode/proposer_worker_base.py
+2 -1
@@ -1,7 +1,8 @@
1 from abc import ABC, abstractmethod
2 from typing import List, Optional, Set, Tuple
3
4 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
4 +from vllm.model_executor.layers.sampler import SamplerOutput
5 +from vllm.sequence import ExecuteModelRequest
6 from vllm.spec_decode.interfaces import SpeculativeProposer
7 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
8
vllm/spec_decode/smaller_tp_proposer_worker.py
+2 -1
@@ -6,7 +6,8 @@ from vllm.distributed.parallel_state import (get_tp_group,
6 init_model_parallel_group,
7 patch_tensor_parallel_group)
8 from vllm.logger import init_logger
9 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 +from vllm.model_executor.layers.sampler import SamplerOutput
10 +from vllm.sequence import ExecuteModelRequest
11 from vllm.spec_decode.interfaces import SpeculativeProposals
12 from vllm.spec_decode.multi_step_worker import MultiStepWorker
13 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
vllm/spec_decode/spec_decode_worker.py
+2 -1
@@ -8,12 +8,13 @@ from vllm.config import ParallelConfig, SpeculativeConfig
8 from vllm.distributed.communication_op import broadcast_tensor_dict
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11 +from vllm.model_executor.layers.sampler import SamplerOutput
12 from vllm.model_executor.layers.spec_decode_base_sampler import (
13 SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
14 from vllm.model_executor.layers.typical_acceptance_sampler import (
15 TypicalAcceptanceSampler)
16 from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16 - HiddenStates, SamplerOutput, SequenceGroupMetadata,
17 + HiddenStates, SequenceGroupMetadata,
18 get_all_seq_ids, get_all_seq_ids_and_request_ids)
19 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
20 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
vllm/spec_decode/top1_proposer.py
+2 -2
@@ -2,8 +2,8 @@ from typing import List, Optional, Set, Tuple
2
3 import torch
4
5 -from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
6 - SequenceGroupMetadata)
5 +from vllm.model_executor.layers.sampler import SamplerOutput
6 +from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
7 from vllm.spec_decode.interfaces import (SpeculativeProposals,
8 SpeculativeProposer)
9 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
vllm/spec_decode/util.py
+2 -2
@@ -4,9 +4,9 @@ from typing import Dict, List, Optional, Sequence, Tuple
4
5 import torch
6
7 +from vllm.model_executor.layers.sampler import SamplerOutput
8 from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
8 - SamplerOutput, SequenceGroupMetadata,
9 - SequenceOutput)
9 + SequenceGroupMetadata, SequenceOutput)
10
11 SeqId = int
12
vllm/worker/cpu_model_runner.py
+2 -2
@@ -10,11 +10,11 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 SchedulerConfig)
11 from vllm.logger import init_logger
12 from vllm.model_executor import SamplingMetadata
13 +from vllm.model_executor.layers.sampler import SamplerOutput
14 from vllm.model_executor.model_loader import get_model
15 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16 MultiModalInputs)
16 -from vllm.sequence import (IntermediateTensors, SamplerOutput,
17 - SequenceGroupMetadata)
17 +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
18 from vllm.utils import make_tensor_with_pad
19 from vllm.worker.model_runner_base import (
20 ModelRunnerBase, ModelRunnerInputBase,
vllm/worker/enc_dec_model_runner.py
+2 -1
@@ -16,9 +16,10 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
16 from vllm.inputs import INPUT_REGISTRY, InputRegistry
17 from vllm.logger import init_logger
18 from vllm.model_executor import SamplingMetadata
19 +from vllm.model_executor.layers.sampler import SamplerOutput
20 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21 from vllm.sampling_params import SamplingParams
21 -from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
22 +from vllm.sequence import (IntermediateTensors, PoolerOutput,
23 SequenceGroupMetadata)
24 from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
25 from vllm.worker.model_runner import (GPUModelRunnerBase,
vllm/worker/model_runner.py
+2 -2
@@ -29,6 +29,7 @@ from vllm.lora.layers import LoRAMapping
29 from vllm.lora.request import LoRARequest
30 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
31 from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
32 +from vllm.model_executor.layers.sampler import SamplerOutput
33 from vllm.model_executor.model_loader import get_model
34 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
35 from vllm.model_executor.models.interfaces import (supports_lora,
@@ -41,8 +42,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
42 from vllm.prompt_adapter.worker_manager import (
43 LRUCacheWorkerPromptAdapterManager)
44 from vllm.sampling_params import SamplingParams
44 -from vllm.sequence import (IntermediateTensors, SamplerOutput,
45 - SequenceGroupMetadata)
45 +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
46 from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
47 flatten_2d_lists, is_hip, is_pin_memory_available,
48 supports_dynamo)
vllm/worker/model_runner_base.py
+2 -2
@@ -5,9 +5,9 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
5
6 import torch
7
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 from vllm.platforms import current_platform
9 -from vllm.sequence import (IntermediateTensors, SamplerOutput,
10 - SequenceGroupMetadata)
10 +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
11
12 if TYPE_CHECKING:
13 from vllm.attention import AttentionMetadata
vllm/worker/multi_step_model_runner.py
+154 -19
@@ -1,7 +1,8 @@
1 import dataclasses
2 import functools
3 from dataclasses import dataclass, field
4 -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
4 +from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
5 + Union)
6
7 try:
8 from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -15,9 +16,12 @@ import torch
16 from vllm import _custom_ops as ops
17 from vllm.distributed import get_pp_group
18 from vllm.logger import init_logger
19 +from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
20 + SamplerOutput,
21 + SamplingMetadata, get_logprobs,
22 + get_pythonized_sample_results)
23 from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
19 - Logprob, SamplerOutput, SequenceGroupMetadata,
20 - SequenceOutput)
24 + Logprob, SequenceGroupMetadata, SequenceOutput)
25 from vllm.worker.model_runner import (GPUModelRunnerBase,
26 ModelInputForGPUWithSamplingMetadata)
27 from vllm.worker.model_runner_base import (
@@ -53,6 +57,8 @@ class ModelOutput:
57 sampler_output_ready_event: torch.cuda.Event
58 sampled_token_ids: Optional[torch.Tensor] = None
59 pythonized: bool = False
60 + # On-device tensor containing the logprobs of each token.
61 + logprobs: Optional["torch.Tensor"] = None
62
63 def pythonize(self, input_metadata: "StatefulModelInput",
64 copy_stream: torch.cuda.Stream,
@@ -78,7 +84,9 @@ class ModelOutput:
84 blocking: bool) -> bool:
85 """
86 If blocking is set, will block until the forward pass for the output is
81 - ready and pythonize the output.
87 + ready and pythonize the output. Upon completing Pythonization, erases
88 + self.logprobs (note that a non-blocking call that is performed when
89 + the sampler output is not yet ready, will not erase self.logprobs.)
90 """
91 assert self.sampled_token_ids is not None
92 if not blocking and not self.sampler_output_ready_event.query():
@@ -89,7 +97,15 @@ class ModelOutput:
97 with torch.cuda.stream(copy_stream):
98 _pythonize_sampler_output(input_metadata, self.sampler_output,
99 pinned_sampled_token_buffer,
92 - self.sampled_token_ids)
100 + self.sampled_token_ids, self.logprobs)
101 +
102 + # Erase the logprobs GPU-side tensor.
103 + # Note that although _pythonize_sampler_output() runs in its
104 + # own CUDA stream, nonetheless _pythonize_sampler_output()
105 + # cannot return until Pythonization is complete; therefore
106 + # we know that by the time the CPU reaches this point,
107 + # `self.logprobs` is no longer needed.
108 + self.logprobs = None
109 return True
110
111
@@ -350,11 +366,16 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
366 0].sampled_token_ids.cpu()
367 model_input.cached_outputs.append(
368 ModelOutput(output[0], output_ready_event,
353 - output[0].sampled_token_ids, False))
354 - # make sure we dont try to serialize any GPU tensors
369 + output[0].sampled_token_ids, False,
370 + output[0].logprobs))
371 +
372 + # These GPU tensors are not required by multi-step;
373 + # erase them to ensure they are not pythonized or
374 + # transferred to CPU
375 output[0].sampled_token_ids = None
376 output[0].sampled_token_probs = None
377 output[0].logprobs = None
378 +
379 # Pythonize the output if CPU is ahead and the previous step is
380 # ready.
381 if not frozen_model_input.use_async_and_multi_step:
@@ -464,12 +485,75 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
485 return self._base_model_runner.vocab_size
486
487
467 -def _pythonize_sampler_output(model_input: StatefulModelInput,
468 - output: SamplerOutput,
469 - pinned_sampled_token_buffer: torch.Tensor,
470 - sampled_token_ids: torch.Tensor) -> None:
488 +DeferredLogprobsReturnType = Tuple[Optional[List[Optional[PromptLogprobs]]],
489 + Optional[List[SampleLogprobs]]]
490 +
491 +
492 +def deferred_pythonize_logprobs(
493 + output: SamplerOutput,
494 + sampling_metadata: SamplingMetadata,
495 + logprobs_tensor: Optional[torch.Tensor],
496 +) -> DeferredLogprobsReturnType:
497 + """Perform deferred logprob Pythonization.
498 +
499 + 1. Pythonize GPU-side sampler result tensors into CPU-side sampler result.
500 + 2. Pythonize GPU-side logprobs tensor into CPU-side logprobs lists,
501 + utilizing the Pythonized sampler result computed in step 1.
502 +
503 + These deferred computations are not required for single-step scheduling
504 + or the `profile_run()` phase of multi-step scheduling.
505 +
506 + Args:
507 + output: sampler output (under deferred Pythonization)
508 + sampling_metadata
509 +
510 + Returns:
511 + prompt_logprobs (CPU), sample_logprobs (CPU)
512 + """
513 +
514 + # - Deferred pythonization of sample result
515 + sampler_result = get_pythonized_sample_results(
516 + output.deferred_sample_results_args)
517 +
518 + # - Erase the GPU-side deferred sample_result
519 + # computation args to ensure it is never
520 + # pythonized or transferred to CPU
521 + output.deferred_sample_results_args = None
522 +
523 + # - Deferred pythonization of logprobs
524 + (
525 + prompt_logprobs,
526 + sample_logprobs,
527 + ) = get_logprobs(logprobs_tensor, sampling_metadata, sampler_result)
528 + assert len(prompt_logprobs) == len(sampling_metadata.seq_groups)
529 + assert len(sample_logprobs) == len(sampling_metadata.seq_groups)
530 +
531 + return prompt_logprobs, sample_logprobs
532 +
533 +
534 +def _pythonize_sampler_output(
535 + model_input: StatefulModelInput,
536 + output: SamplerOutput,
537 + pinned_sampled_token_buffer: torch.Tensor,
538 + sampled_token_ids: torch.Tensor,
539 + logprobs_tensor: Optional[torch.Tensor],
540 +) -> None:
541 """ This function is only called when the output tensors are ready.
472 - See ModelOutput
542 + See :class:`ModelOutput`.
543 +
544 + Modifies `output.outputs` and `pinned_sampled_token_buffer` in-place,
545 + adding a Pythonized output data structure
546 + (:class:`CompletionSequenceGroupOutput`) for each :class:`SequenceGroup`.
547 +
548 + Args:
549 + model_input
550 + output: sampler output
551 + pinned_sampled_token_token_buffer: CPU-side pinned memory
552 + (receives copy of
553 + GPU-side token buffer.)
554 + sampled_token_ids: GPU-side token buffer
555 + logprobs_tensor: GPU-side tensor containing
556 + logprobs computed during sampling
557 """
558
559 assert model_input.frozen_model_input is not None
@@ -489,8 +573,51 @@ def _pythonize_sampler_output(model_input: StatefulModelInput,
573
574 sampling_metadata = frozen_model_input.sampling_metadata
575
492 - for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
493 - samples_list):
576 + skip_sampler_cpu_output = (
577 + frozen_model_input.sampling_metadata.skip_sampler_cpu_output)
578 +
579 + # We are guaranteed output tensors are ready, so it is safe to
580 + # pythonize the sampler output & obtain CPU-side logprobs.
581 + #
582 + # However this computation may be skipped entirely
583 + # if no pythonization was deferred.
584 + seq_groups = sampling_metadata.seq_groups
585 + logprobs_are_requested = any([
586 + sg.sampling_params.logprobs is not None
587 + or sg.sampling_params.prompt_logprobs is not None for sg in seq_groups
588 + ])
589 + do_pythonize_logprobs = (skip_sampler_cpu_output
590 + and logprobs_are_requested)
591 + (
592 + prompt_logprobs,
593 + sample_logprobs,
594 + ) = (deferred_pythonize_logprobs(output, sampling_metadata,
595 + logprobs_tensor)
596 + if do_pythonize_logprobs else (None, None))
597 +
598 + for sgdx, (seq_group,
599 + sample_result) in enumerate(zip(seq_groups, samples_list)):
600 +
601 + if do_pythonize_logprobs:
602 + assert prompt_logprobs is not None
603 + assert sample_logprobs is not None
604 +
605 + (
606 + group_prompt_logprobs,
607 + group_sample_logprobs,
608 + ) = ( # Utilize deferred pythonization results
609 + prompt_logprobs[sgdx],
610 + sample_logprobs[sgdx],
611 + )
612 + elif logprobs_are_requested:
613 + (
614 + group_prompt_logprobs,
615 + group_sample_logprobs,
616 + ) = (
617 + # profile_run: use already-computed logprobs
618 + output.outputs[sgdx].prompt_logprobs,
619 + [sample.logprobs for sample in output.outputs[sgdx].samples])
620 +
621 seq_ids = seq_group.seq_ids
622 next_token_ids = sample_result
623 parent_ids = [0]
@@ -498,11 +625,19 @@ def _pythonize_sampler_output(model_input: StatefulModelInput,
625 if seq_group.sampling_params.logits_processors:
626 assert len(seq_group.sampling_params.logits_processors) == 0, (
627 "Logits Processors are not supported in multi-step decoding")
501 - for parent_id, next_token_id in zip(parent_ids, next_token_ids):
502 - # TODO(will): support logprobs
503 - # Hard coded logprob
628 + for tdx, (parent_id,
629 + next_token_id) in enumerate(zip(parent_ids, next_token_ids)):
630 seq_outputs.append(
631 SequenceOutput(seq_ids[parent_id], next_token_id,
506 - {next_token_id: Logprob(logprob=-1)}))
507 - output.outputs.append(CompletionSequenceGroupOutput(seq_outputs, None))
632 + (group_sample_logprobs[tdx]
633 + if logprobs_are_requested else {
634 + next_token_id:
635 + Logprob(logprob=float('inf'),
636 + rank=None,
637 + decoded_token=None)
638 + })))
639 + output.outputs.append(
640 + CompletionSequenceGroupOutput(
641 + seq_outputs,
642 + (group_prompt_logprobs if logprobs_are_requested else None)))
643 assert len(output.outputs) > 0
vllm/worker/multi_step_worker.py
+2 -1
@@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple
5 import torch
6
7 from vllm.distributed import broadcast_tensor_dict, get_pp_group
8 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
8 +from vllm.model_executor.layers.sampler import SamplerOutput
9 +from vllm.sequence import ExecuteModelRequest
10 from vllm.worker.model_runner_base import BroadcastableModelInput
11 from vllm.worker.multi_step_model_runner import (MultiStepModelRunner,
12 StatefulModelInput)
vllm/worker/neuron_model_runner.py
+2 -2
@@ -8,11 +8,11 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8 SchedulerConfig)
9 from vllm.logger import init_logger
10 from vllm.model_executor import SamplingMetadata
11 +from vllm.model_executor.layers.sampler import SamplerOutput
12 from vllm.model_executor.model_loader.neuron import get_neuron_model
13 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
14 MultiModalInputs)
14 -from vllm.sequence import (IntermediateTensors, SamplerOutput,
15 - SequenceGroupMetadata)
15 +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
16 from vllm.utils import is_pin_memory_available, make_tensor_with_pad
17 from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
18
vllm/worker/openvino_model_runner.py
+2 -1
@@ -11,10 +11,11 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11 SchedulerConfig)
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 +from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.model_executor.model_loader.openvino import get_model
16 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17 MultiModalInputs)
17 -from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18 +from vllm.sequence import SequenceGroupMetadata
19
20 logger = init_logger(__name__)
21
vllm/worker/openvino_worker.py
+2 -1
@@ -14,7 +14,8 @@ from vllm.distributed import (broadcast_tensor_dict,
14 init_distributed_environment)
15 from vllm.logger import init_logger
16 from vllm.model_executor import set_random_seed
17 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
17 +from vllm.model_executor.layers.sampler import SamplerOutput
18 +from vllm.sequence import ExecuteModelRequest
19 from vllm.worker.openvino_model_runner import OpenVINOModelRunner
20 from vllm.worker.worker_base import LoraNotSupportedWorkerBase
21
vllm/worker/tpu_model_runner.py
+2 -2
@@ -14,11 +14,11 @@ from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
14 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
15 ParallelConfig, SchedulerConfig)
16 from vllm.logger import init_logger
17 +from vllm.model_executor.layers.sampler import SamplerOutput
18 from vllm.model_executor.model_loader import get_model
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
20 - Logprob, SamplerOutput, SequenceGroupMetadata,
21 - SequenceOutput)
21 + Logprob, SequenceGroupMetadata, SequenceOutput)
22 from vllm.worker.model_runner_base import (
23 ModelRunnerBase, ModelRunnerInputBase,
24 _add_attn_metadata_broadcastable_dict,
vllm/worker/worker.py
+2 -2
@@ -17,12 +17,12 @@ from vllm.distributed import (ensure_model_parallel_initialized,
17 from vllm.logger import init_logger
18 from vllm.lora.request import LoRARequest
19 from vllm.model_executor import set_random_seed
20 +from vllm.model_executor.layers.sampler import SamplerOutput
21 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
22 from vllm.platforms import current_platform
23 from vllm.prompt_adapter.request import PromptAdapterRequest
24 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
24 - SamplerOutput, SequenceGroupMetadata,
25 - SequenceGroupMetadataDelta)
25 + SequenceGroupMetadata, SequenceGroupMetadataDelta)
26 from vllm.worker.cache_engine import CacheEngine
27 from vllm.worker.embedding_model_runner import EmbeddingModelRunner
28 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
vllm/worker/worker_base.py
+2 -2
@@ -11,9 +11,9 @@ from vllm.config import ObservabilityConfig
11 from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
12 from vllm.logger import init_logger
13 from vllm.lora.request import LoRARequest
14 +from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.platforms import current_platform
15 -from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
16 - SamplerOutput)
16 +from vllm.sequence import ExecuteModelRequest, IntermediateTensors
17 from vllm.utils import (enable_trace_function_call_for_thread,
18 update_environment_variables)
19 from vllm.worker.model_runner_base import (BroadcastableModelInput,
vllm/worker/xpu_model_runner.py
+2 -2
@@ -15,12 +15,12 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15 from vllm.distributed import get_pp_group
16 from vllm.inputs import INPUT_REGISTRY, InputRegistry
17 from vllm.logger import init_logger
18 +from vllm.model_executor.layers.sampler import SamplerOutput
19 from vllm.model_executor.model_loader import get_model
20 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
21 MultiModalInputs, MultiModalRegistry)
22 from vllm.sampling_params import SamplingParams
22 -from vllm.sequence import (IntermediateTensors, SamplerOutput,
23 - SequenceGroupMetadata)
23 +from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
24 from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
25 from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
26 from vllm.worker.model_runner_base import (