A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Logprobs support in Multi-step (#7652)
afeldman-nm committed
Aug 29, 2024 at 22:19 UTC
428dd1445ee3750099967084725849c4920721a5
103 files changed
+874
-378
tests/models/utils.py
+26
-17
@@ -1,7 +1,7 @@
1
import warnings
2
from typing import Dict, List, Optional, Sequence, Tuple, Union
3
4
-from vllm.sequence import SampleLogprobs
4
+from vllm.sequence import Logprob, SampleLogprobs
5
6
TokensText = Tuple[List[int], str]
7
@@ -38,34 +38,39 @@ TokensTextLogprobs = Tuple[List[int], str, Optional[Union[List[Dict[int,
38
float]],
39
SampleLogprobs]]]
40
41
+# Allow for tokens to be represented as str's rather than IDs
42
+TextTextLogprobs = Tuple[List[str], str, Optional[Union[List[Dict[str, float]],
43
+ List[Dict[str,
44
+ Logprob]]]]]
45
+
46
47
def check_logprobs_close(
48
*,
44
- outputs_0_lst: Sequence[TokensTextLogprobs],
45
- outputs_1_lst: Sequence[TokensTextLogprobs],
49
+ outputs_0_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
50
+ outputs_1_lst: Sequence[Union[TokensTextLogprobs, TextTextLogprobs]],
51
name_0: str,
52
name_1: str,
53
num_outputs_0_skip_tokens: int = 0,
54
warn_on_mismatch: bool = True,
50
-):
51
- """
52
- Compare the logprobs of two sequences generated by different models,
55
+ always_check_logprobs: bool = False,
56
+) -> None:
57
+ """Compare the logprobs of two sequences generated by different models,
58
which should be similar but not necessarily equal.
59
55
- Arguments:
56
-
57
- * outputs_0_lst: First sequence to compare
58
- * outputs_0_lst: Second sequence to compare
59
- * name_0: sequence #0 name
60
- * name_1: sequence #1 name
61
- * num_outputs_0_skip_tokens: If > 0, specifies the number of initial
60
+ Args:
61
+ outputs_0_lst: First sequence to compare
62
+ outputs_0_lst: Second sequence to compare
63
+ name_0: sequence #0 name
64
+ name_1: sequence #1 name
65
+ num_outputs_0_skip_tokens: If > 0, specifies the number of initial
66
sequence #0 tokens & logprobs to discard
67
before comparison, i.e. all
68
of sequence #1 will be compared to
69
sequence #0 beginning at index
70
num_outputs_0_skip_tokens
67
- * warn_on_mismatch: Issue a warning if there is token-wise or text-wise
71
+ warn_on_mismatch: Issue a warning if there is token-wise or text-wise
72
mismatch between the two sequences
73
+ always_check_logprobs: If true, check logprobs even when tokens match
74
"""
75
assert len(outputs_0_lst) == len(outputs_1_lst)
76
@@ -94,8 +99,12 @@ def check_logprobs_close(
99
for idx, (output_id_0,
100
output_id_1) in enumerate(zip(output_ids_0, output_ids_1)):
101
97
- # If generated tokens don't match, then
98
- if output_id_0 != output_id_1:
102
+ is_tok_mismatch = output_id_0 != output_id_1
103
+
104
+ # If generated tokens don't match
105
+ # or it is desired to always check logprobs,
106
+ # then
107
+ if is_tok_mismatch or always_check_logprobs:
108
logprobs_elem_0 = logprobs_0[idx]
109
logprobs_elem_1 = logprobs_1[idx]
110
@@ -111,7 +120,7 @@ def check_logprobs_close(
120
assert output_id_0 in logprobs_elem_1, fail_msg
121
assert output_id_1 in logprobs_elem_0, fail_msg
122
114
- if warn_on_mismatch:
123
+ if warn_on_mismatch and is_tok_mismatch:
124
with warnings.catch_warnings():
125
# This ensures that repeated warnings are shown
126
# in the output, not just the first occurrence
tests/multi_step/test_correctness_async_llm.py
+69
-30
@@ -1,10 +1,12 @@
1
# Test the AsyncLLMEngine with multi-step-decoding
2
3
-from typing import List
3
+from typing import List, Optional
4
5
import pytest
6
7
-from ..utils import RemoteOpenAIServer
7
+from ..models.utils import check_logprobs_close
8
+from ..utils import (completions_with_server_args, get_client_text_generations,
9
+ get_client_text_logprob_generations)
10
11
MODELS = [
12
"JackFram/llama-160m",
@@ -23,22 +25,6 @@ DEFAULT_SERVER_ARGS: List[str] = [
25
]
26
27
26
-async def completions_with_server_args(prompts: List[str], model_name: str,
27
- server_cli_args: List[str]):
28
-
29
- outputs = None
30
- with RemoteOpenAIServer(model_name, server_cli_args) as server:
31
- async with server.get_async_client() as client:
32
- outputs = await client.completions.create(model=model_name,
33
- prompt=prompts,
34
- temperature=0,
35
- stream=False,
36
- max_tokens=5)
37
- assert outputs is not None
38
-
39
- return outputs
40
-
41
-
28
@pytest.mark.parametrize("model", MODELS)
29
@pytest.mark.parametrize(("tp_size, pp_size"), [
30
(1, 1),
@@ -47,12 +33,43 @@ async def completions_with_server_args(prompts: List[str], model_name: str,
33
@pytest.mark.parametrize("eager_mode", [False, True])
34
@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
35
@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
36
+@pytest.mark.parametrize("num_logprobs", [None, 5])
37
@pytest.mark.parametrize("is_async", [False, True])
38
@pytest.mark.asyncio
52
-async def test_multi_step(example_prompts, model: str, tp_size: int,
53
- pp_size: int, eager_mode: int,
54
- num_scheduler_steps: int, num_prompts: int,
55
- is_async: bool):
39
+async def test_multi_step(
40
+ example_prompts,
41
+ model: str,
42
+ tp_size: int,
43
+ pp_size: int,
44
+ eager_mode: int,
45
+ num_scheduler_steps: int,
46
+ num_prompts: int,
47
+ is_async: bool,
48
+ num_logprobs: Optional[int],
49
+) -> None:
50
+ """Test vLLM engine with multi-step scheduling in an OpenAI-protocol
51
+ client/server environment.
52
+
53
+ Set up an engine with single-step scheduling as a ground-truth reference.
54
+
55
+ Send a completions API request to both engines with the same prompts.
56
+
57
+ Validate:
58
+ * Generated tokens match
59
+ * Generated logprobs are all very close
60
+
61
+ Args:
62
+ example_prompts: test fixture providing example prompts
63
+ model: model under test (same for single- and multi-step engines)
64
+ tp_size: degree of tensor-parallelism
65
+ pp_size: degree of pipeline-parallelism
66
+ eager_mode
67
+ num_scheduler_steps: for multi-step scheduling, GPU-side steps per
68
+ GPU -> CPU output transfer
69
+ num_prompts: number of example prompts under test
70
+ num_logprobs: corresponds to the `logprobs` argument to the OpenAI
71
+ completions endpoint; `None` -> no logprobs
72
+ """
73
74
prompts = example_prompts
75
if len(prompts) < num_prompts:
@@ -77,14 +94,36 @@ async def test_multi_step(example_prompts, model: str, tp_size: int,
94
str(pp_size),
95
]
96
97
+ # Spin up client/server & issue completion API requests.
98
+ # Default `max_wait_seconds` is 240 but was empirically
99
+ # was raised 3x to 720 *just for this test* due to
100
+ # observed timeouts in GHA CI
101
ref_completions = await completions_with_server_args(
81
- prompts, model, server_args + distributed_args)
102
+ prompts,
103
+ model,
104
+ server_args + distributed_args,
105
+ num_logprobs,
106
+ max_wait_seconds=3 * 240)
107
test_completions = await completions_with_server_args(
83
- prompts, model, ms_server_args + distributed_args)
84
-
85
- def get_text_generations(completions):
86
- return [x.text for x in completions.choices]
87
-
88
- ref_generations = get_text_generations(ref_completions)
89
- test_generations = get_text_generations(test_completions)
108
+ prompts,
109
+ model,
110
+ ms_server_args + distributed_args,
111
+ num_logprobs,
112
+ max_wait_seconds=3 * 240)
113
+
114
+ # Assert multi-step scheduling produces identical tokens
115
+ # to single-step scheduling.
116
+ ref_generations = get_client_text_generations(ref_completions)
117
+ test_generations = get_client_text_generations(test_completions)
118
assert ref_generations == test_generations
119
+
120
+ # Assert multi-step scheduling produces nearly-identical logprobs
121
+ # to single-step scheduling.
122
+ ref_text_logprobs = get_client_text_logprob_generations(ref_completions)
123
+ test_text_logprobs = get_client_text_logprob_generations(test_completions)
124
+ check_logprobs_close(
125
+ outputs_0_lst=ref_text_logprobs,
126
+ outputs_1_lst=test_text_logprobs,
127
+ name_0="hf",
128
+ name_1="vllm",
129
+ )
tests/multi_step/test_correctness_llm.py
+74
-21
@@ -1,8 +1,10 @@
1
# Test the LLMEngine with multi-step-decoding
2
3
+from typing import Optional
4
+
5
import pytest
6
5
-from ..models.utils import check_outputs_equal
7
+from ..models.utils import check_logprobs_close, check_outputs_equal
8
9
MODELS = [
10
"JackFram/llama-160m",
@@ -18,10 +20,45 @@ NUM_PROMPTS = [10]
20
@pytest.mark.parametrize("enforce_eager", [True])
21
@pytest.mark.parametrize("num_scheduler_steps", NUM_SCHEDULER_STEPS)
22
@pytest.mark.parametrize("num_prompts", NUM_PROMPTS)
21
-def test_multi_step_llm(hf_runner, vllm_runner, example_prompts, model: str,
22
- dtype: str, tp_size: int, max_tokens: int,
23
- enforce_eager: int, num_scheduler_steps: int,
24
- num_prompts: int) -> None:
23
+@pytest.mark.parametrize("num_logprobs", [None, 5])
24
+def test_multi_step_llm(
25
+ hf_runner,
26
+ vllm_runner,
27
+ example_prompts,
28
+ model: str,
29
+ dtype: str,
30
+ tp_size: int,
31
+ max_tokens: int,
32
+ enforce_eager: int,
33
+ num_scheduler_steps: int,
34
+ num_prompts: int,
35
+ num_logprobs: Optional[int],
36
+) -> None:
37
+ """Test vLLM engine with multi-step scheduling via sync LLM Engine.
38
+
39
+ Set up a HuggingFace (HF) transformers model as a ground-truth reference.
40
+
41
+ Prompt them with the same example prompts.
42
+
43
+ Validate:
44
+ * Generated tokens match
45
+ * Generated logprobs are all very close
46
+
47
+ Args:
48
+ hf_runner: HF transformers model runner fixture
49
+ vllm_runner: vLLM model runner fixture
50
+ example_prompts: test fixture providing example prompts
51
+ model: model under test (same for single- and multi-step engines)
52
+ dtype: tensor datatype for engine to utilize
53
+ tp_size: degree of tensor-parallelism
54
+ max_tokens: the maximum number of tokens to generate
55
+ enforce_eager
56
+ num_scheduler_steps: for multi-step scheduling, GPU-side steps per
57
+ GPU -> CPU output transfer
58
+ num_prompts: number of example prompts under test
59
+ num_logprobs: corresponds to the `logprobs` argument to the OpenAI
60
+ completions endpoint; `None` -> no logprobs
61
+ """
62
63
prompts = example_prompts
64
if len(prompts) < num_prompts:
@@ -29,21 +66,37 @@ def test_multi_step_llm(hf_runner, vllm_runner, example_prompts, model: str,
66
prompts = prompts[:num_prompts]
67
assert len(prompts) == num_prompts
68
32
- with vllm_runner(model,
33
- dtype=dtype,
34
- enforce_eager=enforce_eager,
35
- gpu_memory_utilization=0.7,
36
- tensor_parallel_size=tp_size,
37
- use_v2_block_manager=True,
38
- num_scheduler_steps=num_scheduler_steps) as vllm_model:
39
- vllm_outputs = vllm_model.generate_greedy(prompts, max_tokens)
69
+ with vllm_runner(
70
+ model,
71
+ dtype=dtype,
72
+ enforce_eager=enforce_eager,
73
+ gpu_memory_utilization=0.7,
74
+ tensor_parallel_size=tp_size,
75
+ use_v2_block_manager=True,
76
+ num_scheduler_steps=num_scheduler_steps,
77
+ ) as vllm_model:
78
+ vllm_outputs = (vllm_model.generate_greedy(prompts, max_tokens)
79
+ if num_logprobs is None else
80
+ vllm_model.generate_greedy_logprobs(
81
+ prompts, max_tokens, num_logprobs))
82
83
with hf_runner(model, dtype=dtype) as hf_model:
42
- hf_outputs = hf_model.generate_greedy(prompts, max_tokens)
43
-
44
- check_outputs_equal(
45
- outputs_0_lst=hf_outputs,
46
- outputs_1_lst=vllm_outputs,
47
- name_0="hf",
48
- name_1="vllm",
49
- )
84
+ hf_outputs = (hf_model.generate_greedy(prompts, max_tokens)
85
+ if num_logprobs is None else
86
+ hf_model.generate_greedy_logprobs_limit(
87
+ prompts, max_tokens, num_logprobs))
88
+
89
+ if num_logprobs is None:
90
+ check_outputs_equal(
91
+ outputs_0_lst=hf_outputs,
92
+ outputs_1_lst=vllm_outputs,
93
+ name_0="hf",
94
+ name_1="vllm",
95
+ )
96
+ else:
97
+ check_logprobs_close(
98
+ outputs_0_lst=hf_outputs,
99
+ outputs_1_lst=vllm_outputs,
100
+ name_0="hf",
101
+ name_1="vllm",
102
+ )
tests/spec_decode/test_multi_step_worker.py
+2
-1
@@ -5,9 +5,10 @@ from unittest.mock import MagicMock
5
import pytest
6
import torch
7
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.model_executor.utils import set_random_seed
10
from vllm.sequence import (ExecuteModelRequest, HiddenStates, Logprob,
10
- SamplerOutput, get_all_seq_ids)
11
+ get_all_seq_ids)
12
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
13
from vllm.spec_decode.multi_step_worker import MultiStepWorker
14
from vllm.spec_decode.top1_proposer import Top1Proposer
tests/spec_decode/test_spec_decode_worker.py
+2
-1
@@ -7,8 +7,9 @@ from unittest.mock import MagicMock
7
import pytest
8
import torch
9
10
+from vllm.model_executor.layers.sampler import SamplerOutput
11
from vllm.model_executor.utils import set_random_seed
11
-from vllm.sequence import ExecuteModelRequest, SamplerOutput, SequenceOutput
12
+from vllm.sequence import ExecuteModelRequest, SequenceOutput
13
from vllm.spec_decode.interfaces import SpeculativeProposals
14
from vllm.spec_decode.metrics import (AsyncMetricsCollector,
15
SpecDecodeWorkerMetrics)
tests/spec_decode/utils.py
+2
-2
@@ -8,12 +8,12 @@ from unittest.mock import MagicMock
8
import torch
9
10
from vllm.engine.arg_utils import EngineArgs
11
+from vllm.model_executor.layers.sampler import SamplerOutput
12
from vllm.model_executor.utils import set_random_seed
13
from vllm.sampling_params import SamplingParams
14
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
15
CompletionSequenceGroupOutput, Logprob,
15
- SamplerOutput, SequenceData, SequenceGroupMetadata,
16
- SequenceOutput)
16
+ SequenceData, SequenceGroupMetadata, SequenceOutput)
17
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
18
from vllm.worker.cache_engine import CacheEngine
19
from vllm.worker.model_runner import ModelRunner
tests/test_sequence.py
+3
-2
@@ -2,9 +2,10 @@ from array import array
2
3
import pytest
4
5
+from vllm.model_executor.layers.sampler import SamplerOutput
6
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE,
6
- CompletionSequenceGroupOutput, SamplerOutput,
7
- SequenceData, SequenceOutput)
7
+ CompletionSequenceGroupOutput, SequenceData,
8
+ SequenceOutput)
9
10
from .core.utils import create_dummy_prompt
11
tests/utils.py
+60
@@ -11,9 +11,11 @@ from typing import Any, Callable, Dict, List, Optional
11
12
import openai
13
import requests
14
+from openai.types.completion import Completion
15
from transformers import AutoTokenizer
16
from typing_extensions import ParamSpec
17
18
+from tests.models.utils import TextTextLogprobs
19
from vllm.distributed import (ensure_model_parallel_initialized,
20
init_distributed_environment)
21
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -432,3 +434,61 @@ def fork_new_process_for_each_test(
434
f" args {args} and kwargs {kwargs}")
435
436
return wrapper
437
+
438
+
439
+async def completions_with_server_args(
440
+ prompts: List[str],
441
+ model_name: str,
442
+ server_cli_args: List[str],
443
+ num_logprobs: Optional[int],
444
+ max_wait_seconds: int = 240,
445
+) -> Completion:
446
+ '''Construct a remote OpenAI server, obtain an async client to the
447
+ server & invoke the completions API to obtain completions.
448
+
449
+ Args:
450
+ prompts: test prompts
451
+ model_name: model to spin up on the vLLM server
452
+ server_cli_args: CLI args for starting the server
453
+ num_logprobs: Number of logprobs to report (or `None`)
454
+ max_wait_seconds: timeout interval for bringing up server.
455
+ Default: 240sec
456
+
457
+ Returns:
458
+ OpenAI Completion instance
459
+ '''
460
+
461
+ outputs = None
462
+ with RemoteOpenAIServer(model_name,
463
+ server_cli_args,
464
+ max_wait_seconds=max_wait_seconds) as server:
465
+ client = server.get_async_client()
466
+ outputs = await client.completions.create(model=model_name,
467
+ prompt=prompts,
468
+ temperature=0,
469
+ stream=False,
470
+ max_tokens=5,
471
+ logprobs=num_logprobs)
472
+ assert outputs is not None
473
+
474
+ return outputs
475
+
476
+
477
+def get_client_text_generations(completions: Completion) -> List[str]:
478
+ '''Extract generated tokens from the output of a
479
+ request made to an Open-AI-protocol completions endpoint.
480
+ '''
481
+ return [x.text for x in completions.choices]
482
+
483
+
484
+def get_client_text_logprob_generations(
485
+ completions: Completion) -> List[TextTextLogprobs]:
486
+ '''Operates on the output of a request made to an Open-AI-protocol
487
+ completions endpoint; obtains top-rank logprobs for each token in
488
+ each :class:`SequenceGroup`
489
+ '''
490
+ text_generations = get_client_text_generations(completions)
491
+ text = ''.join(text_generations)
492
+ return [(text_generations, text,
493
+ (None if x.logprobs is None else x.logprobs.top_logprobs))
494
+ for x in completions.choices]
vllm/engine/async_llm_engine.py
+2
-1
@@ -22,11 +22,12 @@ from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
22
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
23
from vllm.logger import init_logger
24
from vllm.lora.request import LoRARequest
25
+from vllm.model_executor.layers.sampler import SamplerOutput
26
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
27
from vllm.pooling_params import PoolingParams
28
from vllm.prompt_adapter.request import PromptAdapterRequest
29
from vllm.sampling_params import SamplingParams
29
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
30
+from vllm.sequence import ExecuteModelRequest
31
from vllm.transformers_utils.tokenizer import AnyTokenizer
32
from vllm.usage.usage_lib import UsageContext
33
from vllm.utils import print_warning_once
vllm/engine/llm_engine.py
+3
-2
@@ -33,6 +33,7 @@ from vllm.inputs import (INPUT_REGISTRY, EncoderDecoderLLMInputs,
33
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
34
from vllm.logger import init_logger
35
from vllm.lora.request import LoRARequest
36
+from vllm.model_executor.layers.sampler import SamplerOutput
37
from vllm.multimodal import MultiModalDataDict
38
from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
39
RequestOutputFactory)
@@ -40,8 +41,8 @@ from vllm.pooling_params import PoolingParams
41
from vllm.prompt_adapter.request import PromptAdapterRequest
42
from vllm.sampling_params import SamplingParams
43
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
43
- SamplerOutput, Sequence, SequenceGroup,
44
- SequenceGroupMetadata, SequenceStatus)
44
+ Sequence, SequenceGroup, SequenceGroupMetadata,
45
+ SequenceStatus)
46
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
47
init_tracer)
48
from vllm.transformers_utils.config import try_get_generation_config
vllm/engine/output_processor/multi_step.py
+12
-3
@@ -4,6 +4,8 @@ from typing import Callable, List
4
from vllm.core.scheduler import Scheduler
5
from vllm.engine.output_processor.interfaces import (
6
SequenceGroupOutputProcessor)
7
+from vllm.engine.output_processor.single_step import (
8
+ single_step_process_prompt_logprob)
9
from vllm.engine.output_processor.stop_checker import StopChecker
10
from vllm.logger import init_logger
11
from vllm.sampling_params import SamplingParams
@@ -46,9 +48,16 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
48
49
def process_prompt_logprob(self, seq_group: SequenceGroup,
50
outputs: List[SequenceGroupOutput]) -> None:
49
- # TODO(sang): Prompt logprob currently not implemented in multi step
50
- # workers.
51
- self._log_prompt_logprob_unsupported_warning_once()
51
+ """Process prompt logprobs associated with each step of a multi-step-
52
+ scheduled computation.
53
+
54
+ Args:
55
+ seq_group: the outputs are associated with this :class:`SequenceGroup`
56
+ outputs: the :class:`SequenceGroupOutput`s for all scheduler steps
57
+ """
58
+ for output in outputs:
59
+ # Concatenate single-step prompt logprob processing results.
60
+ single_step_process_prompt_logprob(self, seq_group, output)
61
62
@staticmethod
63
@functools.lru_cache()
vllm/engine/output_processor/single_step.py
+46
-19
@@ -15,6 +15,44 @@ from vllm.utils import Counter
15
logger = init_logger(__name__)
16
17
18
+def single_step_process_prompt_logprob(
19
+ sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
20
+ output: SequenceGroupOutput) -> None:
21
+ """Process prompt logprobs associated with the :class:`SequenceGroupOutput`
22
+ for a given step.
23
+
24
+ Do nothing if the output has no prompt logprobs.
25
+
26
+ Account for the fact that transformers do not compute first-token logprobs.
27
+
28
+ Args:
29
+ sg_output_proc: :class:`SequenceGroupOutputProcessor` instance
30
+ seq_group: the output is associated with this :class:`SequenceGroup`
31
+ output: the :class:`SequenceGroupOutput` for a single scheduler step
32
+ """
33
+ prompt_logprobs = output.prompt_logprobs
34
+
35
+ # If this is the first (or only) "chunk" of the prefill, we need
36
+ # to prepend None to the list of prompt logprobs. The reason for this
37
+ # is that for N prompt tokens, the Sampler will generate N-1 total
38
+ # prompt logprobs during prefill since the token at idx 0 will not
39
+ # have a logprob associated with it.
40
+ if prompt_logprobs is not None:
41
+ if not seq_group.prompt_logprobs:
42
+ prompt_logprobs = [None] + prompt_logprobs
43
+ seq_group.prompt_logprobs = []
44
+
45
+ assert hasattr(sg_output_proc, 'detokenizer')
46
+ if (seq_group.sampling_params.detokenize
47
+ and sg_output_proc.detokenizer):
48
+ sg_output_proc.detokenizer.decode_prompt_logprobs_inplace(
49
+ seq_group,
50
+ prompt_logprobs,
51
+ position_offset=len(seq_group.prompt_logprobs))
52
+
53
+ seq_group.prompt_logprobs.extend(prompt_logprobs)
54
+
55
+
56
class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
57
"""SequenceGroupOutputProcessor which handles "output processing" logic,
58
which happens after the model returns generated token ids and before
@@ -60,27 +98,16 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
98
99
def process_prompt_logprob(self, seq_group: SequenceGroup,
100
outputs: List[SequenceGroupOutput]) -> None:
101
+ """Process prompt logprobs associated with one step of a single-step-
102
+ scheduled computation.
103
+
104
+ Args:
105
+ seq_group: the output is associated with this :class:`SequenceGroup`
106
+ output: the :class:`SequenceGroupOutput` for a single scheduler step
107
+ """
108
assert len(outputs) == 1, ("Single step should only has 1 output.")
109
output = outputs[0]
65
- prompt_logprobs = output.prompt_logprobs
66
-
67
- # If this is the first (or only) "chunk" of the prefill, we need
68
- # to prepend None to the list of prompt logprobs. The reason for this
69
- # is that for N prompt tokens, the Sampler will generate N-1 total
70
- # prompt logprobs during prefill since the token at idx 0 will not
71
- # have a logprob associated with it.
72
- if prompt_logprobs is not None:
73
- if not seq_group.prompt_logprobs:
74
- prompt_logprobs = [None] + prompt_logprobs
75
- seq_group.prompt_logprobs = []
76
-
77
- if seq_group.sampling_params.detokenize and self.detokenizer:
78
- self.detokenizer.decode_prompt_logprobs_inplace(
79
- seq_group,
80
- prompt_logprobs,
81
- position_offset=len(seq_group.prompt_logprobs))
82
-
83
- seq_group.prompt_logprobs.extend(prompt_logprobs)
110
+ single_step_process_prompt_logprob(self, seq_group, output)
111
112
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
113
outputs: SequenceGroupOutput,
vllm/engine/output_processor/util.py
+2
-1
@@ -2,7 +2,8 @@ from typing import List
2
from typing import Sequence as GenericSequence
3
from typing import Union
4
5
-from vllm.sequence import PoolerOutput, SamplerOutput, SequenceGroupOutput
5
+from vllm.model_executor.layers.sampler import SamplerOutput
6
+from vllm.sequence import PoolerOutput, SequenceGroupOutput
7
8
9
def create_output_by_sequence_group(
vllm/engine/protocol.py
+1
-1
@@ -5,11 +5,11 @@ from vllm.config import DecodingConfig, ModelConfig
5
from vllm.core.scheduler import SchedulerOutputs
6
from vllm.inputs.data import PromptInputs
7
from vllm.lora.request import LoRARequest
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
10
from vllm.pooling_params import PoolingParams
11
from vllm.prompt_adapter.request import PromptAdapterRequest
12
from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import SamplerOutput
13
from vllm.transformers_utils.tokenizer import AnyTokenizer
14
15
vllm/executor/cpu_executor.py
+2
-1
@@ -11,8 +11,9 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
11
ResultHandler, WorkerMonitor)
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
+from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.prompt_adapter.request import PromptAdapterRequest
15
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
16
+from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import (GiB_bytes, get_distributed_init_method, get_open_port,
18
get_vllm_instance_id, make_async)
19
from vllm.worker.worker_base import WorkerWrapperBase
vllm/executor/distributed_gpu_executor.py
+2
-1
@@ -6,7 +6,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase
6
from vllm.executor.gpu_executor import GPUExecutor
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
9
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
+from vllm.model_executor.layers.sampler import SamplerOutput
10
+from vllm.sequence import ExecuteModelRequest
11
12
logger = init_logger(__name__)
13
vllm/executor/executor_base.py
+2
-1
@@ -6,8 +6,9 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
PromptAdapterConfig, SchedulerConfig,
7
SpeculativeConfig)
8
from vllm.lora.request import LoRARequest
9
+from vllm.model_executor.layers.sampler import SamplerOutput
10
from vllm.prompt_adapter.request import PromptAdapterRequest
10
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
11
+from vllm.sequence import ExecuteModelRequest
12
13
14
class ExecutorBase(ABC):
vllm/executor/gpu_executor.py
+2
-1
@@ -3,8 +3,9 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
+from vllm.model_executor.layers.sampler import SamplerOutput
7
from vllm.prompt_adapter.request import PromptAdapterRequest
7
-from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
8
+from vllm.sequence import ExecuteModelRequest, PoolerOutput
9
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10
make_async)
11
from vllm.worker.worker_base import WorkerBase, WorkerWrapperBase
vllm/executor/multiproc_gpu_executor.py
+2
-1
@@ -14,7 +14,8 @@ from vllm.executor.gpu_executor import create_worker
14
from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
15
ResultHandler, WorkerMonitor)
16
from vllm.logger import init_logger
17
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
17
+from vllm.model_executor.layers.sampler import SamplerOutput
18
+from vllm.sequence import ExecuteModelRequest
19
from vllm.triton_utils import maybe_set_triton_cache_manager
20
from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
21
get_distributed_init_method, get_open_port,
vllm/executor/neuron_executor.py
+2
-1
@@ -3,7 +3,8 @@ from typing import List, Set, Tuple
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
6
+from vllm.model_executor.layers.sampler import SamplerOutput
7
+from vllm.sequence import ExecuteModelRequest
8
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
9
make_async)
10
vllm/executor/openvino_executor.py
+2
-1
@@ -9,7 +9,8 @@ from vllm.config import CacheConfig, ModelConfig
9
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
12
+from vllm.model_executor.layers.sampler import SamplerOutput
13
+from vllm.sequence import ExecuteModelRequest
14
from vllm.utils import (GiB_bytes, get_distributed_init_method, get_ip,
15
get_open_port, make_async)
16
vllm/executor/ray_gpu_executor.py
+2
-1
@@ -12,7 +12,8 @@ from vllm.executor.distributed_gpu_executor import ( # yapf: disable
12
from vllm.executor.msgspec_utils import encode_hook
13
from vllm.executor.ray_utils import RayWorkerWrapper, ray
14
from vllm.logger import init_logger
15
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
15
+from vllm.model_executor.layers.sampler import SamplerOutput
16
+from vllm.sequence import ExecuteModelRequest
17
from vllm.utils import (_run_task_with_lock, get_distributed_init_method,
18
get_ip, get_open_port, get_vllm_instance_id,
19
make_async)
vllm/executor/ray_tpu_executor.py
+2
-1
@@ -10,7 +10,8 @@ from vllm.executor.executor_base import ExecutorAsyncBase
10
from vllm.executor.ray_utils import RayWorkerWrapper, ray
11
from vllm.executor.tpu_executor import TPUExecutor
12
from vllm.logger import init_logger
13
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
13
+from vllm.model_executor.layers.sampler import SamplerOutput
14
+from vllm.sequence import ExecuteModelRequest
15
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
16
get_vllm_instance_id, make_async)
17
vllm/executor/tpu_executor.py
+2
-1
@@ -5,7 +5,8 @@ import torch
5
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
+from vllm.sequence import ExecuteModelRequest
10
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
11
make_async)
12
vllm/executor/xpu_executor.py
+2
-1
@@ -9,7 +9,8 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
from vllm.executor.executor_base import ExecutorAsyncBase
10
from vllm.executor.gpu_executor import GPUExecutor
11
from vllm.logger import init_logger
12
-from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
12
+from vllm.model_executor.layers.sampler import SamplerOutput
13
+from vllm.sequence import ExecuteModelRequest, PoolerOutput
14
from vllm.utils import make_async
15
from vllm.worker.worker_base import WorkerBase
16
vllm/model_executor/layers/sampler.py
+247
-43
@@ -1,13 +1,16 @@
1
"""A layer that samples the next tokens from the model's outputs."""
2
import itertools
3
import warnings
4
+from dataclasses import dataclass
5
from importlib.util import find_spec
6
from math import inf
6
-from typing import Dict, List, Optional, Tuple
7
+from typing import Dict, List, Optional, Tuple, Union
8
9
+import msgspec
10
import torch
11
import torch.nn as nn
12
13
+from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
14
from vllm.triton_utils import HAS_TRITON
15
16
if HAS_TRITON:
@@ -19,8 +22,7 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
22
SequenceGroupToSample)
23
from vllm.sampling_params import SamplingType
24
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
22
- PromptLogprobs, SampleLogprobs, SamplerOutput,
23
- SequenceOutput)
25
+ PromptLogprobs, SampleLogprobs, SequenceOutput)
26
27
if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
28
import flashinfer.sampling
@@ -35,6 +37,116 @@ else:
37
# (num_token_ids, num_parent_ids) per sequence group.
38
SampleResultType = List[Tuple[List[int], List[int]]]
39
40
+# Types of temporary data structures used for
41
+# computing sample_result
42
+SampleMetadataType = Dict[SamplingType, Tuple[List[int],
43
+ List[SequenceGroupToSample]]]
44
+MultinomialSamplesType = Dict[SamplingType, torch.Tensor]
45
+SampleResultsDictType = Dict[int, Tuple[List[int], List[int]]]
46
+
47
+
48
+# Encapsulates temporary data structures for computing
49
+# sample_result.
50
+#
51
+# * For multi-step scheduling: must be returned
52
+# by `Sampler.forward()` and used later to compute the pythonized
53
+# sample_result
54
+#
55
+# * For single-step scheduling: consumed immediately
56
+# inside `Sampler.forward()` to compute pythonized sample_result.
57
+@dataclass
58
+class SampleResultArgsType:
59
+ sample_metadata: SampleMetadataType
60
+ multinomial_samples: MultinomialSamplesType
61
+ sample_results_dict: SampleResultsDictType
62
+ sampling_metadata: SamplingMetadata
63
+ greedy_samples: Optional[torch.Tensor]
64
+ beam_search_logprobs: Optional[torch.Tensor]
65
+
66
+
67
+# Union of non-deferred (single-step scheduling)
68
+# vs deferred (multi-step scheduling)
69
+# sample result types
70
+MaybeDeferredSampleResultType = Union[SampleResultType, SampleResultArgsType]
71
+
72
+# Abbreviation of the _sample() return type
73
+SampleReturnType = Tuple[MaybeDeferredSampleResultType, Optional[torch.Tensor]]
74
+
75
+
76
+class SamplerOutput(
77
+ msgspec.Struct,
78
+ omit_defaults=True, # type: ignore[call-arg]
79
+ array_like=True): # type: ignore[call-arg]
80
+ """For each sequence group, we generate a list of SequenceOutput object,
81
+ each of which contains one possible candidate for the next token.
82
+
83
+ This data structure implements methods, so it can be used like a list, but
84
+ also has optional fields for device tensors.
85
+ """
86
+
87
+ outputs: List[CompletionSequenceGroupOutput]
88
+
89
+ # On-device tensor containing probabilities of each token.
90
+ sampled_token_probs: Optional[torch.Tensor] = None
91
+
92
+ # On-device tensor containing the logprobs of each token.
93
+ logprobs: Optional["torch.Tensor"] = None
94
+
95
+ # Holds either (1) the pythonized sampler result (single-step scheduling)
96
+ # or (2) what will be arguments for later deferred pythonization of the
97
+ # sampler result (muliti-step scheduling)
98
+ deferred_sample_results_args: Optional[SampleResultArgsType] = None
99
+
100
+ # On-device tensor containing the sampled token ids.
101
+ sampled_token_ids: Optional[torch.Tensor] = None
102
+ # CPU tensor containing the sampled token ids. Used during multi-step to
103
+ # return the sampled token ids from last rank to AsyncLLMEngine to be
104
+ # 'broadcasted' to all other PP ranks for next step.
105
+ sampled_token_ids_cpu: Optional[torch.Tensor] = None
106
+
107
+ # Spec decode metrics populated by workers.
108
+ spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
109
+
110
+ # Optional last hidden states from the model.
111
+ hidden_states: Optional[torch.Tensor] = None
112
+
113
+ # Optional prefill hidden states from the model
114
+ # (used for models like EAGLE).
115
+ prefill_hidden_states: Optional[torch.Tensor] = None
116
+
117
+ # Time taken in the forward pass for this across all workers
118
+ model_forward_time: Optional[float] = None
119
+
120
+ # Time taken in the model execute function. This will include model forward,
121
+ # block/sync across workers, cpu-gpu sync time and sampling time.
122
+ model_execute_time: Optional[float] = None
123
+
124
+ def __getitem__(self, idx: int):
125
+ return self.outputs[idx]
126
+
127
+ def __setitem__(self, idx: int, value):
128
+ self.outputs[idx] = value
129
+
130
+ def __len__(self):
131
+ return len(self.outputs)
132
+
133
+ def __eq__(self, other: object):
134
+ return isinstance(other,
135
+ self.__class__) and self.outputs == other.outputs
136
+
137
+ def __repr__(self) -> str:
138
+ """Show the shape of a tensor instead of its values to reduce noise.
139
+ """
140
+ sampled_token_probs_repr = ("None" if self.sampled_token_probs is None
141
+ else self.sampled_token_probs.shape)
142
+ sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else
143
+ self.sampled_token_ids.shape)
144
+ return (
145
+ f"SamplerOutput(outputs={self.outputs}, "
146
+ f"sampled_token_probs={sampled_token_probs_repr}, "
147
+ f"sampled_token_ids={sampled_token_ids_repr}, "
148
+ f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
149
+
150
151
class Sampler(nn.Module):
152
"""Samples the next tokens from the model's outputs.
@@ -98,6 +210,19 @@ class Sampler(nn.Module):
210
sampling_metadata: SamplingMetadata,
211
) -> Optional[SamplerOutput]:
212
"""
213
+ Single-step scheduling:
214
+ * Perform GPU-side sampling computation & compute
215
+ GPU-side logprobs tensor
216
+ * Pythonize sampling result & logprobs tensor
217
+
218
+ Multi-step scheduling:
219
+ * Perform GPU-side sampling computation & compute
220
+ GPU-side logprobs tensor
221
+ * Defer Pythonization of sampling result & logprobs
222
+ tensor
223
+ * Encapsulate arguments required for deferred Pythonization
224
+ in the :class:`SamplerOutput` structure
225
+
226
Args:
227
logits: (num_tokens, vocab_size).
228
sampling_metadata: Metadata for sampling.
@@ -150,7 +275,7 @@ class Sampler(nn.Module):
275
logprobs = torch.log_softmax(logits, dim=-1, dtype=torch.float)
276
277
# Sample the next tokens.
153
- sample_results, maybe_sampled_tokens_tensor = _sample(
278
+ maybe_deferred_sample_results, maybe_sampled_tokens_tensor = _sample(
279
probs,
280
logprobs,
281
sampling_metadata,
@@ -160,20 +285,28 @@ class Sampler(nn.Module):
285
)
286
287
if self.include_gpu_probs_tensor:
288
+ # Since we will defer sampler result Pythonization,
289
+ # preserve GPU-side tensors in support of later
290
+ # deferred pythonization of logprobs
291
assert maybe_sampled_tokens_tensor is not None
292
on_device_tensors = (probs, logprobs, maybe_sampled_tokens_tensor)
293
else:
294
+ # Since Pythonization has already happened, don't preserve
295
+ # GPU-side tensors.
296
on_device_tensors = None
297
298
# Get the logprobs query results.
299
prompt_logprobs = None
300
sample_logprobs = None
301
if not sampling_metadata.skip_sampler_cpu_output:
172
- prompt_logprobs, sample_logprobs = _get_logprobs(
173
- logprobs, sampling_metadata, sample_results)
302
+ # Pythonize logprobs now (GPU -> CPU); do not defer.
303
+ assert not isinstance(maybe_deferred_sample_results,
304
+ SampleResultArgsType)
305
+ prompt_logprobs, sample_logprobs = get_logprobs(
306
+ logprobs, sampling_metadata, maybe_deferred_sample_results)
307
308
return _build_sampler_output(
176
- sample_results,
309
+ maybe_deferred_sample_results,
310
sampling_metadata,
311
prompt_logprobs,
312
sample_logprobs,
@@ -543,6 +676,60 @@ def _top_k_top_p_multinomial_with_flashinfer(
676
return batch_next_token_ids.view(-1, num_samples)
677
678
679
+def get_pythonized_sample_results(
680
+ sample_result_args: SampleResultArgsType) -> SampleResultType:
681
+ '''This function consumes GPU-side sampler results and computes
682
+ Pythonized CPU-side sampler results (GPU -> CPU sync.)
683
+
684
+ Single-step scheduling: this function is invoked at sampling-time
685
+ for immediate Pythonization.
686
+
687
+ Multi-step scheduling: Pythonization is deferred until after multiple
688
+ GPU-side steps have been completed.
689
+
690
+ Args:
691
+ sample_result_args: GPU-side inputs to the Pythonization process
692
+
693
+ Returns:
694
+ Pythonized sampler results
695
+ '''
696
+
697
+ (
698
+ sample_metadata,
699
+ sampling_metadata,
700
+ greedy_samples,
701
+ multinomial_samples,
702
+ beam_search_logprobs,
703
+ sample_results_dict,
704
+ ) = (
705
+ sample_result_args.sample_metadata,
706
+ sample_result_args.sampling_metadata,
707
+ sample_result_args.greedy_samples,
708
+ sample_result_args.multinomial_samples,
709
+ sample_result_args.beam_search_logprobs,
710
+ sample_result_args.sample_results_dict,
711
+ )
712
+
713
+ for sampling_type in SamplingType:
714
+ if sampling_type not in sample_metadata:
715
+ continue
716
+ (seq_group_id, seq_groups) = sample_metadata[sampling_type]
717
+ if sampling_type == SamplingType.GREEDY:
718
+ sample_results = _greedy_sample(seq_groups, greedy_samples)
719
+ elif sampling_type in (SamplingType.RANDOM, SamplingType.RANDOM_SEED):
720
+ sample_results = _random_sample(seq_groups,
721
+ multinomial_samples[sampling_type])
722
+ elif sampling_type == SamplingType.BEAM:
723
+ sample_results = _beam_search_sample(seq_groups,
724
+ beam_search_logprobs)
725
+ sample_results_dict.update(zip(seq_group_id, sample_results))
726
+
727
+ return [
728
+ sample_results_dict.get(i, ([], []))
729
+ for i in range(len(sampling_metadata.seq_groups))
730
+ ]
731
+
732
+
733
def _sample_with_torch(
734
probs: torch.Tensor,
735
logprobs: torch.Tensor,
@@ -550,7 +737,19 @@ def _sample_with_torch(
737
sampling_tensors: SamplingTensors,
738
include_gpu_probs_tensor: bool,
739
modify_greedy_probs: bool,
553
-) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
740
+) -> SampleReturnType:
741
+ '''Torch-oriented _sample() implementation.
742
+
743
+ Single-step scheduling:
744
+ * Perform GPU-side sampling computation
745
+ * Immediately Pythonize sampling result
746
+
747
+ Multi-step scheduling:
748
+ * Perform GPU-side sampling computation
749
+ * Defer Pythonization & preserve GPU-side
750
+ tensors required for Pythonization
751
+ '''
752
+
753
categorized_seq_group_ids: Dict[SamplingType,
754
List[int]] = {t: []
755
for t in SamplingType}
@@ -560,10 +759,11 @@ def _sample_with_torch(
759
sampling_type = sampling_params.sampling_type
760
categorized_seq_group_ids[sampling_type].append(i)
761
563
- sample_results_dict: Dict[int, Tuple[List[int], List[int]]] = {}
564
- sample_metadata: Dict[SamplingType,
565
- Tuple[List[int], List[SequenceGroupToSample]]] = {}
566
- multinomial_samples: Dict[SamplingType, torch.Tensor] = {}
762
+ sample_results_dict: SampleResultsDictType = {}
763
+ sample_metadata: SampleMetadataType = {}
764
+ multinomial_samples: MultinomialSamplesType = {}
765
+ greedy_samples: Optional[torch.Tensor] = None
766
+ beam_search_logprobs: Optional[torch.Tensor] = None
767
768
# Create output tensor for sampled token ids.
769
if include_gpu_probs_tensor:
@@ -638,32 +838,29 @@ def _sample_with_torch(
838
else:
839
raise ValueError(f"Unsupported sampling type: {sampling_type}")
840
641
- # GPU<->CPU sync happens in the loop below.
642
- # This also converts the sample output to Python objects.
841
+ # Encapsulate arguments for computing Pythonized sampler
842
+ # results, whether deferred or otherwise.
843
+ maybe_deferred_args = SampleResultArgsType(
844
+ sampling_metadata=sampling_metadata,
845
+ sample_metadata=sample_metadata,
846
+ multinomial_samples=multinomial_samples,
847
+ greedy_samples=greedy_samples,
848
+ beam_search_logprobs=beam_search_logprobs,
849
+ sample_results_dict=sample_results_dict)
850
+
851
if not sampling_metadata.skip_sampler_cpu_output:
644
- for sampling_type in SamplingType:
645
- if sampling_type not in sample_metadata:
646
- continue
647
- (seq_group_id, seq_groups) = sample_metadata[sampling_type]
648
- if sampling_type == SamplingType.GREEDY:
649
- sample_results = _greedy_sample(seq_groups, greedy_samples)
650
- elif sampling_type in (SamplingType.RANDOM,
651
- SamplingType.RANDOM_SEED):
652
- sample_results = _random_sample(
653
- seq_groups, multinomial_samples[sampling_type])
654
- elif sampling_type == SamplingType.BEAM:
655
- sample_results = _beam_search_sample(seq_groups,
656
- beam_search_logprobs)
657
- sample_results_dict.update(zip(seq_group_id, sample_results))
658
-
659
- sample_results = [
660
- sample_results_dict.get(i, ([], []))
661
- for i in range(len(sampling_metadata.seq_groups))
662
- ]
852
+ # GPU<->CPU sync happens here.
853
+ # This also converts the sampler output to a Python object.
854
+ # Return Pythonized sampler result & sampled token ids
855
+ return get_pythonized_sample_results(
856
+ maybe_deferred_args), sampled_token_ids_tensor
857
else:
664
- sample_results = []
665
-
666
- return sample_results, sampled_token_ids_tensor
858
+ # Defer sampler result Pythonization; return deferred
859
+ # Pythonization args & sampled token ids
860
+ return (
861
+ maybe_deferred_args,
862
+ sampled_token_ids_tensor,
863
+ )
864
865
866
def _sample_with_triton_kernel(
@@ -755,7 +952,7 @@ def _sample(
952
sampling_tensors: SamplingTensors,
953
include_gpu_probs_tensor: bool,
954
modify_greedy_probs: bool,
758
-) -> Tuple[SampleResultType, Optional[torch.Tensor]]:
955
+) -> SampleReturnType:
956
"""
957
Args:
958
probs: (num_query_tokens_in_batch, num_vocab)
@@ -803,7 +1000,7 @@ def _get_ranks(x: torch.Tensor, indices: torch.Tensor) -> torch.Tensor:
1000
return result.sum(1).add_(1)
1001
1002
806
-def _get_logprobs(
1003
+def get_logprobs(
1004
logprobs: torch.Tensor,
1005
sampling_metadata: SamplingMetadata,
1006
sample_results: SampleResultType,
@@ -1126,7 +1323,7 @@ def _modify_greedy_probs_inplace(logprobs: torch.Tensor, probs: torch.Tensor,
1323
1324
1325
def _build_sampler_output(
1129
- sample_results: SampleResultType,
1326
+ maybe_deferred_sample_results: MaybeDeferredSampleResultType,
1327
sampling_metadata: SamplingMetadata,
1328
prompt_logprobs: Optional[List[Optional[PromptLogprobs]]],
1329
sample_logprobs: Optional[List[SampleLogprobs]],
@@ -1143,14 +1340,21 @@ def _build_sampler_output(
1340
speculative decoding rejection sampling.
1341
"""
1342
sampler_output: List[CompletionSequenceGroupOutput] = []
1146
- if not skip_sampler_cpu_output:
1343
+
1344
+ if skip_sampler_cpu_output:
1345
+ assert isinstance(maybe_deferred_sample_results, SampleResultArgsType)
1346
+ deferred_sample_results_args = maybe_deferred_sample_results
1347
+ else:
1348
assert prompt_logprobs is not None
1349
assert sample_logprobs is not None
1350
+ assert not isinstance(maybe_deferred_sample_results,
1351
+ SampleResultArgsType)
1352
+ deferred_sample_results_args = None
1353
1354
for (seq_group, sample_result, group_prompt_logprobs,
1355
group_sample_logprobs) in zip(sampling_metadata.seq_groups,
1152
- sample_results, prompt_logprobs,
1153
- sample_logprobs):
1356
+ maybe_deferred_sample_results,
1357
+ prompt_logprobs, sample_logprobs):
1358
seq_ids = seq_group.seq_ids
1359
next_token_ids, parent_ids = sample_result
1360
seq_outputs: List[SequenceOutput] = []
@@ -1176,7 +1380,7 @@ def _build_sampler_output(
1380
sampled_token_probs=sampled_token_probs,
1381
sampled_token_ids=sampled_token_ids,
1382
logprobs=logprobs_tensor,
1179
- )
1383
+ deferred_sample_results_args=deferred_sample_results_args)
1384
1385
1386
def _get_next_prompt_tokens(seq_group: SequenceGroupToSample) -> List[int]:
vllm/model_executor/model_loader/neuron.py
+1
-2
@@ -10,9 +10,8 @@ from transformers import PretrainedConfig
10
11
from vllm.config import ModelConfig, ParallelConfig, SchedulerConfig
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
-from vllm.model_executor.layers.sampler import Sampler
13
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
14
from vllm.model_executor.sampling_metadata import SamplingMetadata
15
-from vllm.sequence import SamplerOutput
15
16
TORCH_DTYPE_TO_NEURON_AMP = {
17
"auto": "f32",
vllm/model_executor/model_loader/openvino.py
+1
-2
@@ -15,9 +15,8 @@ from vllm.config import DeviceConfig, ModelConfig
15
from vllm.logger import init_logger
16
from vllm.model_executor.layers.logits_processor import (LogitsProcessor,
17
_prune_hidden_states)
18
-from vllm.model_executor.layers.sampler import Sampler
18
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
-from vllm.sequence import SamplerOutput
20
21
logger = init_logger(__name__)
22
vllm/model_executor/models/arctic.py
+2
-2
@@ -23,13 +23,13 @@ from vllm.model_executor.layers.quantization.base_config import (
23
from vllm.model_executor.layers.quantization.deepspeedfp import (
24
DeepSpeedFPConfig, DeepSpeedFPParameter)
25
from vllm.model_executor.layers.rotary_embedding import get_rope
26
-from vllm.model_executor.layers.sampler import Sampler
26
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
27
from vllm.model_executor.layers.vocab_parallel_embedding import (
28
ParallelLMHead, VocabParallelEmbedding)
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.sampling_metadata import SamplingMetadata
31
from vllm.model_executor.utils import set_weight_attrs
32
-from vllm.sequence import IntermediateTensors, SamplerOutput
32
+from vllm.sequence import IntermediateTensors
33
from vllm.transformers_utils.configs.arctic import ArcticConfig
34
35
logger = init_logger(__name__)
vllm/model_executor/models/baichuan.py
+2
-2
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization.base_config import (
39
QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler
41
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import IntermediateTensors, SamplerOutput
46
+from vllm.sequence import IntermediateTensors
47
48
from .interfaces import SupportsLoRA
49
vllm/model_executor/models/bart.py
+2
-2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
-from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
44
logger = logging.get_logger(__name__)
45
vllm/model_executor/models/blip2.py
+2
-2
@@ -13,13 +13,13 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.logits_processor import LogitsProcessor
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
-from vllm.model_executor.layers.sampler import Sampler
16
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
17
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.model_executor.models.opt import OPTModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
22
- SamplerOutput, SequenceData)
22
+ SequenceData)
23
24
from .blip import (BlipVisionModel, dummy_image_for_blip,
25
get_max_blip_image_tokens)
vllm/model_executor/models/bloom.py
+2
-2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
-from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
44
45
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
vllm/model_executor/models/chameleon.py
+2
-2
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22
from vllm.model_executor.layers.logits_processor import LogitsProcessor
23
from vllm.model_executor.layers.quantization import QuantizationConfig
24
from vllm.model_executor.layers.rotary_embedding import get_rope
25
-from vllm.model_executor.layers.sampler import Sampler
25
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26
from vllm.model_executor.layers.vocab_parallel_embedding import (
27
ParallelLMHead, VocabParallelEmbedding)
28
from vllm.model_executor.model_loader.weight_utils import (
@@ -33,7 +33,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.utils import (cached_get_tokenizer,
34
repeat_and_pad_placeholder_tokens)
35
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
36
- SamplerOutput, SequenceData)
36
+ SequenceData)
37
from vllm.utils import print_warning_once
38
39
from .interfaces import SupportsMultiModal
vllm/model_executor/models/chatglm.py
+2
-2
@@ -20,12 +20,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
from vllm.model_executor.layers.quantization.base_config import (
21
QuantizationConfig)
22
from vllm.model_executor.layers.rotary_embedding import get_rope
23
-from vllm.model_executor.layers.sampler import Sampler
23
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
ParallelLMHead, VocabParallelEmbedding)
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
-from vllm.sequence import IntermediateTensors, SamplerOutput
28
+from vllm.sequence import IntermediateTensors
29
from vllm.transformers_utils.configs import ChatGLMConfig
30
31
from .interfaces import SupportsLoRA
vllm/model_executor/models/commandr.py
+2
-2
@@ -38,14 +38,14 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization.base_config import (
39
QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler
41
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import (
45
default_weight_loader, row_parallel_weight_loader)
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.model_executor.utils import set_weight_attrs
48
-from vllm.sequence import IntermediateTensors, SamplerOutput
48
+from vllm.sequence import IntermediateTensors
49
50
51
@torch.compile
vllm/model_executor/models/dbrx.py
+2
-2
@@ -17,13 +17,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
from vllm.model_executor.layers.quantization.base_config import (
18
QuantizationConfig)
19
from vllm.model_executor.layers.rotary_embedding import get_rope
20
-from vllm.model_executor.layers.sampler import Sampler
20
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.model_executor.utils import set_weight_attrs
26
-from vllm.sequence import IntermediateTensors, SamplerOutput
26
+from vllm.sequence import IntermediateTensors
27
from vllm.transformers_utils.configs.dbrx import DbrxConfig
28
29
vllm/model_executor/models/deepseek.py
+2
-2
@@ -43,12 +43,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
from vllm.model_executor.layers.quantization.base_config import (
44
QuantizationConfig)
45
from vllm.model_executor.layers.rotary_embedding import get_rope
46
-from vllm.model_executor.layers.sampler import Sampler
46
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
48
ParallelLMHead, VocabParallelEmbedding)
49
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
-from vllm.sequence import IntermediateTensors, SamplerOutput
51
+from vllm.sequence import IntermediateTensors
52
53
54
class DeepseekMLP(nn.Module):
vllm/model_executor/models/deepseek_v2.py
+2
-2
@@ -43,12 +43,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
43
from vllm.model_executor.layers.quantization.base_config import (
44
QuantizationConfig)
45
from vllm.model_executor.layers.rotary_embedding import get_rope
46
-from vllm.model_executor.layers.sampler import Sampler
46
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
47
from vllm.model_executor.layers.vocab_parallel_embedding import (
48
ParallelLMHead, VocabParallelEmbedding)
49
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
-from vllm.sequence import IntermediateTensors, SamplerOutput
51
+from vllm.sequence import IntermediateTensors
52
53
from .utils import PPMissingLayer, is_pp_missing_parameter, make_layers
54
vllm/model_executor/models/eagle.py
+2
-1
@@ -5,12 +5,13 @@ import torch.nn as nn
5
6
from vllm.attention.backends.abstract import AttentionMetadata
7
from vllm.model_executor.layers.logits_processor import LogitsProcessor
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.model_executor.layers.vocab_parallel_embedding import (
10
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
11
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
12
from vllm.model_executor.models import ModelRegistry
13
from vllm.model_executor.sampling_metadata import SamplingMetadata
13
-from vllm.sequence import IntermediateTensors, SamplerOutput
14
+from vllm.sequence import IntermediateTensors
15
from vllm.transformers_utils.configs.eagle import EAGLEConfig
16
17
vllm/model_executor/models/falcon.py
+2
-2
@@ -39,12 +39,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization.base_config import (
40
QuantizationConfig)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
-from vllm.model_executor.layers.sampler import Sampler
42
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
-from vllm.sequence import IntermediateTensors, SamplerOutput
47
+from vllm.sequence import IntermediateTensors
48
from vllm.transformers_utils.configs import RWConfig
49
50
FalconConfig = Union[HF_FalconConfig, RWConfig]
vllm/model_executor/models/fuyu.py
+2
-1
@@ -31,6 +31,7 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31
from vllm.logger import init_logger
32
from vllm.model_executor.layers.linear import ColumnParallelLinear
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
+from vllm.model_executor.layers.sampler import SamplerOutput
35
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
36
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -39,7 +40,7 @@ from vllm.multimodal.base import MultiModalInputs
40
from vllm.multimodal.image import cached_get_image_processor
41
from vllm.multimodal.utils import cached_get_tokenizer
42
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
- SamplerOutput, SequenceData)
43
+ SequenceData)
44
45
from .interfaces import SupportsMultiModal
46
from .utils import merge_multimodal_embeddings
vllm/model_executor/models/gemma.py
+2
-2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.quantization.base_config import (
35
QuantizationConfig)
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
-from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
44
from .interfaces import SupportsLoRA
45
vllm/model_executor/models/gemma2.py
+2
-2
@@ -33,12 +33,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization.base_config import (
34
QuantizationConfig)
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler
36
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.sequence import IntermediateTensors, SamplerOutput
41
+from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsLoRA
44
vllm/model_executor/models/gpt2.py
+2
-2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
-from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
44
from .utils import is_pp_missing_parameter, make_layers
45
vllm/model_executor/models/gpt_bigcode.py
+2
-2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
-from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
44
from .interfaces import SupportsLoRA
45
vllm/model_executor/models/gpt_j.py
+2
-2
@@ -33,12 +33,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization.base_config import (
34
QuantizationConfig)
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler
36
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.sequence import IntermediateTensors, SamplerOutput
41
+from vllm.sequence import IntermediateTensors
42
43
44
class GPTJAttention(nn.Module):
vllm/model_executor/models/gpt_neox.py
+2
-2
@@ -33,12 +33,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization.base_config import (
34
QuantizationConfig)
35
from vllm.model_executor.layers.rotary_embedding import get_rope
36
-from vllm.model_executor.layers.sampler import Sampler
36
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
ParallelLMHead, VocabParallelEmbedding)
39
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.sampling_metadata import SamplingMetadata
41
-from vllm.sequence import IntermediateTensors, SamplerOutput
41
+from vllm.sequence import IntermediateTensors
42
43
44
class GPTNeoXAttention(nn.Module):
vllm/model_executor/models/internlm2.py
+2
-2
@@ -17,12 +17,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
from vllm.model_executor.layers.quantization.base_config import (
18
QuantizationConfig)
19
from vllm.model_executor.layers.rotary_embedding import get_rope
20
-from vllm.model_executor.layers.sampler import Sampler
20
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
ParallelLMHead, VocabParallelEmbedding)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
-from vllm.sequence import IntermediateTensors, SamplerOutput
25
+from vllm.sequence import IntermediateTensors
26
27
28
class InternLM2MLP(nn.Module):
vllm/model_executor/models/internvl.py
+2
-1
@@ -18,13 +18,14 @@ from vllm.attention import AttentionMetadata
18
from vllm.config import CacheConfig, MultiModalConfig
19
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
20
from vllm.model_executor.layers.quantization import QuantizationConfig
21
+from vllm.model_executor.layers.sampler import SamplerOutput
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.models.intern_vit import InternVisionModel
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
from vllm.multimodal import MULTIMODAL_REGISTRY
26
from vllm.multimodal.base import MultiModalInputs
27
from vllm.multimodal.utils import cached_get_tokenizer
27
-from vllm.sequence import IntermediateTensors, SamplerOutput
28
+from vllm.sequence import IntermediateTensors
29
30
from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
31
get_clip_num_patches)
vllm/model_executor/models/jais.py
+2
-2
@@ -35,12 +35,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
35
from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.quantization.base_config import (
37
QuantizationConfig)
38
-from vllm.model_executor.layers.sampler import Sampler
38
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
40
ParallelLMHead, VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
-from vllm.sequence import IntermediateTensors, SamplerOutput
43
+from vllm.sequence import IntermediateTensors
44
from vllm.transformers_utils.configs import JAISConfig
45
46
from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/jamba.py
+2
-2
@@ -27,14 +27,14 @@ from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
27
selective_scan_fn, selective_state_update)
28
from vllm.model_executor.layers.quantization.base_config import (
29
QuantizationConfig)
30
-from vllm.model_executor.layers.sampler import Sampler
30
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
31
from vllm.model_executor.layers.vocab_parallel_embedding import (
32
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
33
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
34
from vllm.model_executor.models.interfaces import HasInnerState
35
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
from vllm.model_executor.utils import set_weight_attrs
37
-from vllm.sequence import IntermediateTensors, SamplerOutput
37
+from vllm.sequence import IntermediateTensors
38
from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
39
_get_graph_batch_size)
40
vllm/model_executor/models/llama.py
+2
-2
@@ -42,13 +42,13 @@ from vllm.model_executor.layers.quantization.base_config import (
42
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
43
get_compressed_tensors_cache_scale)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler
45
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import (
49
default_weight_loader, kv_cache_scales_loader, maybe_remap_kv_scale_name)
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
-from vllm.sequence import IntermediateTensors, SamplerOutput
51
+from vllm.sequence import IntermediateTensors
52
from vllm.utils import is_hip
53
54
from .interfaces import SupportsLoRA
vllm/model_executor/models/llava.py
+2
-1
@@ -11,10 +11,11 @@ from vllm.config import CacheConfig, MultiModalConfig
11
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
12
from vllm.model_executor.layers.activation import get_act_fn
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
+from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
17
-from vllm.sequence import IntermediateTensors, SamplerOutput
18
+from vllm.sequence import IntermediateTensors
19
20
from .clip import (CLIPVisionModel, dummy_image_for_clip,
21
dummy_seq_data_for_clip, get_max_clip_image_tokens,
vllm/model_executor/models/llava_next.py
+2
-1
@@ -15,10 +15,11 @@ from vllm.config import CacheConfig, MultiModalConfig
15
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
16
from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
+from vllm.model_executor.layers.sampler import SamplerOutput
19
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.model_executor.sampling_metadata import SamplingMetadata
21
from vllm.multimodal import MULTIMODAL_REGISTRY
21
-from vllm.sequence import IntermediateTensors, SamplerOutput
22
+from vllm.sequence import IntermediateTensors
23
from vllm.utils import is_list_of
24
25
from .clip import (CLIPVisionModel, dummy_image_for_clip,
vllm/model_executor/models/medusa.py
+1
-1
@@ -4,11 +4,11 @@ import torch
4
import torch.nn as nn
5
6
from vllm.model_executor.layers.logits_processor import LogitsProcessor
7
+from vllm.model_executor.layers.sampler import SamplerOutput
8
from vllm.model_executor.layers.vocab_parallel_embedding import (
9
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
10
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
11
from vllm.model_executor.sampling_metadata import SamplingMetadata
11
-from vllm.sequence import SamplerOutput
12
from vllm.transformers_utils.configs.medusa import MedusaConfig
13
14
vllm/model_executor/models/minicpm.py
+2
-2
@@ -44,13 +44,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
44
from vllm.model_executor.layers.quantization.base_config import (
45
QuantizationConfig)
46
from vllm.model_executor.layers.rotary_embedding import get_rope
47
-from vllm.model_executor.layers.sampler import Sampler
47
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
48
from vllm.model_executor.layers.vocab_parallel_embedding import (
49
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.model_executor.utils import set_weight_attrs
53
-from vllm.sequence import IntermediateTensors, SamplerOutput
53
+from vllm.sequence import IntermediateTensors
54
55
from .interfaces import SupportsLoRA
56
vllm/model_executor/models/minicpmv.py
+2
-2
@@ -44,7 +44,7 @@ from vllm.logger import init_logger
44
from vllm.model_executor.layers.linear import ReplicatedLinear
45
from vllm.model_executor.layers.logits_processor import LogitsProcessor
46
from vllm.model_executor.layers.quantization import QuantizationConfig
47
-from vllm.model_executor.layers.sampler import Sampler
47
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
48
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
50
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -57,7 +57,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
57
from vllm.multimodal.image import cached_get_image_processor
58
from vllm.multimodal.utils import cached_get_tokenizer
59
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
60
- SamplerOutput, SequenceData)
60
+ SequenceData)
61
62
from .idefics2_vision_model import Idefics2VisionTransformer
63
vllm/model_executor/models/mixtral.py
+2
-2
@@ -39,13 +39,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
39
from vllm.model_executor.layers.quantization.base_config import (
40
QuantizationConfig)
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
-from vllm.model_executor.layers.sampler import Sampler
42
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import (
46
default_weight_loader, maybe_remap_kv_scale_name)
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
-from vllm.sequence import IntermediateTensors, SamplerOutput
48
+from vllm.sequence import IntermediateTensors
49
50
from .interfaces import SupportsLoRA
51
from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/mixtral_quant.py
+2
-2
@@ -42,12 +42,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization.base_config import (
43
QuantizationConfig)
44
from vllm.model_executor.layers.rotary_embedding import get_rope
45
-from vllm.model_executor.layers.sampler import Sampler
45
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
46
from vllm.model_executor.layers.vocab_parallel_embedding import (
47
ParallelLMHead, VocabParallelEmbedding)
48
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
-from vllm.sequence import IntermediateTensors, SamplerOutput
50
+from vllm.sequence import IntermediateTensors
51
52
53
class MixtralMLP(nn.Module):
vllm/model_executor/models/mlp_speculator.py
+1
-2
@@ -6,11 +6,10 @@ import torch.nn as nn
6
7
from vllm.model_executor import SamplingMetadata
8
from vllm.model_executor.layers.logits_processor import LogitsProcessor
9
-from vllm.model_executor.layers.sampler import Sampler
9
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
10
from vllm.model_executor.layers.vocab_parallel_embedding import (
11
ParallelLMHead, VocabParallelEmbedding)
12
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
-from vllm.sequence import SamplerOutput
13
from vllm.transformers_utils.configs import MLPSpeculatorConfig
14
15
SQRT2 = 2**0.5
vllm/model_executor/models/mpt.py
+2
-2
@@ -17,12 +17,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.quantization.base_config import (
19
QuantizationConfig)
20
-from vllm.model_executor.layers.sampler import Sampler
20
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
VocabParallelEmbedding)
23
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
24
from vllm.model_executor.sampling_metadata import SamplingMetadata
25
-from vllm.sequence import IntermediateTensors, SamplerOutput
25
+from vllm.sequence import IntermediateTensors
26
from vllm.transformers_utils.configs.mpt import MPTConfig
27
28
vllm/model_executor/models/nemotron.py
+2
-2
@@ -37,13 +37,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.quantization.base_config import (
38
QuantizationConfig)
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.model_loader.weight_utils import (
44
default_weight_loader, maybe_remap_kv_scale_name)
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import IntermediateTensors, SamplerOutput
46
+from vllm.sequence import IntermediateTensors
47
from vllm.transformers_utils.configs import NemotronConfig
48
49
from .interfaces import SupportsLoRA
vllm/model_executor/models/olmo.py
+2
-2
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization.base_config import (
39
QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler
41
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import IntermediateTensors, SamplerOutput
46
+from vllm.sequence import IntermediateTensors
47
48
49
class OlmoAttention(nn.Module):
vllm/model_executor/models/opt.py
+2
-2
@@ -34,12 +34,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
-from vllm.model_executor.layers.sampler import Sampler
37
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
ParallelLMHead, VocabParallelEmbedding)
40
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
41
from vllm.model_executor.sampling_metadata import SamplingMetadata
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
44
45
class OPTLearnedPositionalEmbedding(nn.Embedding):
vllm/model_executor/models/orion.py
+2
-2
@@ -21,12 +21,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
from vllm.model_executor.layers.quantization.base_config import (
22
QuantizationConfig)
23
from vllm.model_executor.layers.rotary_embedding import get_rope
24
-from vllm.model_executor.layers.sampler import Sampler
24
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25
from vllm.model_executor.layers.vocab_parallel_embedding import (
26
ParallelLMHead, VocabParallelEmbedding)
27
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
-from vllm.sequence import IntermediateTensors, SamplerOutput
29
+from vllm.sequence import IntermediateTensors
30
31
32
class OrionMLP(nn.Module):
vllm/model_executor/models/paligemma.py
+2
-2
@@ -11,13 +11,13 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11
from vllm.logger import init_logger
12
from vllm.model_executor.layers.logits_processor import LogitsProcessor
13
from vllm.model_executor.layers.quantization import QuantizationConfig
14
-from vllm.model_executor.layers.sampler import Sampler
14
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
15
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
16
from vllm.model_executor.models.gemma import GemmaModel
17
from vllm.model_executor.sampling_metadata import SamplingMetadata
18
from vllm.multimodal import MULTIMODAL_REGISTRY
19
from vllm.multimodal.utils import cached_get_tokenizer
20
-from vllm.sequence import IntermediateTensors, SamplerOutput
20
+from vllm.sequence import IntermediateTensors
21
22
from .interfaces import SupportsMultiModal
23
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
vllm/model_executor/models/persimmon.py
+2
-2
@@ -37,12 +37,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
37
from vllm.model_executor.layers.quantization.base_config import (
38
QuantizationConfig)
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
-from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
42
ParallelLMHead, VocabParallelEmbedding)
43
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.sequence import IntermediateTensors, SamplerOutput
45
+from vllm.sequence import IntermediateTensors
46
47
48
class PersimmonMLP(nn.Module):
vllm/model_executor/models/phi.py
+2
-2
@@ -52,12 +52,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
52
from vllm.model_executor.layers.quantization.base_config import (
53
QuantizationConfig)
54
from vllm.model_executor.layers.rotary_embedding import get_rope
55
-from vllm.model_executor.layers.sampler import Sampler
55
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
56
from vllm.model_executor.layers.vocab_parallel_embedding import (
57
ParallelLMHead, VocabParallelEmbedding)
58
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
59
from vllm.model_executor.sampling_metadata import SamplingMetadata
60
-from vllm.sequence import IntermediateTensors, SamplerOutput
60
+from vllm.sequence import IntermediateTensors
61
62
from .interfaces import SupportsLoRA
63
vllm/model_executor/models/phi3_small.py
+2
-2
@@ -16,12 +16,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
from vllm.model_executor.layers.quantization.base_config import (
17
QuantizationConfig)
18
from vllm.model_executor.layers.rotary_embedding import get_rope
19
-from vllm.model_executor.layers.sampler import Sampler
19
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
20
from vllm.model_executor.layers.vocab_parallel_embedding import (
21
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
-from vllm.sequence import IntermediateTensors, SamplerOutput
24
+from vllm.sequence import IntermediateTensors
25
26
27
def load_column_parallel_weight(param: torch.nn.Parameter,
vllm/model_executor/models/phi3v.py
+2
-2
@@ -31,7 +31,7 @@ from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
31
from vllm.logger import init_logger
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
-from vllm.model_executor.layers.sampler import Sampler
34
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
35
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
36
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
37
from vllm.model_executor.models.clip import CLIPVisionModel
@@ -39,7 +39,7 @@ from vllm.model_executor.models.llama import LlamaModel
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.multimodal import MULTIMODAL_REGISTRY
41
from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
42
-from vllm.sequence import IntermediateTensors, SamplerOutput
42
+from vllm.sequence import IntermediateTensors
43
from vllm.utils import is_list_of
44
45
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
vllm/model_executor/models/qwen.py
+2
-2
@@ -22,12 +22,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
22
from vllm.model_executor.layers.quantization.base_config import (
23
QuantizationConfig)
24
from vllm.model_executor.layers.rotary_embedding import get_rope
25
-from vllm.model_executor.layers.sampler import Sampler
25
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26
from vllm.model_executor.layers.vocab_parallel_embedding import (
27
ParallelLMHead, VocabParallelEmbedding)
28
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
29
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
-from vllm.sequence import IntermediateTensors, SamplerOutput
30
+from vllm.sequence import IntermediateTensors
31
from vllm.utils import print_warning_once
32
33
from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/qwen2.py
+2
-2
@@ -40,13 +40,13 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
40
from vllm.model_executor.layers.quantization.base_config import (
41
QuantizationConfig)
42
from vllm.model_executor.layers.rotary_embedding import get_rope
43
-from vllm.model_executor.layers.sampler import Sampler
43
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
44
from vllm.model_executor.layers.vocab_parallel_embedding import (
45
ParallelLMHead, VocabParallelEmbedding)
46
from vllm.model_executor.model_loader.weight_utils import (
47
default_weight_loader, maybe_remap_kv_scale_name)
48
from vllm.model_executor.sampling_metadata import SamplingMetadata
49
-from vllm.sequence import IntermediateTensors, SamplerOutput
49
+from vllm.sequence import IntermediateTensors
50
51
from .interfaces import SupportsLoRA
52
from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/qwen2_moe.py
+2
-2
@@ -45,12 +45,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
45
from vllm.model_executor.layers.quantization.base_config import (
46
QuantizationConfig)
47
from vllm.model_executor.layers.rotary_embedding import get_rope
48
-from vllm.model_executor.layers.sampler import Sampler
48
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
49
from vllm.model_executor.layers.vocab_parallel_embedding import (
50
ParallelLMHead, VocabParallelEmbedding)
51
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
-from vllm.sequence import IntermediateTensors, SamplerOutput
53
+from vllm.sequence import IntermediateTensors
54
from vllm.utils import print_warning_once
55
56
from .utils import is_pp_missing_parameter, make_layers
vllm/model_executor/models/stablelm.py
+2
-2
@@ -36,12 +36,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
36
from vllm.model_executor.layers.quantization.base_config import (
37
QuantizationConfig)
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
-from vllm.model_executor.layers.sampler import Sampler
39
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
-from vllm.sequence import IntermediateTensors, SamplerOutput
44
+from vllm.sequence import IntermediateTensors
45
46
47
class StablelmMLP(nn.Module):
vllm/model_executor/models/starcoder2.py
+2
-2
@@ -35,12 +35,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
from vllm.model_executor.layers.rotary_embedding import get_rope
38
-from vllm.model_executor.layers.sampler import Sampler
38
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
from vllm.model_executor.layers.vocab_parallel_embedding import (
40
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
-from vllm.sequence import IntermediateTensors, SamplerOutput
43
+from vllm.sequence import IntermediateTensors
44
45
46
class Starcoder2Attention(nn.Module):
vllm/model_executor/models/ultravox.py
+2
-1
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
27
from vllm.model_executor.layers.layernorm import RMSNorm
28
from vllm.model_executor.layers.quantization.base_config import (
29
QuantizationConfig)
30
+from vllm.model_executor.layers.sampler import SamplerOutput
31
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32
from vllm.model_executor.models.interfaces import SupportsMultiModal
33
from vllm.model_executor.models.utils import (filter_weights,
@@ -37,7 +38,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
38
from vllm.multimodal.base import MultiModalInputs
39
from vllm.multimodal.utils import (cached_get_tokenizer,
40
repeat_and_pad_placeholder_tokens)
40
-from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SamplerOutput, SequenceData
41
+from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
42
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
43
44
_AUDIO_PLACEHOLDER_TOKEN = 128002
vllm/model_executor/models/xverse.py
+2
-2
@@ -38,12 +38,12 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.quantization.base_config import (
39
QuantizationConfig)
40
from vllm.model_executor.layers.rotary_embedding import get_rope
41
-from vllm.model_executor.layers.sampler import Sampler
41
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
42
from vllm.model_executor.layers.vocab_parallel_embedding import (
43
ParallelLMHead, VocabParallelEmbedding)
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.sampling_metadata import SamplingMetadata
46
-from vllm.sequence import IntermediateTensors, SamplerOutput
46
+from vllm.sequence import IntermediateTensors
47
48
from .interfaces import SupportsLoRA
49
vllm/sequence.py
-70
@@ -1060,76 +1060,6 @@ class IntermediateTensors(
1060
return f"IntermediateTensors(tensors={self.tensors})"
1061
1062
1063
-class SamplerOutput(
1064
- msgspec.Struct,
1065
- omit_defaults=True, # type: ignore[call-arg]
1066
- array_like=True): # type: ignore[call-arg]
1067
- """For each sequence group, we generate a list of SequenceOutput object,
1068
- each of which contains one possible candidate for the next token.
1069
-
1070
- This data structure implements methods, so it can be used like a list, but
1071
- also has optional fields for device tensors.
1072
- """
1073
-
1074
- outputs: List[CompletionSequenceGroupOutput]
1075
-
1076
- # On-device tensor containing probabilities of each token.
1077
- sampled_token_probs: Optional[torch.Tensor] = None
1078
-
1079
- # On-device tensor containing the logprobs of each token.
1080
- logprobs: Optional["torch.Tensor"] = None
1081
-
1082
- # On-device tensor containing the sampled token ids.
1083
- sampled_token_ids: Optional[torch.Tensor] = None
1084
- # CPU tensor containing the sampled token ids. Used during multi-step to
1085
- # return the sampled token ids from last rank to AsyncLLMEngine to be
1086
- # 'broadcasted' to all other PP ranks for next step.
1087
- sampled_token_ids_cpu: Optional[torch.Tensor] = None
1088
-
1089
- # Spec decode metrics populated by workers.
1090
- spec_decode_worker_metrics: Optional[SpecDecodeWorkerMetrics] = None
1091
-
1092
- # Optional last hidden states from the model.
1093
- hidden_states: Optional[torch.Tensor] = None
1094
-
1095
- # Optional prefill hidden states from the model
1096
- # (used for models like EAGLE).
1097
- prefill_hidden_states: Optional[torch.Tensor] = None
1098
-
1099
- # Time taken in the forward pass for this across all workers
1100
- model_forward_time: Optional[float] = None
1101
-
1102
- # Time taken in the model execute function. This will include model forward,
1103
- # block/sync across workers, cpu-gpu sync time and sampling time.
1104
- model_execute_time: Optional[float] = None
1105
-
1106
- def __getitem__(self, idx: int):
1107
- return self.outputs[idx]
1108
-
1109
- def __setitem__(self, idx: int, value):
1110
- self.outputs[idx] = value
1111
-
1112
- def __len__(self):
1113
- return len(self.outputs)
1114
-
1115
- def __eq__(self, other: object):
1116
- return isinstance(other,
1117
- self.__class__) and self.outputs == other.outputs
1118
-
1119
- def __repr__(self) -> str:
1120
- """Show the shape of a tensor instead of its values to reduce noise.
1121
- """
1122
- sampled_token_probs_repr = ("None" if self.sampled_token_probs is None
1123
- else self.sampled_token_probs.shape)
1124
- sampled_token_ids_repr = ("None" if self.sampled_token_ids is None else
1125
- self.sampled_token_ids.shape)
1126
- return (
1127
- f"SamplerOutput(outputs={self.outputs}, "
1128
- f"sampled_token_probs={sampled_token_probs_repr}, "
1129
- f"sampled_token_ids={sampled_token_ids_repr}, "
1130
- f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
1131
-
1132
-
1063
class PoolerOutput(
1064
msgspec.Struct,
1065
omit_defaults=True, # type: ignore[call-arg]
vllm/spec_decode/batch_expansion.py
+2
-1
@@ -5,8 +5,9 @@ from typing import Iterator, List, Optional, Tuple
5
import torch
6
7
from vllm import SamplingParams
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, ExecuteModelRequest,
9
- SamplerOutput, SequenceData, SequenceGroupMetadata,
10
+ SequenceData, SequenceGroupMetadata,
11
get_all_seq_ids)
12
from vllm.spec_decode.interfaces import (SpeculativeProposals,
13
SpeculativeScorer, SpeculativeScores)
vllm/spec_decode/draft_model_runner.py
+2
-2
@@ -3,6 +3,7 @@ from typing import List, Optional
3
import torch
4
5
from vllm import _custom_ops as ops
6
+from vllm.model_executor.layers.sampler import SamplerOutput
7
8
try:
9
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -16,8 +17,7 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
17
PromptAdapterConfig, SchedulerConfig)
18
from vllm.logger import init_logger
19
from vllm.multimodal import MultiModalInputs
19
-from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
20
- SamplerOutput)
20
+from vllm.sequence import ExecuteModelRequest, IntermediateTensors
21
from vllm.worker.model_runner import (ModelInputForGPUWithSamplingMetadata,
22
ModelRunner)
23
vllm/spec_decode/medusa_worker.py
+2
-2
@@ -4,8 +4,8 @@ from typing import List, Optional, Set, Tuple
4
import torch
5
6
from vllm.model_executor import SamplingMetadata
7
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
8
- SequenceGroupMetadata)
7
+from vllm.model_executor.layers.sampler import SamplerOutput
8
+from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
9
from vllm.spec_decode.interfaces import SpeculativeProposals
10
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11
from vllm.spec_decode.top1_proposer import Top1Proposer
vllm/spec_decode/mlp_speculator_worker.py
+2
-2
@@ -3,8 +3,8 @@ from typing import List, Optional, Set, Tuple
3
import torch
4
5
from vllm.model_executor import SamplingMetadata
6
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
7
- SequenceGroupMetadata)
6
+from vllm.model_executor.layers.sampler import SamplerOutput
7
+from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
8
from vllm.spec_decode.multi_step_worker import MultiStepWorker
9
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10
vllm/spec_decode/multi_step_worker.py
+3
-2
@@ -4,8 +4,9 @@ from typing import Dict, List, Set, Tuple
4
5
import torch
6
7
-from vllm.sequence import (ExecuteModelRequest, HiddenStates, SamplerOutput,
8
- SequenceData, SequenceGroupMetadata)
7
+from vllm.model_executor.layers.sampler import SamplerOutput
8
+from vllm.sequence import (ExecuteModelRequest, HiddenStates, SequenceData,
9
+ SequenceGroupMetadata)
10
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
11
from vllm.spec_decode.interfaces import (SpeculativeProposals,
12
SpeculativeProposer)
vllm/spec_decode/ngram_worker.py
+2
-1
@@ -3,7 +3,8 @@ from typing import List, Optional, Set, Tuple
3
4
import torch
5
6
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
6
+from vllm.model_executor.layers.sampler import SamplerOutput
7
+from vllm.sequence import ExecuteModelRequest
8
from vllm.spec_decode.interfaces import SpeculativeProposals
9
from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
10
from vllm.spec_decode.top1_proposer import Top1Proposer
vllm/spec_decode/proposer_worker_base.py
+2
-1
@@ -1,7 +1,8 @@
1
from abc import ABC, abstractmethod
2
from typing import List, Optional, Set, Tuple
3
4
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
4
+from vllm.model_executor.layers.sampler import SamplerOutput
5
+from vllm.sequence import ExecuteModelRequest
6
from vllm.spec_decode.interfaces import SpeculativeProposer
7
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
8
vllm/spec_decode/smaller_tp_proposer_worker.py
+2
-1
@@ -6,7 +6,8 @@ from vllm.distributed.parallel_state import (get_tp_group,
6
init_model_parallel_group,
7
patch_tensor_parallel_group)
8
from vllm.logger import init_logger
9
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
+from vllm.model_executor.layers.sampler import SamplerOutput
10
+from vllm.sequence import ExecuteModelRequest
11
from vllm.spec_decode.interfaces import SpeculativeProposals
12
from vllm.spec_decode.multi_step_worker import MultiStepWorker
13
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
vllm/spec_decode/spec_decode_worker.py
+2
-1
@@ -8,12 +8,13 @@ from vllm.config import ParallelConfig, SpeculativeConfig
8
from vllm.distributed.communication_op import broadcast_tensor_dict
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.rejection_sampler import RejectionSampler
11
+from vllm.model_executor.layers.sampler import SamplerOutput
12
from vllm.model_executor.layers.spec_decode_base_sampler import (
13
SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
14
from vllm.model_executor.layers.typical_acceptance_sampler import (
15
TypicalAcceptanceSampler)
16
from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16
- HiddenStates, SamplerOutput, SequenceGroupMetadata,
17
+ HiddenStates, SequenceGroupMetadata,
18
get_all_seq_ids, get_all_seq_ids_and_request_ids)
19
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
20
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
vllm/spec_decode/top1_proposer.py
+2
-2
@@ -2,8 +2,8 @@ from typing import List, Optional, Set, Tuple
2
3
import torch
4
5
-from vllm.sequence import (ExecuteModelRequest, SamplerOutput,
6
- SequenceGroupMetadata)
5
+from vllm.model_executor.layers.sampler import SamplerOutput
6
+from vllm.sequence import ExecuteModelRequest, SequenceGroupMetadata
7
from vllm.spec_decode.interfaces import (SpeculativeProposals,
8
SpeculativeProposer)
9
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
vllm/spec_decode/util.py
+2
-2
@@ -4,9 +4,9 @@ from typing import Dict, List, Optional, Sequence, Tuple
4
5
import torch
6
7
+from vllm.model_executor.layers.sampler import SamplerOutput
8
from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
8
- SamplerOutput, SequenceGroupMetadata,
9
- SequenceOutput)
9
+ SequenceGroupMetadata, SequenceOutput)
10
11
SeqId = int
12
vllm/worker/cpu_model_runner.py
+2
-2
@@ -10,11 +10,11 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
SchedulerConfig)
11
from vllm.logger import init_logger
12
from vllm.model_executor import SamplingMetadata
13
+from vllm.model_executor.layers.sampler import SamplerOutput
14
from vllm.model_executor.model_loader import get_model
15
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
16
MultiModalInputs)
16
-from vllm.sequence import (IntermediateTensors, SamplerOutput,
17
- SequenceGroupMetadata)
17
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
18
from vllm.utils import make_tensor_with_pad
19
from vllm.worker.model_runner_base import (
20
ModelRunnerBase, ModelRunnerInputBase,
vllm/worker/enc_dec_model_runner.py
+2
-1
@@ -16,9 +16,10 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry
17
from vllm.logger import init_logger
18
from vllm.model_executor import SamplingMetadata
19
+from vllm.model_executor.layers.sampler import SamplerOutput
20
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
21
from vllm.sampling_params import SamplingParams
21
-from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
22
+from vllm.sequence import (IntermediateTensors, PoolerOutput,
23
SequenceGroupMetadata)
24
from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
25
from vllm.worker.model_runner import (GPUModelRunnerBase,
vllm/worker/model_runner.py
+2
-2
@@ -29,6 +29,7 @@ from vllm.lora.layers import LoRAMapping
29
from vllm.lora.request import LoRARequest
30
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
31
from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
32
+from vllm.model_executor.layers.sampler import SamplerOutput
33
from vllm.model_executor.model_loader import get_model
34
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
35
from vllm.model_executor.models.interfaces import (supports_lora,
@@ -41,8 +42,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
42
from vllm.prompt_adapter.worker_manager import (
43
LRUCacheWorkerPromptAdapterManager)
44
from vllm.sampling_params import SamplingParams
44
-from vllm.sequence import (IntermediateTensors, SamplerOutput,
45
- SequenceGroupMetadata)
45
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
46
from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
47
flatten_2d_lists, is_hip, is_pin_memory_available,
48
supports_dynamo)
vllm/worker/model_runner_base.py
+2
-2
@@ -5,9 +5,9 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Type,
5
6
import torch
7
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
from vllm.platforms import current_platform
9
-from vllm.sequence import (IntermediateTensors, SamplerOutput,
10
- SequenceGroupMetadata)
10
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
11
12
if TYPE_CHECKING:
13
from vllm.attention import AttentionMetadata
vllm/worker/multi_step_model_runner.py
+154
-19
@@ -1,7 +1,8 @@
1
import dataclasses
2
import functools
3
from dataclasses import dataclass, field
4
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
4
+from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple,
5
+ Union)
6
7
try:
8
from vllm.attention.backends.flash_attn import FlashAttentionMetadata
@@ -15,9 +16,12 @@ import torch
16
from vllm import _custom_ops as ops
17
from vllm.distributed import get_pp_group
18
from vllm.logger import init_logger
19
+from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
20
+ SamplerOutput,
21
+ SamplingMetadata, get_logprobs,
22
+ get_pythonized_sample_results)
23
from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
19
- Logprob, SamplerOutput, SequenceGroupMetadata,
20
- SequenceOutput)
24
+ Logprob, SequenceGroupMetadata, SequenceOutput)
25
from vllm.worker.model_runner import (GPUModelRunnerBase,
26
ModelInputForGPUWithSamplingMetadata)
27
from vllm.worker.model_runner_base import (
@@ -53,6 +57,8 @@ class ModelOutput:
57
sampler_output_ready_event: torch.cuda.Event
58
sampled_token_ids: Optional[torch.Tensor] = None
59
pythonized: bool = False
60
+ # On-device tensor containing the logprobs of each token.
61
+ logprobs: Optional["torch.Tensor"] = None
62
63
def pythonize(self, input_metadata: "StatefulModelInput",
64
copy_stream: torch.cuda.Stream,
@@ -78,7 +84,9 @@ class ModelOutput:
84
blocking: bool) -> bool:
85
"""
86
If blocking is set, will block until the forward pass for the output is
81
- ready and pythonize the output.
87
+ ready and pythonize the output. Upon completing Pythonization, erases
88
+ self.logprobs (note that a non-blocking call that is performed when
89
+ the sampler output is not yet ready, will not erase self.logprobs.)
90
"""
91
assert self.sampled_token_ids is not None
92
if not blocking and not self.sampler_output_ready_event.query():
@@ -89,7 +97,15 @@ class ModelOutput:
97
with torch.cuda.stream(copy_stream):
98
_pythonize_sampler_output(input_metadata, self.sampler_output,
99
pinned_sampled_token_buffer,
92
- self.sampled_token_ids)
100
+ self.sampled_token_ids, self.logprobs)
101
+
102
+ # Erase the logprobs GPU-side tensor.
103
+ # Note that although _pythonize_sampler_output() runs in its
104
+ # own CUDA stream, nonetheless _pythonize_sampler_output()
105
+ # cannot return until Pythonization is complete; therefore
106
+ # we know that by the time the CPU reaches this point,
107
+ # `self.logprobs` is no longer needed.
108
+ self.logprobs = None
109
return True
110
111
@@ -350,11 +366,16 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
366
0].sampled_token_ids.cpu()
367
model_input.cached_outputs.append(
368
ModelOutput(output[0], output_ready_event,
353
- output[0].sampled_token_ids, False))
354
- # make sure we dont try to serialize any GPU tensors
369
+ output[0].sampled_token_ids, False,
370
+ output[0].logprobs))
371
+
372
+ # These GPU tensors are not required by multi-step;
373
+ # erase them to ensure they are not pythonized or
374
+ # transferred to CPU
375
output[0].sampled_token_ids = None
376
output[0].sampled_token_probs = None
377
output[0].logprobs = None
378
+
379
# Pythonize the output if CPU is ahead and the previous step is
380
# ready.
381
if not frozen_model_input.use_async_and_multi_step:
@@ -464,12 +485,75 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
485
return self._base_model_runner.vocab_size
486
487
467
-def _pythonize_sampler_output(model_input: StatefulModelInput,
468
- output: SamplerOutput,
469
- pinned_sampled_token_buffer: torch.Tensor,
470
- sampled_token_ids: torch.Tensor) -> None:
488
+DeferredLogprobsReturnType = Tuple[Optional[List[Optional[PromptLogprobs]]],
489
+ Optional[List[SampleLogprobs]]]
490
+
491
+
492
+def deferred_pythonize_logprobs(
493
+ output: SamplerOutput,
494
+ sampling_metadata: SamplingMetadata,
495
+ logprobs_tensor: Optional[torch.Tensor],
496
+) -> DeferredLogprobsReturnType:
497
+ """Perform deferred logprob Pythonization.
498
+
499
+ 1. Pythonize GPU-side sampler result tensors into CPU-side sampler result.
500
+ 2. Pythonize GPU-side logprobs tensor into CPU-side logprobs lists,
501
+ utilizing the Pythonized sampler result computed in step 1.
502
+
503
+ These deferred computations are not required for single-step scheduling
504
+ or the `profile_run()` phase of multi-step scheduling.
505
+
506
+ Args:
507
+ output: sampler output (under deferred Pythonization)
508
+ sampling_metadata
509
+
510
+ Returns:
511
+ prompt_logprobs (CPU), sample_logprobs (CPU)
512
+ """
513
+
514
+ # - Deferred pythonization of sample result
515
+ sampler_result = get_pythonized_sample_results(
516
+ output.deferred_sample_results_args)
517
+
518
+ # - Erase the GPU-side deferred sample_result
519
+ # computation args to ensure it is never
520
+ # pythonized or transferred to CPU
521
+ output.deferred_sample_results_args = None
522
+
523
+ # - Deferred pythonization of logprobs
524
+ (
525
+ prompt_logprobs,
526
+ sample_logprobs,
527
+ ) = get_logprobs(logprobs_tensor, sampling_metadata, sampler_result)
528
+ assert len(prompt_logprobs) == len(sampling_metadata.seq_groups)
529
+ assert len(sample_logprobs) == len(sampling_metadata.seq_groups)
530
+
531
+ return prompt_logprobs, sample_logprobs
532
+
533
+
534
+def _pythonize_sampler_output(
535
+ model_input: StatefulModelInput,
536
+ output: SamplerOutput,
537
+ pinned_sampled_token_buffer: torch.Tensor,
538
+ sampled_token_ids: torch.Tensor,
539
+ logprobs_tensor: Optional[torch.Tensor],
540
+) -> None:
541
""" This function is only called when the output tensors are ready.
472
- See ModelOutput
542
+ See :class:`ModelOutput`.
543
+
544
+ Modifies `output.outputs` and `pinned_sampled_token_buffer` in-place,
545
+ adding a Pythonized output data structure
546
+ (:class:`CompletionSequenceGroupOutput`) for each :class:`SequenceGroup`.
547
+
548
+ Args:
549
+ model_input
550
+ output: sampler output
551
+ pinned_sampled_token_token_buffer: CPU-side pinned memory
552
+ (receives copy of
553
+ GPU-side token buffer.)
554
+ sampled_token_ids: GPU-side token buffer
555
+ logprobs_tensor: GPU-side tensor containing
556
+ logprobs computed during sampling
557
"""
558
559
assert model_input.frozen_model_input is not None
@@ -489,8 +573,51 @@ def _pythonize_sampler_output(model_input: StatefulModelInput,
573
574
sampling_metadata = frozen_model_input.sampling_metadata
575
492
- for (seq_group, sample_result) in zip(sampling_metadata.seq_groups,
493
- samples_list):
576
+ skip_sampler_cpu_output = (
577
+ frozen_model_input.sampling_metadata.skip_sampler_cpu_output)
578
+
579
+ # We are guaranteed output tensors are ready, so it is safe to
580
+ # pythonize the sampler output & obtain CPU-side logprobs.
581
+ #
582
+ # However this computation may be skipped entirely
583
+ # if no pythonization was deferred.
584
+ seq_groups = sampling_metadata.seq_groups
585
+ logprobs_are_requested = any([
586
+ sg.sampling_params.logprobs is not None
587
+ or sg.sampling_params.prompt_logprobs is not None for sg in seq_groups
588
+ ])
589
+ do_pythonize_logprobs = (skip_sampler_cpu_output
590
+ and logprobs_are_requested)
591
+ (
592
+ prompt_logprobs,
593
+ sample_logprobs,
594
+ ) = (deferred_pythonize_logprobs(output, sampling_metadata,
595
+ logprobs_tensor)
596
+ if do_pythonize_logprobs else (None, None))
597
+
598
+ for sgdx, (seq_group,
599
+ sample_result) in enumerate(zip(seq_groups, samples_list)):
600
+
601
+ if do_pythonize_logprobs:
602
+ assert prompt_logprobs is not None
603
+ assert sample_logprobs is not None
604
+
605
+ (
606
+ group_prompt_logprobs,
607
+ group_sample_logprobs,
608
+ ) = ( # Utilize deferred pythonization results
609
+ prompt_logprobs[sgdx],
610
+ sample_logprobs[sgdx],
611
+ )
612
+ elif logprobs_are_requested:
613
+ (
614
+ group_prompt_logprobs,
615
+ group_sample_logprobs,
616
+ ) = (
617
+ # profile_run: use already-computed logprobs
618
+ output.outputs[sgdx].prompt_logprobs,
619
+ [sample.logprobs for sample in output.outputs[sgdx].samples])
620
+
621
seq_ids = seq_group.seq_ids
622
next_token_ids = sample_result
623
parent_ids = [0]
@@ -498,11 +625,19 @@ def _pythonize_sampler_output(model_input: StatefulModelInput,
625
if seq_group.sampling_params.logits_processors:
626
assert len(seq_group.sampling_params.logits_processors) == 0, (
627
"Logits Processors are not supported in multi-step decoding")
501
- for parent_id, next_token_id in zip(parent_ids, next_token_ids):
502
- # TODO(will): support logprobs
503
- # Hard coded logprob
628
+ for tdx, (parent_id,
629
+ next_token_id) in enumerate(zip(parent_ids, next_token_ids)):
630
seq_outputs.append(
631
SequenceOutput(seq_ids[parent_id], next_token_id,
506
- {next_token_id: Logprob(logprob=-1)}))
507
- output.outputs.append(CompletionSequenceGroupOutput(seq_outputs, None))
632
+ (group_sample_logprobs[tdx]
633
+ if logprobs_are_requested else {
634
+ next_token_id:
635
+ Logprob(logprob=float('inf'),
636
+ rank=None,
637
+ decoded_token=None)
638
+ })))
639
+ output.outputs.append(
640
+ CompletionSequenceGroupOutput(
641
+ seq_outputs,
642
+ (group_prompt_logprobs if logprobs_are_requested else None)))
643
assert len(output.outputs) > 0
vllm/worker/multi_step_worker.py
+2
-1
@@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple
5
import torch
6
7
from vllm.distributed import broadcast_tensor_dict, get_pp_group
8
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
8
+from vllm.model_executor.layers.sampler import SamplerOutput
9
+from vllm.sequence import ExecuteModelRequest
10
from vllm.worker.model_runner_base import BroadcastableModelInput
11
from vllm.worker.multi_step_model_runner import (MultiStepModelRunner,
12
StatefulModelInput)
vllm/worker/neuron_model_runner.py
+2
-2
@@ -8,11 +8,11 @@ from vllm.config import (DeviceConfig, ModelConfig, ParallelConfig,
8
SchedulerConfig)
9
from vllm.logger import init_logger
10
from vllm.model_executor import SamplingMetadata
11
+from vllm.model_executor.layers.sampler import SamplerOutput
12
from vllm.model_executor.model_loader.neuron import get_neuron_model
13
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
14
MultiModalInputs)
14
-from vllm.sequence import (IntermediateTensors, SamplerOutput,
15
- SequenceGroupMetadata)
15
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
16
from vllm.utils import is_pin_memory_available, make_tensor_with_pad
17
from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
18
vllm/worker/openvino_model_runner.py
+2
-1
@@ -11,10 +11,11 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
11
SchedulerConfig)
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
+from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.model_executor.model_loader.openvino import get_model
16
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
17
MultiModalInputs)
17
-from vllm.sequence import SamplerOutput, SequenceGroupMetadata
18
+from vllm.sequence import SequenceGroupMetadata
19
20
logger = init_logger(__name__)
21
vllm/worker/openvino_worker.py
+2
-1
@@ -14,7 +14,8 @@ from vllm.distributed import (broadcast_tensor_dict,
14
init_distributed_environment)
15
from vllm.logger import init_logger
16
from vllm.model_executor import set_random_seed
17
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
17
+from vllm.model_executor.layers.sampler import SamplerOutput
18
+from vllm.sequence import ExecuteModelRequest
19
from vllm.worker.openvino_model_runner import OpenVINOModelRunner
20
from vllm.worker.worker_base import LoraNotSupportedWorkerBase
21
vllm/worker/tpu_model_runner.py
+2
-2
@@ -14,11 +14,11 @@ from vllm.compilation.wrapper import TorchCompileWrapperWithCustomDispacther
14
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
15
ParallelConfig, SchedulerConfig)
16
from vllm.logger import init_logger
17
+from vllm.model_executor.layers.sampler import SamplerOutput
18
from vllm.model_executor.model_loader import get_model
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
from vllm.sequence import (CompletionSequenceGroupOutput, IntermediateTensors,
20
- Logprob, SamplerOutput, SequenceGroupMetadata,
21
- SequenceOutput)
21
+ Logprob, SequenceGroupMetadata, SequenceOutput)
22
from vllm.worker.model_runner_base import (
23
ModelRunnerBase, ModelRunnerInputBase,
24
_add_attn_metadata_broadcastable_dict,
vllm/worker/worker.py
+2
-2
@@ -17,12 +17,12 @@ from vllm.distributed import (ensure_model_parallel_initialized,
17
from vllm.logger import init_logger
18
from vllm.lora.request import LoRARequest
19
from vllm.model_executor import set_random_seed
20
+from vllm.model_executor.layers.sampler import SamplerOutput
21
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
22
from vllm.platforms import current_platform
23
from vllm.prompt_adapter.request import PromptAdapterRequest
24
from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
24
- SamplerOutput, SequenceGroupMetadata,
25
- SequenceGroupMetadataDelta)
25
+ SequenceGroupMetadata, SequenceGroupMetadataDelta)
26
from vllm.worker.cache_engine import CacheEngine
27
from vllm.worker.embedding_model_runner import EmbeddingModelRunner
28
from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
vllm/worker/worker_base.py
+2
-2
@@ -11,9 +11,9 @@ from vllm.config import ObservabilityConfig
11
from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
12
from vllm.logger import init_logger
13
from vllm.lora.request import LoRARequest
14
+from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.platforms import current_platform
15
-from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
16
- SamplerOutput)
16
+from vllm.sequence import ExecuteModelRequest, IntermediateTensors
17
from vllm.utils import (enable_trace_function_call_for_thread,
18
update_environment_variables)
19
from vllm.worker.model_runner_base import (BroadcastableModelInput,
vllm/worker/xpu_model_runner.py
+2
-2
@@ -15,12 +15,12 @@ from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
from vllm.distributed import get_pp_group
16
from vllm.inputs import INPUT_REGISTRY, InputRegistry
17
from vllm.logger import init_logger
18
+from vllm.model_executor.layers.sampler import SamplerOutput
19
from vllm.model_executor.model_loader import get_model
20
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
21
MultiModalInputs, MultiModalRegistry)
22
from vllm.sampling_params import SamplingParams
22
-from vllm.sequence import (IntermediateTensors, SamplerOutput,
23
- SequenceGroupMetadata)
23
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
24
from vllm.utils import CudaMemoryProfiler, make_tensor_with_pad
25
from vllm.worker.model_runner import AttentionMetadata, SamplingMetadata
26
from vllm.worker.model_runner_base import (