A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Chore][V0 Deprecation] Move LogProb to a separate file (#24055)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Sep 1, 2025 at 12:07 UTC
5685370271d7f3e8222e26efb854e72e826b9af7
14 files changed
+49
-42
vllm/beam_search.py
+1
-1
@@ -4,8 +4,8 @@
4
from dataclasses import dataclass
5
from typing import TYPE_CHECKING, Any, Optional, Union
6
7
+from vllm.logprobs import Logprob
8
from vllm.lora.request import LoRARequest
8
-from vllm.sequence import Logprob
9
10
if TYPE_CHECKING:
11
from vllm.multimodal import MultiModalDataDict
vllm/entrypoints/openai/protocol.py
+1
-1
@@ -43,10 +43,10 @@ from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
43
from vllm.entrypoints.score_utils import (ScoreContentPartParam,
44
ScoreMultiModalParam)
45
from vllm.logger import init_logger
46
+from vllm.logprobs import Logprob
47
from vllm.pooling_params import PoolingParams
48
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
49
RequestOutputKind, SamplingParams)
49
-from vllm.sequence import Logprob
50
from vllm.utils import random_uuid, resolve_obj_by_qualname
51
52
logger = init_logger(__name__)
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -43,10 +43,10 @@ from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
43
from vllm.entrypoints.utils import get_max_tokens
44
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
45
from vllm.logger import init_logger
46
+from vllm.logprobs import Logprob
47
from vllm.outputs import CompletionOutput, RequestOutput
48
from vllm.reasoning import ReasoningParser, ReasoningParserManager
49
from vllm.sampling_params import BeamSearchParams, SamplingParams
49
-from vllm.sequence import Logprob
50
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
51
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
52
truncate_tool_call_ids,
vllm/entrypoints/openai/serving_completion.py
+1
-1
@@ -38,9 +38,9 @@ from vllm.entrypoints.utils import get_max_tokens
38
from vllm.inputs.data import (EmbedsPrompt, TokensPrompt, is_embeds_prompt,
39
is_tokens_prompt)
40
from vllm.logger import init_logger
41
+from vllm.logprobs import Logprob
42
from vllm.outputs import RequestOutput
43
from vllm.sampling_params import BeamSearchParams, SamplingParams
43
-from vllm.sequence import Logprob
44
from vllm.transformers_utils.tokenizer import AnyTokenizer
45
from vllm.utils import as_list, merge_async_iterators
46
vllm/entrypoints/openai/serving_engine.py
+1
-1
@@ -67,13 +67,13 @@ from vllm.inputs.data import EmbedsPrompt as EngineEmbedsPrompt
67
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
68
from vllm.inputs.parse import parse_and_batch_prompt
69
from vllm.logger import init_logger
70
+from vllm.logprobs import Logprob, PromptLogprobs
71
from vllm.lora.request import LoRARequest
72
from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin
73
MultiModalDataDict, MultiModalUUIDDict)
74
from vllm.outputs import PoolingRequestOutput, RequestOutput
75
from vllm.pooling_params import PoolingParams
76
from vllm.sampling_params import BeamSearchParams, SamplingParams
76
-from vllm.sequence import Logprob, PromptLogprobs
77
from vllm.tracing import (contains_trace_headers, extract_trace_headers,
78
log_tracing_disabled_warning)
79
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
vllm/entrypoints/openai/serving_responses.py
+2
-2
@@ -58,11 +58,11 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels
58
from vllm.entrypoints.tool_server import MCPToolServer, ToolServer
59
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
60
from vllm.logger import init_logger
61
+from vllm.logprobs import Logprob as SampleLogprob
62
+from vllm.logprobs import SampleLogprobs
63
from vllm.outputs import CompletionOutput
64
from vllm.reasoning import ReasoningParser, ReasoningParserManager
65
from vllm.sampling_params import SamplingParams
64
-from vllm.sequence import Logprob as SampleLogprob
65
-from vllm.sequence import SampleLogprobs
66
from vllm.transformers_utils.tokenizer import AnyTokenizer
67
from vllm.utils import random_uuid
68
vllm/logprobs.py
new
+28
@@ -0,0 +1,28 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from dataclasses import dataclass
4
+from typing import Optional
5
+
6
+
7
+# We use dataclass for now because it is used for
8
+# openai server output, and msgspec is not serializable.
9
+# TODO(sang): Fix it.
10
+@dataclass
11
+class Logprob:
12
+ """Infos for supporting OpenAI compatible logprobs and token ranks.
13
+
14
+ Attributes:
15
+ logprob: The logprob of chosen token
16
+ rank: The vocab rank of chosen token (>=1)
17
+ decoded_token: The decoded chosen token index
18
+ """
19
+ logprob: float
20
+ rank: Optional[int] = None
21
+ decoded_token: Optional[str] = None
22
+
23
+
24
+# {token_id -> logprob} per each sequence group. None if the corresponding
25
+# sequence group doesn't require prompt logprob.
26
+PromptLogprobs = list[Optional[dict[int, Logprob]]]
27
+# {token_id -> logprob} for each sequence group.
28
+SampleLogprobs = list[dict[int, Logprob]]
vllm/model_executor/layers/sampler.py
+2
-2
@@ -13,14 +13,14 @@ import torch
13
import torch.nn as nn
14
15
import vllm.envs as envs
16
+from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
17
from vllm.model_executor.layers.utils import apply_penalties
18
from vllm.model_executor.sampling_metadata import (SamplingMetadata,
19
SamplingTensors,
20
SequenceGroupToSample)
21
from vllm.sampling_params import SamplingType
22
from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
22
- CompletionSequenceGroupOutput, Logprob,
23
- PromptLogprobs, SampleLogprobs, SequenceOutput)
23
+ CompletionSequenceGroupOutput, SequenceOutput)
24
25
if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
26
# yapf: disable
vllm/model_executor/model_loader/neuron.py
+2
-2
@@ -14,12 +14,12 @@ from transformers import PretrainedConfig
14
15
from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
16
SpeculativeConfig)
17
+from vllm.logprobs import Logprob
18
from vllm.model_executor.layers.logits_processor import LogitsProcessor
19
from vllm.model_executor.layers.quantization import get_quantization_config
20
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
21
-from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
22
- SequenceOutput)
22
+from vllm.sequence import CompletionSequenceGroupOutput, SequenceOutput
23
24
TORCH_DTYPE_TO_NEURON_AMP = {
25
"auto": "f32",
vllm/model_executor/model_loader/neuronx_distributed.py
+2
-2
@@ -27,11 +27,11 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, PretrainedConfig
27
from vllm.config import (ModelConfig, ParallelConfig, SchedulerConfig,
28
SpeculativeConfig)
29
from vllm.logger import init_logger
30
+from vllm.logprobs import Logprob
31
from vllm.model_executor.layers.logits_processor import LogitsProcessor
32
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
-from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
34
- SequenceOutput)
34
+from vllm.sequence import CompletionSequenceGroupOutput, SequenceOutput
35
36
# yapf: enable
37
logger = init_logger(__name__)
vllm/outputs.py
+3
-2
@@ -11,11 +11,12 @@ import torch
11
from typing_extensions import TypeVar
12
13
from vllm.logger import init_logger
14
+from vllm.logprobs import PromptLogprobs, SampleLogprobs
15
from vllm.lora.request import LoRARequest
16
from vllm.multimodal.inputs import MultiModalPlaceholderDict
17
from vllm.sampling_params import RequestOutputKind
17
-from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
18
- SequenceGroup, SequenceGroupBase, SequenceStatus)
18
+from vllm.sequence import (RequestMetrics, SequenceGroup, SequenceGroupBase,
19
+ SequenceStatus)
20
21
logger = init_logger(__name__)
22
vllm/sequence.py
+1
-24
@@ -16,6 +16,7 @@ import msgspec
16
import torch
17
18
from vllm.inputs import SingletonInputs
19
+from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
20
from vllm.multimodal import MultiModalKwargs, MultiModalPlaceholderDict
21
from vllm.pooling_params import PoolingParams
22
from vllm.sampling_params import RequestOutputKind, SamplingParams
@@ -38,30 +39,6 @@ def array_full(token_id: int, count: int):
39
return array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
40
41
41
-# We use dataclass for now because it is used for
42
-# openai server output, and msgspec is not serializable.
43
-# TODO(sang): Fix it.
44
-@dataclass
45
-class Logprob:
46
- """Infos for supporting OpenAI compatible logprobs and token ranks.
47
-
48
- Attributes:
49
- logprob: The logprob of chosen token
50
- rank: The vocab rank of chosen token (>=1)
51
- decoded_token: The decoded chosen token index
52
- """
53
- logprob: float
54
- rank: Optional[int] = None
55
- decoded_token: Optional[str] = None
56
-
57
-
58
-# {token_id -> logprob} per each sequence group. None if the corresponding
59
-# sequence group doesn't require prompt logprob.
60
-PromptLogprobs = list[Optional[dict[int, Logprob]]]
61
-# {token_id -> logprob} for each sequence group.
62
-SampleLogprobs = list[dict[int, Logprob]]
63
-
64
-
42
class SequenceStatus(enum.IntEnum):
43
"""Status of a sequence."""
44
WAITING = 0
vllm/transformers_utils/detokenizer.py
+3
-2
@@ -3,8 +3,9 @@
3
4
from typing import Optional
5
6
-from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Logprob, SamplingParams,
7
- Sequence, SequenceGroup)
6
+from vllm.logprobs import Logprob
7
+from vllm.sequence import (VLLM_INVALID_TOKEN_ID, SamplingParams, Sequence,
8
+ SequenceGroup)
9
10
from .detokenizer_utils import (convert_prompt_ids_to_tokens,
11
detokenize_incrementally)
vllm/v1/engine/logprobs.py
+1
-1
@@ -7,7 +7,7 @@ from dataclasses import dataclass
7
from typing import Optional
8
9
from vllm.logger import init_logger
10
-from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
10
+from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
11
from vllm.transformers_utils.detokenizer_utils import (
12
AnyTokenizer, convert_ids_list_to_tokens)
13
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest