@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86647d1cd0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[V0 Deprecation] Remove V0 Output Processor (#25320)

Signed-off-by: Woosuk Kwon <woosuk@thinkingmachines.ai> Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Sep 20, 2025 at 17:57 UTC 86647d1cd0f3c82c7d678324db7e925654ac5665
6 files changed +40 -384
tests/build_cython.py deleted
-39
@@ -1,39 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import Cython.Compiler.Options
4 -from Cython.Build import cythonize
5 -from setuptools import setup
6 -
7 -Cython.Compiler.Options.annotate = True
8 -
9 -infiles = []
10 -
11 -infiles += [
12 - "vllm/engine/llm_engine.py",
13 - "vllm/transformers_utils/detokenizer.py",
14 - "vllm/engine/output_processor/single_step.py",
15 - "vllm/outputs.py",
16 - "vllm/engine/output_processor/stop_checker.py",
17 -]
18 -
19 -infiles += [
20 - "vllm/core/scheduler.py",
21 - "vllm/sequence.py",
22 - "vllm/core/block_manager.py",
23 -]
24 -
25 -infiles += [
26 - "vllm/model_executor/layers/sampler.py",
27 - "vllm/sampling_params.py",
28 - "vllm/utils/__init__.py",
29 -]
30 -
31 -setup(ext_modules=cythonize(infiles,
32 - annotate=False,
33 - force=True,
34 - compiler_directives={
35 - 'language_level': "3",
36 - 'infer_types': True
37 - }))
38 -
39 -# example usage: python3 build_cython.py build_ext --inplace
vllm/engine/output_processor/__init__.py
vllm/engine/output_processor/interfaces.py deleted
-59
@@ -1,59 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from abc import ABC, abstractmethod
5 -from typing import List
6 -
7 -from vllm.config import SchedulerConfig
8 -from vllm.core.scheduler import Scheduler
9 -from vllm.engine.output_processor.stop_checker import StopChecker
10 -from vllm.sequence import SequenceGroup, SequenceGroupOutput
11 -from vllm.transformers_utils.detokenizer import Detokenizer
12 -from vllm.utils import Counter
13 -
14 -
15 -class SequenceGroupOutputProcessor(ABC):
16 - """Interface for logic that processes new token ids in sequence groups,
17 - managing detokenization, stop checking, and freeing/forking sequences with
18 - the scheduler.
19 -
20 - This is highly coupled with the LLMEngine and should be seen as an extension
21 - of it. The logic is separated to simplify the LLMEngine class and allow
22 - separate implementations for single-step decoding (which supports beam
23 - search sequence forking) and multi-step decoding (which does not support
24 - beam search, but does support speculative decoding).
25 - """
26 -
27 - @staticmethod
28 - def create_output_processor(
29 - scheduler_config: SchedulerConfig,
30 - detokenizer: Detokenizer,
31 - scheduler: List[Scheduler],
32 - seq_counter: Counter,
33 - stop_checker: "StopChecker",
34 - ):
35 - """Create an output processor.
36 -
37 - Multi-step scheduling is no longer supported. Always return a
38 - single-step output processor.
39 - """
40 - from vllm.engine.output_processor.single_step import (
41 - SingleStepOutputProcessor)
42 - return SingleStepOutputProcessor(scheduler_config, detokenizer,
43 - scheduler, seq_counter, stop_checker)
44 -
45 - @abstractmethod
46 - def process_outputs(self, sequence_group: SequenceGroup,
47 - outputs: List[SequenceGroupOutput],
48 - is_async: bool) -> None:
49 - """Process new token ids for the sequence group. Handles logic such as
50 - detokenization, stop checking, and freeing/forking sequences in the
51 - scheduler.
52 - """
53 - pass
54 -
55 - @abstractmethod
56 - def process_prompt_logprob(self, seq_group: SequenceGroup,
57 - outputs: List[SequenceGroupOutput]) -> None:
58 - """Update prompt logprobs received from outputs to seq_group."""
59 - pass
vllm/engine/output_processor/single_step.py deleted
-145
@@ -1,145 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from typing import List
5 -
6 -from vllm.config import SchedulerConfig
7 -from vllm.core.scheduler import Scheduler
8 -from vllm.engine.output_processor.interfaces import (
9 - SequenceGroupOutputProcessor)
10 -from vllm.engine.output_processor.stop_checker import StopChecker
11 -from vllm.logger import init_logger
12 -from vllm.sequence import (CompletionSequenceGroupOutput, SequenceGroup,
13 - SequenceGroupOutput)
14 -from vllm.transformers_utils.detokenizer import Detokenizer
15 -from vllm.utils import Counter
16 -
17 -logger = init_logger(__name__)
18 -
19 -
20 -def single_step_process_prompt_logprob(
21 - sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
22 - output: CompletionSequenceGroupOutput) -> None:
23 - """Process prompt logprobs associated with the
24 - [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput] for a given step.
25 -
26 - Do nothing if the output has no prompt logprobs.
27 -
28 - Account for the fact that transformers do not compute first-token logprobs.
29 -
30 - Args:
31 - sg_output_proc:
32 - [`SequenceGroupOutputProcessor`][vllm.engine.output_processor.interfaces.SequenceGroupOutputProcessor]
33 - instance
34 - seq_group: the output is associated with this
35 - [`SequenceGroup`][vllm.sequence.SequenceGroup]
36 - output: the [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput]
37 - for a single scheduler step
38 - """
39 - prompt_logprobs = output.prompt_logprobs
40 -
41 - # If this is the first (or only) "chunk" of the prefill, we need
42 - # to prepend None to the list of prompt logprobs. The reason for this
43 - # is that for N prompt tokens, the Sampler will generate N-1 total
44 - # prompt logprobs during prefill since the token at idx 0 will not
45 - # have a logprob associated with it.
46 - if prompt_logprobs is not None:
47 - if not seq_group.prompt_logprobs:
48 - prompt_logprobs = [None] + prompt_logprobs
49 - seq_group.prompt_logprobs = []
50 -
51 - assert hasattr(sg_output_proc, 'detokenizer')
52 - if (seq_group.sampling_params.detokenize
53 - and sg_output_proc.detokenizer):
54 - sg_output_proc.detokenizer.decode_prompt_logprobs_inplace(
55 - seq_group,
56 - prompt_logprobs,
57 - position_offset=len(seq_group.prompt_logprobs))
58 -
59 - seq_group.prompt_logprobs.extend(prompt_logprobs)
60 -
61 -
62 -class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
63 - """SequenceGroupOutputProcessor which handles "output processing" logic,
64 - which happens after the model returns generated token ids and before
65 - scheduling of the next batch. Output processing logic includes
66 - detokenization, and determining if a sequence is finished (e.g. via max len
67 - or eos token).
68 -
69 - The SingleStepOutputProcessor is specialized to the case where the model
70 - emits at most a single token per invocation, which precludes configurations
71 - such as speculative decoding or multi-step decoding. This enables beam
72 - search sampling, which requires forking/finishing/freeing sequences in a way
73 - that is currently difficult to schedule multiple steps ahead of time.
74 - """
75 -
76 - def __init__(self, scheduler_config: SchedulerConfig,
77 - detokenizer: Detokenizer, scheduler: List[Scheduler],
78 - seq_counter: Counter, stop_checker: StopChecker):
79 - self.scheduler_config = scheduler_config
80 - self.detokenizer = detokenizer
81 - self.scheduler = scheduler
82 - self.seq_counter = seq_counter
83 - self.stop_checker = stop_checker
84 -
85 - def process_outputs(self, sequence_group: SequenceGroup,
86 - outputs: List[SequenceGroupOutput],
87 - is_async: bool) -> None:
88 - """Append all new tokens to sequences in the sequence group. Fork any
89 - surviving beam candidates; free any unsurviving ones.
90 -
91 - Invokes detokenizer to detokenize new tokens, and also marks sequences
92 - as finished if they meet stop conditions.
93 -
94 - is_async - Indicates whether this postprocessor runs in
95 - parallel with the GPU forward pass and is processing
96 - tokens from the previous step. If this is true, then
97 - no tokens need to be appended since it is already done
98 - externally (before the next schedule() call)
99 - """
100 - assert (len(outputs) == 1
101 - ), f"{type(self)} does not support multiple outputs per step"
102 - return self._process_sequence_group_outputs(sequence_group, outputs[0],
103 - is_async)
104 -
105 - def process_prompt_logprob(self, seq_group: SequenceGroup,
106 - outputs: List[SequenceGroupOutput]) -> None:
107 - """Process prompt logprobs associated with one step of a single-step-
108 - scheduled computation.
109 -
110 - Args:
111 - seq_group: the output is associated with this
112 - [`SequenceGroup`][vllm.sequence.SequenceGroup]
113 - outputs: the
114 - [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput]
115 - for a single scheduler step
116 - """
117 - assert len(outputs) == 1, "Single step should only have 1 output."
118 - output = outputs[0]
119 - assert isinstance(output, CompletionSequenceGroupOutput)
120 - single_step_process_prompt_logprob(self, seq_group, output)
121 -
122 - def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
123 - outputs: SequenceGroupOutput,
124 - is_async: bool) -> None:
125 - sampling_params = seq_group.sampling_params
126 -
127 - sample = outputs.samples[0]
128 - seq = seq_group.first_seq
129 - if not is_async:
130 - seq.append_token_id(sample.output_token, sample.logprobs,
131 - sample.output_embed)
132 - if sampling_params.detokenize and self.detokenizer:
133 - new_char_count = self.detokenizer.decode_sequence_inplace(
134 - seq, sampling_params)
135 - else:
136 - new_char_count = 0
137 - self.stop_checker.maybe_stop_sequence(
138 - seq,
139 - new_char_count,
140 - sampling_params,
141 - lora_req=seq_group.lora_request,
142 - )
143 - if seq.is_finished():
144 - for scheduler in self.scheduler:
145 - scheduler.free_seq(seq)
vllm/engine/output_processor/stop_checker.py deleted
-139
@@ -1,139 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -from typing import List, Optional, Tuple
5 -
6 -from vllm.lora.request import LoRARequest
7 -from vllm.reasoning import ReasoningParser
8 -from vllm.sampling_params import SamplingParams
9 -from vllm.sequence import Sequence, SequenceStatus
10 -
11 -
12 -class StopChecker:
13 - """LLMEngine helper class which separates out the logic involving stop
14 - checking. This checks things such as: whether the eos token was emitted,
15 - whether the max_tokens has been consumed, whether a stop string has been
16 - emitted, or if we have exceeded the max model len.
17 - """
18 -
19 - def __init__(
20 - self,
21 - max_model_len: int,
22 - reasoner: Optional[ReasoningParser] = None,
23 - ):
24 - # Do not use it directly, but use `self._get_max_model_len`.
25 - self._max_model_len = max_model_len
26 - self.reasoner = reasoner
27 -
28 - def _get_max_model_len(self, lora_req: Optional[LoRARequest]):
29 - if lora_req and lora_req.long_lora_max_len:
30 - return lora_req.long_lora_max_len
31 - else:
32 - return self._max_model_len
33 -
34 - def maybe_stop_sequence(
35 - self,
36 - seq: Sequence,
37 - new_char_count: int,
38 - sampling_params: SamplingParams,
39 - lora_req: Optional[LoRARequest] = None,
40 - ) -> None:
41 - """Stop the finished sequences.
42 -
43 - new_char_count is the number of chars added to the
44 - sequence's output text for the newly generated token
45 - """
46 -
47 - # Check if the minimum number of tokens has been generated yet;
48 - # skip the stop string/token checks if not
49 - if seq.get_output_len() < sampling_params.min_tokens:
50 - return
51 -
52 - # Check if the sequence has generated the EOS token.
53 - if ((not sampling_params.ignore_eos)
54 - and seq.get_last_token_id() == seq.eos_token_id):
55 - # Remove the last EOS token unless explicitly specified
56 - # This prevents unintended exposure of the EOS token
57 - if new_char_count and (
58 - not sampling_params.include_stop_str_in_output):
59 - seq.output_text = seq.output_text[:-new_char_count]
60 - seq.status = SequenceStatus.FINISHED_STOPPED
61 - return
62 -
63 - # Skip stop string/token checks if in reasoning content generation
64 - if self.reasoner is not None and \
65 - not self.reasoner.is_reasoning_end(seq.get_token_ids()):
66 - return
67 -
68 - # Check if a stop token was encountered.
69 - # This assumes a single token produced per step.
70 - last_token_id = seq.get_last_token_id()
71 - if last_token_id in (sampling_params.stop_token_ids or ()):
72 - if new_char_count and (
73 - not sampling_params.include_stop_str_in_output):
74 - # Remove last token
75 - seq.output_text = seq.output_text[:-new_char_count]
76 - seq.status = SequenceStatus.FINISHED_STOPPED
77 - seq.stop_reason = last_token_id
78 - return
79 -
80 - # Check if any stop strings are matched.
81 - stop = self.check_stop_strings(
82 - seq.output_text, new_char_count, sampling_params.stop,
83 - sampling_params.include_stop_str_in_output)
84 - if stop is not None:
85 - stop_str, truncate_to = stop
86 - if truncate_to != -1:
87 - seq.output_text = seq.output_text[:truncate_to]
88 - seq.status = SequenceStatus.FINISHED_STOPPED
89 - seq.stop_reason = stop_str
90 - return
91 -
92 - # Check if the sequence has reached max_model_len.
93 - if seq.get_len() >= self._get_max_model_len(lora_req):
94 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
95 - return
96 -
97 - # Check if the sequence has reached max_tokens.
98 - if seq.get_output_len() == sampling_params.max_tokens:
99 - seq.status = SequenceStatus.FINISHED_LENGTH_CAPPED
100 - return
101 -
102 - @staticmethod
103 - def check_stop_strings(
104 - output_text: str,
105 - new_char_count: int,
106 - stop: List[str],
107 - include_in_output: bool,
108 - ) -> Optional[Tuple[str, int]]:
109 - """Check if any stop strings are matched and truncate sequence
110 - output text accordingly.
111 -
112 - Returns tuple (stop_string, offset) if matched or else None.
113 -
114 - Where stop_string is the matched stop string and offset is the
115 - length to which output_text should be truncated, or -1 for no
116 - truncation.
117 - """
118 - if not new_char_count or not stop:
119 - return None
120 -
121 - for stop_str in stop:
122 - stop_string_len = len(stop_str)
123 - # Avoid searching already-searched text.
124 - stop_index = output_text.find(stop_str,
125 - 1 - new_char_count - stop_string_len)
126 - if stop_index == -1:
127 - continue
128 -
129 - if include_in_output:
130 - # Truncate to end of stop string.
131 - stop_index += stop_string_len
132 - if stop_index >= len(output_text):
133 - # No truncation required.
134 - return stop_str, -1
135 -
136 - # Truncate the output text to either the beginning
137 - # or end of the stop string.
138 - return stop_str, stop_index
139 - return None
vllm/v1/engine/detokenizer.py
+40 -2
@@ -9,7 +9,6 @@ from tokenizers import Tokenizer
9 from tokenizers.decoders import DecodeStream
10 from transformers import PreTrainedTokenizerFast
11
12 -from vllm.engine.output_processor.stop_checker import StopChecker
12 from vllm.logger import init_logger
13 from vllm.transformers_utils.detokenizer_utils import (
14 AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
@@ -129,7 +128,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC):
128 # 2) Evaluate stop strings.
129 stop_string = None
130 if self.stop and len(self.output_token_ids) > self.min_tokens:
132 - stop = StopChecker.check_stop_strings(
131 + stop = check_stop_strings(
132 output_text=self.output_text,
133 new_char_count=len(self.output_text) - stop_check_offset,
134 stop=self.stop,
@@ -309,3 +308,42 @@ class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
308 self.read_offset = read_offset
309
310 return decoded_text
311 +
312 +
313 +def check_stop_strings(
314 + output_text: str,
315 + new_char_count: int,
316 + stop: list[str],
317 + include_in_output: bool,
318 +) -> Optional[tuple[str, int]]:
319 + """Check if any stop strings are matched and truncate sequence
320 + output text accordingly.
321 +
322 + Returns tuple (stop_string, offset) if matched or else None.
323 +
324 + Where stop_string is the matched stop string and offset is the
325 + length to which output_text should be truncated, or -1 for no
326 + truncation.
327 + """
328 + if not new_char_count or not stop:
329 + return None
330 +
331 + for stop_str in stop:
332 + stop_string_len = len(stop_str)
333 + # Avoid searching already-searched text.
334 + stop_index = output_text.find(stop_str,
335 + 1 - new_char_count - stop_string_len)
336 + if stop_index == -1:
337 + continue
338 +
339 + if include_in_output:
340 + # Truncate to end of stop string.
341 + stop_index += stop_string_len
342 + if stop_index >= len(output_text):
343 + # No truncation required.
344 + return stop_str, -1
345 +
346 + # Truncate the output text to either the beginning
347 + # or end of the stop string.
348 + return stop_str, stop_index
349 + return None