@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5a2d420c17

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 16:00 UTC

[Bugfix] Use dedicated MM processor cache in /tokenize to prevent sender-cache pollution (#38545)

Signed-off-by: Sergey Zinchenko <sergey.zinchenko.rnd@gmail.com>

Sergey Zinchenko committed Apr 2, 2026 at 07:14 UTC 5a2d420c175653c05ff9273bbda95eb23cdb4155

4 files changed +187 -19

tests/entrypoints/serve/tokenize/test_tokenize_then_chat_vlm.py new

+85

@@ -0,0 +1,85 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +Regression test: calling ``/tokenize`` with multimodal data followed by

5 +``/v1/chat/completions`` with the same data must not cause an error.

6 +

7 +Ensures that the ``/tokenize`` endpoint does not pollute internal caches

8 +(e.g. multimodal feature caches) and that a subsequent

9 +``/v1/chat/completions`` request with the same multimodal payload

10 +completes successfully.

11 +"""

12 +

13 +import json

14 +

15 +import openai

16 +import pytest

17 +import pytest_asyncio

18 +import requests

19 +

20 +from tests.utils import RemoteOpenAIServer

21 +

22 +MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct"

23 +

24 +

25 +@pytest.fixture(scope="module")

26 +def server():

27 + args = [

28 + "--dtype",

29 + "bfloat16",

30 + "--max-model-len",

31 + "4096",

32 + "--max-num-seqs",

33 + "5",

34 + "--enforce-eager",

35 + "--limit-mm-per-prompt",

36 + json.dumps({"image": 1}),

37 + ]

38 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

39 + yield remote_server

40 +

41 +

42 +@pytest_asyncio.fixture

43 +async def client(server):

44 + async with server.get_async_client() as async_client:

45 + yield async_client

46 +

47 +

48 +@pytest.mark.asyncio

49 +async def test_tokenize_then_chat_completion_with_image(

50 + client: openai.AsyncOpenAI,

51 + server: RemoteOpenAIServer,

52 + local_asset_server,

53 +):

54 + """Tokenize a multimodal message, then send the same message to chat

55 + completions. The chat completion must succeed (not 500)."""

56 +

57 + image_url = local_asset_server.url_for("stop_sign.jpg")

58 + messages = [

59 + {

60 + "role": "user",

61 + "content": [

62 + {"type": "image_url", "image_url": {"url": image_url}},

63 + {"type": "text", "text": "Describe this image briefly."},

64 + ],

65 + }

66 + ]

67 +

68 + tok_resp = requests.post(

69 + server.url_for("tokenize"),

70 + json={"model": MODEL_NAME, "messages": messages},

71 + )

72 + tok_resp.raise_for_status()

73 + tok_data = tok_resp.json()

74 + assert tok_data["count"] > 0, "Tokenization must return tokens"

75 +

76 + chat_completion = await client.chat.completions.create(

77 + model=MODEL_NAME,

78 + messages=messages,

79 + max_tokens=10,

80 + temperature=0.0,

81 + )

82 +

83 + assert chat_completion.choices[0].message.content, (

84 + "Chat completion must produce non-empty content after tokenize"

85 + )

vllm/entrypoints/serve/render/serving.py

+9 -1

@@ -451,6 +451,8 @@ class OpenAIServingRender:

451 request: Any,

452 prompt_input: str | list[str] | list[int] | list[list[int]] | None,

453 prompt_embeds: bytes | list[bytes] | None,

454 + *,

455 + skip_mm_cache: bool = False,

456 ) -> list[EngineInput]:

457 """Copied from OpenAIServing._preprocess_completion."""

458 prompts = list[SingletonPrompt | bytes]()

@@ -458,12 +460,14 @@ class OpenAIServingRender:

460 prompts.extend(prompt_to_seq(prompt_embeds))

461 if prompt_input is not None:

462 prompts.extend(prompt_to_seq(prompt_input))

461 - return await self.preprocess_cmpl(request, prompts)

463 + return await self.preprocess_cmpl(request, prompts, skip_mm_cache=skip_mm_cache)

464

465 async def preprocess_cmpl(

466 self,

467 request: Any,

468 prompts: Sequence[PromptType | bytes],

469 + *,

470 + skip_mm_cache: bool = False,

471 ) -> list[EngineInput]:

472 """Copied from OpenAIServing._preprocess_cmpl."""

473 renderer = self.renderer

@@ -487,6 +491,7 @@ class OpenAIServingRender:

491 for k in ("mm_processor_kwargs", "cache_salt")

492 if (v := getattr(request, k, None)) is not None

493 },

494 + skip_mm_cache=skip_mm_cache,

495 )

496

497 async def preprocess_chat(

@@ -498,6 +503,8 @@ class OpenAIServingRender:

503 default_template_kwargs: dict[str, Any] | None,

504 tool_dicts: list[dict[str, Any]] | None = None,

505 tool_parser: type[ToolParser] | None = None,

506 + *,

507 + skip_mm_cache: bool = False,

508 ) -> tuple[list[ConversationMessage], list[EngineInput]]:

509 """Copied from OpenAIServing._preprocess_chat."""

510 renderer = self.renderer

@@ -529,6 +536,7 @@ class OpenAIServingRender:

536 for k in ("mm_processor_kwargs", "cache_salt")

537 if (v := getattr(request, k, None)) is not None

538 },

539 + skip_mm_cache=skip_mm_cache,

540 )

541

542 # tool parsing is done only if a tool_parser has been set and if

vllm/entrypoints/serve/tokenize/serving.py

@@ -86,12 +86,14 @@ class OpenAIServingTokenization(OpenAIServing):

86 default_template_content_format=self.chat_template_content_format,

87 default_template_kwargs=self.default_chat_template_kwargs,

88 tool_dicts=tool_dicts,

89 + skip_mm_cache=True,

90 )

91 else:

92 engine_inputs = await self.openai_serving_render.preprocess_completion(

93 request,

94 prompt_input=request.prompt,

95 prompt_embeds=None,

96 + skip_mm_cache=True,

97 )

99 input_ids: list[int] = []

vllm/renderers/base.py

+91 -18

@@ -97,6 +97,7 @@ class BaseRenderer(ABC, Generic[_T]):

97 self._async_tokenizer: AsyncMicrobatchTokenizer | None = None

99 self.mm_processor: BaseMultiModalProcessor | None = None

100 + self._readonly_mm_processor: BaseMultiModalProcessor | None = None

101 self._mm_cache_stats: MultiModalCacheStats | None = None

102 self._clear_mm_cache_async = make_async(

103 self.clear_mm_cache, executor=self._executor

@@ -124,6 +125,19 @@ class BaseRenderer(ABC, Generic[_T]):

125 if mm_processor_cache:

126 self._mm_cache_stats = MultiModalCacheStats()

127

128 + # A second processor with its own processor-only cache.

129 + # Used by the tokenize endpoint so that tokenize-only

130 + # requests don't pollute the sender cache.

131 + ro_cache = mm_registry.processor_only_cache_from_config(config)

132 + if ro_cache is not None:

133 + ro_tokenizer = copy.deepcopy(tokenizer)

134 + with set_default_torch_num_threads():

135 + self._readonly_mm_processor = mm_registry.create_processor(

136 + config.model_config,

137 + tokenizer=ro_tokenizer,

138 + cache=ro_cache,

139 + )

140 +

141 # This is used to generate internal request ID for MM processing

142 # It has no relation to the request ID for engine core

143 self._mm_req_counter = AtomicCounter()

@@ -625,10 +639,15 @@ class BaseRenderer(ABC, Generic[_T]):

639 mm_uuids: MultiModalUUIDDict | None,

640 mm_processor_kwargs: Mapping[str, object] | None,

641 tokenization_kwargs: dict[str, Any] | None,

642 + *,

643 + skip_mm_cache: bool = False,

644 ) -> "MultiModalInput":

645 mm_req_id = f"renderer{self.api_process_rank}-mm-{self._mm_req_counter.inc(1)}"

646

631 - mm_processor = self.get_mm_processor()

647 + if skip_mm_cache and self._readonly_mm_processor is not None:

648 + mm_processor = self._readonly_mm_processor

649 + else:

650 + mm_processor = self.get_mm_processor()

651

652 mm_data_items = mm_processor.info.parse_mm_data(mm_data)

653 mm_uuid_items = parse_mm_uuids(mm_uuids)

@@ -656,6 +675,8 @@ class BaseRenderer(ABC, Generic[_T]):

675 def _process_tokens(

676 self,

677 prompt: TokensPrompt,

678 + *,

679 + skip_mm_cache: bool = False,

680 ) -> TokensInput | MultiModalInput:

681 """Process token inputs, with multimodal preprocessing offloaded

682 to the shared thread pool in the async variant.

@@ -670,6 +691,7 @@ class BaseRenderer(ABC, Generic[_T]):

691 mm_processor_kwargs=prompt.get("mm_processor_kwargs"),

692 tokenization_kwargs=None, # Tokenization already done in Step 2

693 mm_uuids=prompt.get("multi_modal_uuids"),

694 + skip_mm_cache=skip_mm_cache,

695 )

696 else:

697 engine_input = tokens_input(prompt_token_ids)

@@ -712,6 +734,8 @@ class BaseRenderer(ABC, Generic[_T]):

734 async def _process_tokens_async(

735 self,

736 prompt: TokensPrompt,

737 + *,

738 + skip_mm_cache: bool = False,

739 ) -> TokensInput | MultiModalInput:

740 prompt_token_ids = prompt["prompt_token_ids"]

741

@@ -723,6 +747,7 @@ class BaseRenderer(ABC, Generic[_T]):

747 mm_processor_kwargs=prompt.get("mm_processor_kwargs"),

748 tokenization_kwargs=None,

749 mm_uuids=prompt.get("multi_modal_uuids"),

750 + skip_mm_cache=skip_mm_cache,

751 )

752 else:

753 engine_input = tokens_input(prompt_token_ids)

@@ -734,24 +759,33 @@ class BaseRenderer(ABC, Generic[_T]):

759

760 return engine_input

761

737 - def _process_singleton(self, prompt: SingletonTokPrompt) -> SingletonInput:

762 + def _process_singleton(

763 + self,

764 + prompt: SingletonTokPrompt,

765 + *,

766 + skip_mm_cache: bool = False,

767 + ) -> SingletonInput:

768 if "prompt_embeds" in prompt:

769 return self._process_embeds(prompt) # type: ignore[arg-type]

770

741 - return self._process_tokens(prompt) # type: ignore[arg-type]

771 + return self._process_tokens(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]

772

773 async def _process_singleton_async(

774 self,

775 prompt: SingletonTokPrompt,

776 + *,

777 + skip_mm_cache: bool = False,

778 ) -> SingletonInput:

779 if "prompt_embeds" in prompt:

780 return self._process_embeds(prompt) # type: ignore[arg-type]

781

750 - return await self._process_tokens_async(prompt) # type: ignore[arg-type]

782 + return await self._process_tokens_async(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]

783

784 def _process_enc_dec(

785 self,

786 prompt: EncoderDecoderTokPrompt,

787 + *,

788 + skip_mm_cache: bool = False,

789 ) -> EncoderDecoderInput:

790 enc_prompt = prompt["encoder_prompt"]

791 dec_prompt = prompt["decoder_prompt"]

@@ -764,9 +798,13 @@ class BaseRenderer(ABC, Generic[_T]):

798 skip_decoder_start_token = self.mm_processor.skip_decoder_start_token

799

800 return build_enc_dec_input(

767 - encoder_input=self._process_singleton(enc_prompt),

801 + encoder_input=self._process_singleton(

802 + enc_prompt, skip_mm_cache=skip_mm_cache

803 + ),

804 decoder_input=(

769 - None if dec_prompt is None else self._process_singleton(dec_prompt)

805 + None

806 + if dec_prompt is None

807 + else self._process_singleton(dec_prompt, skip_mm_cache=skip_mm_cache)

808 ),

809 decoder_start_token_id=self.get_dec_start_token_id(),

810 skip_decoder_start_token=skip_decoder_start_token,

@@ -775,16 +813,20 @@ class BaseRenderer(ABC, Generic[_T]):

813 async def _process_enc_dec_async(

814 self,

815 prompt: EncoderDecoderTokPrompt,

816 + *,

817 + skip_mm_cache: bool = False,

818 ) -> EncoderDecoderInput:

819 enc_prompt = prompt["encoder_prompt"]

820 dec_prompt = prompt["decoder_prompt"]

821

822 encoder_input, decoder_input = await asyncio.gather(

783 - self._process_singleton_async(enc_prompt),

823 + self._process_singleton_async(enc_prompt, skip_mm_cache=skip_mm_cache),

824 (

825 asyncio.sleep(0)

826 if dec_prompt is None

787 - else self._process_singleton_async(dec_prompt)

827 + else self._process_singleton_async(

828 + dec_prompt, skip_mm_cache=skip_mm_cache

829 + )

830 ),

831 )

832

@@ -794,27 +836,40 @@ class BaseRenderer(ABC, Generic[_T]):

836 decoder_start_token_id=self.get_dec_start_token_id(),

837 )

838

797 - def process_for_engine(self, prompt: TokPrompt, arrival_time: float) -> EngineInput:

839 + def process_for_engine(

840 + self,

841 + prompt: TokPrompt,

842 + arrival_time: float,

843 + *,

844 + skip_mm_cache: bool = False,

845 + ) -> EngineInput:

846 engine_input: EngineInput

847 if "encoder_prompt" in prompt:

800 - engine_input = self._process_enc_dec(prompt) # type: ignore[arg-type]

848 + engine_input = self._process_enc_dec(prompt, skip_mm_cache=skip_mm_cache) # type: ignore[arg-type]

849 else:

802 - engine_input = self._process_singleton(prompt)

850 + engine_input = self._process_singleton(prompt, skip_mm_cache=skip_mm_cache)

851

852 engine_input["arrival_time"] = arrival_time

853

854 return engine_input

855

856 async def process_for_engine_async(

809 - self, prompt: TokPrompt, arrival_time: float

857 + self,

858 + prompt: TokPrompt,

859 + arrival_time: float,

860 + *,

861 + skip_mm_cache: bool = False,

862 ) -> EngineInput:

863 engine_input: EngineInput

864 if "encoder_prompt" in prompt:

865 engine_input = await self._process_enc_dec_async(

814 - prompt # type: ignore[arg-type]

866 + prompt, # type: ignore[arg-type]

867 + skip_mm_cache=skip_mm_cache,

868 )

869 else:

817 - engine_input = await self._process_singleton_async(prompt)

870 + engine_input = await self._process_singleton_async(

871 + prompt, skip_mm_cache=skip_mm_cache

872 + )

873

874 engine_input["arrival_time"] = arrival_time

875

@@ -827,6 +882,7 @@ class BaseRenderer(ABC, Generic[_T]):

882 tok_params: TokenizeParams | None = None,

883 *,

884 prompt_extras: dict[str, Any] | None = None,

885 + skip_mm_cache: bool = False,

886 ):

887 arrival_time = time.time()

888

@@ -838,7 +894,10 @@ class BaseRenderer(ABC, Generic[_T]):

894

895 self._apply_prompt_extras(tok_prompts, prompt_extras)

896

841 - return [self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts]

897 + return [

898 + self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)

899 + for prompt in tok_prompts

900 + ]

901

902 async def render_cmpl_async(

903 self,

@@ -846,6 +905,7 @@ class BaseRenderer(ABC, Generic[_T]):

905 tok_params: TokenizeParams | None = None,

906 *,

907 prompt_extras: dict[str, Any] | None = None,

908 + skip_mm_cache: bool = False,

909 ):

910 arrival_time = time.time()

911

@@ -858,7 +918,12 @@ class BaseRenderer(ABC, Generic[_T]):

918 self._apply_prompt_extras(tok_prompts, prompt_extras)

919

920 return await asyncio.gather(

861 - *(self.process_for_engine_async(p, arrival_time) for p in tok_prompts)

921 + *(

922 + self.process_for_engine_async(

923 + p, arrival_time, skip_mm_cache=skip_mm_cache

924 + )

925 + for p in tok_prompts

926 + )

927 )

928

929 def render_chat(

@@ -868,6 +933,7 @@ class BaseRenderer(ABC, Generic[_T]):

933 tok_params: TokenizeParams | None = None,

934 *,

935 prompt_extras: dict[str, Any] | None = None,

936 + skip_mm_cache: bool = False,

937 ):

938 arrival_time = time.time()

939

@@ -890,7 +956,8 @@ class BaseRenderer(ABC, Generic[_T]):

956 self._apply_prompt_extras(tok_prompts, prompt_extras)

957

958 eng_prompts = [

893 - self.process_for_engine(prompt, arrival_time) for prompt in tok_prompts

959 + self.process_for_engine(prompt, arrival_time, skip_mm_cache=skip_mm_cache)

960 + for prompt in tok_prompts

961 ]

962

963 return out_conversations, eng_prompts

@@ -902,6 +969,7 @@ class BaseRenderer(ABC, Generic[_T]):

969 tok_params: TokenizeParams | None = None,

970 *,

971 prompt_extras: dict[str, Any] | None = None,

972 + skip_mm_cache: bool = False,

973 ):

974 arrival_time = time.time()

975

@@ -924,7 +992,12 @@ class BaseRenderer(ABC, Generic[_T]):

992 self._apply_prompt_extras(tok_prompts, prompt_extras)

993

994 eng_prompts = await asyncio.gather(

927 - *(self.process_for_engine_async(p, arrival_time) for p in tok_prompts)

995 + *(

996 + self.process_for_engine_async(

997 + p, arrival_time, skip_mm_cache=skip_mm_cache

998 + )

999 + for p in tok_prompts

1000 + )

1001 )

1002

1003 return out_conversations, eng_prompts