A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
fix: use cache_salt for gpt-oss (#23186)
Signed-off-by: Marko Rosenmueller <5467316+dr75@users.noreply.github.com>
Marko Rosenmueller committed
Aug 19, 2025 at 20:12 UTC
80141bbf2f1b8b0beaac097f94923f95773734ef
3 files changed
+13
-1
tests/entrypoints/openai/test_serving_chat.py
+3
-1
@@ -282,9 +282,11 @@ async def test_serving_chat_could_load_correct_generation_config():
282
assert mock_engine.generate.call_args.args[1].repetition_penalty == 1.05
283
284
285
+@pytest.mark.parametrize("model_type", ["gpt_oss", "any"])
286
@pytest.mark.asyncio
286
-async def test_serving_chat_did_set_correct_cache_salt():
287
+async def test_serving_chat_did_set_correct_cache_salt(model_type):
288
mock_model_config = MockModelConfig()
289
+ mock_model_config.hf_config.model_type = model_type
290
291
mock_engine = MagicMock(spec=MQLLMEngineClient)
292
mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
vllm/entrypoints/openai/serving_chat.py
+5
@@ -1483,4 +1483,9 @@ class OpenAIServingChat(OpenAIServing):
1483
# Render prompt token ids.
1484
prompt_token_ids = render_for_completion(messages)
1485
engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1486
+
1487
+ # Add cache_salt if provided in the request
1488
+ if request.cache_salt is not None:
1489
+ engine_prompt["cache_salt"] = request.cache_salt
1490
+
1491
return messages, [prompt_token_ids], [engine_prompt]
vllm/entrypoints/openai/serving_responses.py
+5
@@ -408,6 +408,11 @@ class OpenAIServingResponses(OpenAIServing):
408
request, prev_response)
409
prompt_token_ids = render_for_completion(messages)
410
engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
411
+
412
+ # Add cache_salt if provided in the request
413
+ if request.cache_salt is not None:
414
+ engine_prompt["cache_salt"] = request.cache_salt
415
+
416
return messages, [prompt_token_ids], [engine_prompt]
417
418
async def responses_full_generator(