| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import json |
| 4 | |
| 5 | import openai |
| 6 | import pytest |
| 7 | |
| 8 | from tests.utils import RemoteOpenAIServer |
| 9 | |
| 10 | MODEL = "ibm-granite/granite-4.0-h-tiny" |
| 11 | |
| 12 | |
| 13 | @pytest.fixture(scope="module") |
| 14 | def server(): |
| 15 | model = MODEL |
| 16 | args_for_model = [ |
| 17 | "--enforce-eager", |
| 18 | "--enable-auto-tool-choice", |
| 19 | "--tool-call-parser", |
| 20 | "granite4", |
| 21 | "--tokenizer", |
| 22 | "ibm-granite/granite-4.0-h-tiny", |
| 23 | "--max-model-len", |
| 24 | "4096", |
| 25 | "--max-num-seqs", |
| 26 | "2", |
| 27 | ] |
| 28 | with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server: |
| 29 | yield server |
| 30 | |
| 31 | |
| 32 | tools = [ |
| 33 | { |
| 34 | "type": "function", |
| 35 | "function": { |
| 36 | "name": "get_acme_region_name_for_transaction_id", |
| 37 | "description": "Returns ACME transaction/transaction ID information" |
| 38 | " including ACME regions\n\nArgs:\n start_time " |
| 39 | "(str): Start date and time in datetime format " |
| 40 | '"%Y-%m-%dT%H:%M:%S.%f"\n end_time (str): End ' |
| 41 | "date and time in datetime format " |
| 42 | '"%Y-%m-%dT%H:%M:%S.%f"\n size (int, optional): ' |
| 43 | "Number of ACME Transaction IDs to return\n " |
| 44 | "order (str, optional): Sort by most run " |
| 45 | "transaction IDs. The value can be 'asc' for " |
| 46 | "ascending or 'desc' for descending\n " |
| 47 | "transaction_id (str, optional): ACME Transaction " |
| 48 | "ID to filter on\n acme_region (str, optional): " |
| 49 | "ACME Region to filter on\nReturns:\n - A " |
| 50 | "dictionary containing a list of ACME transaction " |
| 51 | "ids and the ACME regions they run in:\n {\n" |
| 52 | ' "Number of transaction IDs" : int,\n' |
| 53 | ' "Total transaction IDs available": int' |
| 54 | ',\n "ACME Transaction IDs": [\n ' |
| 55 | ' {\n "Transaction ID": ' |
| 56 | 'str,\n "Number of runs": int,\n' |
| 57 | ' "ACME Regions": [str],\n ' |
| 58 | " },\n ...\n ]," |
| 59 | '\n "Start time" : datetime,\n ' |
| 60 | ' "End time" : datetime,\n ' |
| 61 | ' "Order" : str\n }\n ' |
| 62 | " - If no ACME region found for transaction id, " |
| 63 | 'returns:\n {"Success": "No ACME region ' |
| 64 | 'found for transaction id."}\n - If an error ' |
| 65 | 'occurs, returns:\n {"Error": "{exception' |
| 66 | ' message}"}', |
| 67 | "parameters": { |
| 68 | "properties": { |
| 69 | "start_time": {}, |
| 70 | "end_time": {}, |
| 71 | "size": {"default": 500}, |
| 72 | "order": {"default": "desc"}, |
| 73 | "transaction_id": {"default": None}, |
| 74 | "acme_region": {"default": None}, |
| 75 | }, |
| 76 | "required": ["start_time", "end_time"], |
| 77 | "type": "object", |
| 78 | }, |
| 79 | }, |
| 80 | } |
| 81 | ] |
| 82 | |
| 83 | tools2 = [ |
| 84 | { |
| 85 | "type": "function", |
| 86 | "function": { |
| 87 | "name": "get_current_weather", |
| 88 | "description": "Get the current weather", |
| 89 | "parameters": { |
| 90 | "type": "object", |
| 91 | "properties": { |
| 92 | "location": { |
| 93 | "description": "The city and state, e.g. San Francisco, CA", |
| 94 | "type": "string", |
| 95 | } |
| 96 | }, |
| 97 | "required": ["location"], |
| 98 | }, |
| 99 | }, |
| 100 | }, |
| 101 | { |
| 102 | "type": "function", |
| 103 | "function": { |
| 104 | "name": "get_stock_price", |
| 105 | "description": "Retrieves the current stock price for a given " |
| 106 | "ticker symbol. The ticker symbol must be a valid " |
| 107 | "symbol for a publicly traded company on a major US" |
| 108 | " stock exchange like NYSE or NASDAQ. The tool will" |
| 109 | " return the latest trade price in USD. It should " |
| 110 | "be used when the user asks about the current or " |
| 111 | "most recent price of a specific stock. It will not" |
| 112 | " provide any other information about the stock or" |
| 113 | " company.", |
| 114 | "parameters": { |
| 115 | "type": "object", |
| 116 | "properties": { |
| 117 | "ticker": { |
| 118 | "description": "The stock ticker symbol, e.g." |
| 119 | " AAPL for Apple Inc.", |
| 120 | "type": "string", |
| 121 | } |
| 122 | }, |
| 123 | }, |
| 124 | }, |
| 125 | }, |
| 126 | ] |
| 127 | |
| 128 | messages = [ |
| 129 | { |
| 130 | "content": "\n\nSystem: You are a helpful, precise, and methodical AI" |
| 131 | " assistant that uses tool outputs provided inline.\nAlways" |
| 132 | " assume the current datetime is 2026-01-29T13:59:09.238901" |
| 133 | "+00:00.\n\nIf you receive a ToolMessage with `tool_call_id" |
| 134 | '` equal to "get_time_range" (or "time_range_tool"), you ' |
| 135 | "MUST:\n 1. Parse that JSON and use the values `start` and" |
| 136 | " `end` directly when calling other tools.\n 2. Do not " |
| 137 | "re-call or re-compute the time range.\n 3. Pass resolved " |
| 138 | "values (ISO strings) as arguments to any subsequent tool " |
| 139 | "(do not pass function metadata or placeholders).\n 4. If " |
| 140 | "a tool requires datetime objects rather than strings, " |
| 141 | "convert the ISO strings into language-native datetime " |
| 142 | "objects before invoking.\n\nAlways return fully resolved " |
| 143 | "arguments in correct types (e.g., ISO datetime strings or" |
| 144 | " datetime objects) and never include placeholders like " |
| 145 | '"<start>".\n\n', |
| 146 | "role": "system", |
| 147 | }, |
| 148 | { |
| 149 | "content": "What are the transaction IDs that ran in the" |
| 150 | " ACME region A9345 over the last two months?", |
| 151 | "role": "user", |
| 152 | }, |
| 153 | { |
| 154 | "content": '["2026-01-26T09: 51: 55.467722Z", "2026-01-27T09: 51: 55.467722Z"]', |
| 155 | "role": "tool", |
| 156 | "tool_call_id": "time_range_tool", |
| 157 | }, |
| 158 | ] |
| 159 | messages2 = [{"role": "user", "content": "What's stock price for IBM?"}] |
| 160 | |
| 161 | messages3 = [{"role": "user", "content": "What's the current weather in New York?"}] |
| 162 | |
| 163 | |
| 164 | def get_args(client: openai.OpenAI, _tools, _messages, _stop): |
| 165 | response = client.chat.completions.create( |
| 166 | model=MODEL, |
| 167 | messages=_messages, |
| 168 | temperature=0, |
| 169 | tools=_tools, |
| 170 | max_tokens=200, |
| 171 | stop=_stop, |
| 172 | tool_choice="auto", |
| 173 | ) |
| 174 | |
| 175 | return response.choices[0].message.tool_calls[0].function.arguments |
| 176 | |
| 177 | |
| 178 | async def get_args_streaming( |
| 179 | async_client: openai.AsyncOpenAI, _tools, _messages, _stop |
| 180 | ): |
| 181 | stream = await async_client.chat.completions.create( |
| 182 | model=MODEL, |
| 183 | messages=_messages, |
| 184 | temperature=0, |
| 185 | tools=_tools, |
| 186 | max_tokens=200, |
| 187 | stop=_stop, |
| 188 | tool_choice="auto", |
| 189 | stream=True, |
| 190 | ) |
| 191 | full_call = [] |
| 192 | async for chunk in stream: |
| 193 | tc = chunk.choices[0].delta.tool_calls |
| 194 | if tc and tc[0].function.arguments: |
| 195 | full_call.append(tc[0].function.arguments) |
| 196 | return "".join(full_call) |
| 197 | |
| 198 | |
| 199 | async def run_scenario(server: RemoteOpenAIServer, _tools, _messages, _stop): |
| 200 | non_streaming = get_args(server.get_client(), _tools, _messages, _stop) |
| 201 | json.loads(non_streaming) # verify that it is json loadable |
| 202 | streaming = await get_args_streaming( |
| 203 | server.get_async_client(), _tools, _messages, _stop |
| 204 | ) |
| 205 | json.loads(streaming) |
| 206 | assert non_streaming == streaming, f"{non_streaming=}, {streaming=}" |
| 207 | |
| 208 | |
| 209 | @pytest.mark.asyncio |
| 210 | async def test_stop_sequence_interference(server: RemoteOpenAIServer): |
| 211 | print("Testing scenario 1") |
| 212 | await run_scenario(server, tools, messages, "veroniqueprattyushveroniqueprattyush") |
| 213 | |
| 214 | print("Testing scenario 2") |
| 215 | await run_scenario( |
| 216 | server, tools2, messages2, "veroniqueprattyushveroniqueprattyush" |
| 217 | ) |
| 218 | |
| 219 | print("Testing scenario 3") |
| 220 | await run_scenario(server, tools2, messages3, "prattyush") |