@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 173 lines 6.04 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 """Unit tests for stop_token_ids propagation from default_sampling_params
5 to SamplingParams in ChatCompletionRequest and CompletionRequest.
6
7 Regression test for https://github.com/vllm-project/vllm/issues/22519
8 where gpt-oss model stop tokens (e.g., </call> = 200012) were loaded into
9 default_sampling_params at server startup but silently discarded on every
10 request because to_sampling_params() never fell back to defaults.
11 """
12
13 import pytest
14
15 from vllm.entrypoints.openai.chat_completion.protocol import (
16 ChatCompletionRequest,
17 )
18 from vllm.entrypoints.openai.completion.protocol import (
19 CompletionRequest,
20 )
21
22
23 class TestChatCompletionStopTokenIds:
24 """Test stop_token_ids merging in ChatCompletionRequest.to_sampling_params()."""
25
26 @pytest.fixture
27 def minimal_chat_request(self):
28 return ChatCompletionRequest(
29 model="test-model",
30 messages=[{"role": "user", "content": "hello"}],
31 )
32
33 def test_default_stop_token_ids_applied(self, minimal_chat_request):
34 """Server-default stop_token_ids are applied when client sends none."""
35 default_sampling_params = {
36 "stop_token_ids": [200012, 200002],
37 }
38
39 sampling_params = minimal_chat_request.to_sampling_params(
40 max_tokens=100,
41 default_sampling_params=default_sampling_params,
42 )
43
44 assert set(sampling_params.stop_token_ids) == {200012, 200002}
45
46 def test_client_stop_token_ids_merged_with_defaults(self):
47 """Client-specified stop_token_ids are merged with server defaults."""
48 request = ChatCompletionRequest(
49 model="test-model",
50 messages=[{"role": "user", "content": "hello"}],
51 stop_token_ids=[99999],
52 )
53 default_sampling_params = {
54 "stop_token_ids": [200012, 200002],
55 }
56
57 sampling_params = request.to_sampling_params(
58 max_tokens=100,
59 default_sampling_params=default_sampling_params,
60 )
61
62 assert set(sampling_params.stop_token_ids) == {200012, 200002, 99999}
63 assert sampling_params.stop_token_ids == [99999, 200012, 200002]
64
65 def test_no_stop_token_ids_anywhere(self, minimal_chat_request):
66 """When neither client nor server specifies stop_token_ids, result is empty."""
67 sampling_params = minimal_chat_request.to_sampling_params(
68 max_tokens=100,
69 default_sampling_params={},
70 )
71
72 assert not sampling_params.stop_token_ids
73
74 def test_only_client_stop_token_ids(self):
75 """Client stop_token_ids work when no server defaults exist."""
76 request = ChatCompletionRequest(
77 model="test-model",
78 messages=[{"role": "user", "content": "hello"}],
79 stop_token_ids=[42, 43],
80 )
81
82 sampling_params = request.to_sampling_params(
83 max_tokens=100,
84 default_sampling_params={},
85 )
86
87 assert set(sampling_params.stop_token_ids) == {42, 43}
88
89 def test_routed_experts_prompt_start_is_forwarded(self):
90 request = ChatCompletionRequest(
91 model="test-model",
92 messages=[{"role": "user", "content": "hello"}],
93 routed_experts_prompt_start=3,
94 )
95
96 sampling_params = request.to_sampling_params(
97 max_tokens=100,
98 default_sampling_params={},
99 )
100
101 assert sampling_params.routed_experts_prompt_start == 3
102
103 def test_duplicate_stop_token_ids_deduplicated(self):
104 """Overlapping stop_token_ids between client and server are deduplicated."""
105 request = ChatCompletionRequest(
106 model="test-model",
107 messages=[{"role": "user", "content": "hello"}],
108 stop_token_ids=[200012, 55555],
109 )
110 default_sampling_params = {
111 "stop_token_ids": [200012, 200002],
112 }
113
114 sampling_params = request.to_sampling_params(
115 max_tokens=100,
116 default_sampling_params=default_sampling_params,
117 )
118
119 assert set(sampling_params.stop_token_ids) == {200012, 200002, 55555}
120 assert sampling_params.stop_token_ids == [200012, 55555, 200002]
121 assert len(sampling_params.stop_token_ids) == 3
122
123
124 class TestCompletionStopTokenIds:
125 """Test stop_token_ids merging in CompletionRequest.to_sampling_params()."""
126
127 @pytest.fixture
128 def minimal_completion_request(self):
129 return CompletionRequest(
130 model="test-model",
131 prompt="hello",
132 )
133
134 def test_default_stop_token_ids_applied(self, minimal_completion_request):
135 """Server-default stop_token_ids are applied when client sends none."""
136 default_sampling_params = {
137 "stop_token_ids": [200012, 200002],
138 }
139
140 sampling_params = minimal_completion_request.to_sampling_params(
141 max_tokens=100,
142 default_sampling_params=default_sampling_params,
143 )
144
145 assert set(sampling_params.stop_token_ids) == {200012, 200002}
146
147 def test_client_stop_token_ids_merged_with_defaults(self):
148 """Client-specified stop_token_ids are merged with server defaults."""
149 request = CompletionRequest(
150 model="test-model",
151 prompt="hello",
152 stop_token_ids=[99999],
153 )
154 default_sampling_params = {
155 "stop_token_ids": [200012, 200002],
156 }
157
158 sampling_params = request.to_sampling_params(
159 max_tokens=100,
160 default_sampling_params=default_sampling_params,
161 )
162
163 assert set(sampling_params.stop_token_ids) == {200012, 200002, 99999}
164 assert sampling_params.stop_token_ids == [99999, 200012, 200002]
165
166 def test_no_stop_token_ids_anywhere(self, minimal_completion_request):
167 """When neither client nor server specifies stop_token_ids, result is empty."""
168 sampling_params = minimal_completion_request.to_sampling_params(
169 max_tokens=100,
170 default_sampling_params={},
171 )
172
173 assert not sampling_params.stop_token_ids