@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
proto 267 lines 7.31 KB
Raw
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 syntax = "proto3";
5 package vllm;
6
7 import "google/protobuf/struct.proto";
8
9
10 service Inference {
11 // Generates text given a prompt
12 rpc Generate (GenerateRequest) returns (GenerateResponse) {}
13 // Generates text given a prompt, streaming the outputs
14 rpc GenerateStream (GenerateRequest) returns (stream GenerateResponse) {}
15 }
16
17 // ======================================================================================
18 // Generate Request
19 // ======================================================================================
20
21 message GenerateRequest {
22 string request_id = 1;
23 string model = 2;
24
25 oneof prompt {
26 string text = 3;
27 TokenIds token_ids = 4;
28 }
29
30 // Temperature, defaults to model-specific default or 0
31 optional float temperature = 5;
32 // Parameters controlling random sampling, not applicable if temperature == 0
33 RandomSampling sampling = 6;
34 // Parameters for conditionally penalizing/boosting
35 // candidate tokens during decoding
36 DecodingParameters decoding = 7;
37 // Parameters controlling when generation should stop
38 StoppingCriteria stopping = 8;
39 // Flags to control what is returned in the response
40 ResponseOptions response = 9;
41 // Parameters controlling KV cache/distribution
42 KVCacheParameters kv = 10;
43
44 // Truncate prompt tokens; default (0) means no truncation
45 uint32 truncate_prompt_tokens = 11;
46
47 int32 priority = 12;
48
49 optional string session_id = 13;
50
51 // Multimodal inputs aligned with placeholder markers in token_ids.
52 repeated MediaItem media = 14;
53
54 // Loaded LoRA name; empty selects the base model.
55 string lora_name = 15;
56
57 // Apply the engine's configured watermark. Defaults to true when omitted.
58 optional bool watermarking = 16;
59
60 KVHintsEnvelope kv_hints = 17;
61 }
62
63 message KVHintsEnvelope {
64 string protocol_version = 1;
65 string message_id = 2;
66 repeated KVHintAction actions = 3;
67 }
68
69 message KVHintAction {
70 string action_id = 1;
71 string action_type = 2;
72 string action_version = 3;
73 google.protobuf.Struct payload = 4;
74 }
75
76 message RandomSampling {
77 uint32 num_sequences = 1; // "n", default (0) means 1
78 uint32 top_k = 2; // 0 means default
79 float top_p = 3; // 0 means default
80 float min_p = 4; // 0 means default
81 optional int64 seed = 5;
82 }
83
84 message DecodingParameters {
85 // Penalties
86 float presence_penalty = 1; // Default (0.0) means no penalty
87 float frequency_penalty = 2; // Default (0.0) means no penalty
88 float repetition_penalty = 3; // Default (0.0) means no penalty
89 map<uint32, float> logit_bias = 4;
90 repeated uint32 allowed_token_ids = 5;
91
92 message StringChoices {
93 repeated string choices = 1;
94 }
95
96 // Control structured outputs
97 oneof structured_output {
98 string json = 6;
99 string regex = 7;
100 StringChoices choice = 8;
101 string grammar = 9;
102 bool json_object = 10;
103 string structural_tag = 11;
104 }
105 }
106
107 message StoppingCriteria {
108 // Default (0) is currently 20
109 uint32 max_new_tokens = 1;
110 // Default (0) means no minimum
111 uint32 min_new_tokens = 2;
112
113 repeated uint32 stop_token_ids = 3;
114 repeated string stop_strings = 4;
115 bool include_stop_strings = 5;
116
117 bool ignore_eos = 6;
118 }
119
120 message ResponseOptions {
121 // Prompt options
122 bool prompt_token_ids = 1;
123 bool prompt_logprobs = 2;
124 optional CandidateTokens prompt_candidates = 3;
125
126 // Output options; output_text defaults to true
127 optional bool output_text = 4;
128 bool output_token_ids = 5;
129 bool output_logprobs = 6;
130 optional CandidateTokens output_candidates = 7;
131 // Defaults to true when omitted.
132 optional bool skip_special_tokens = 8;
133 }
134
135 message KVCacheParameters {
136 bool bypass_prefix_cache = 1;
137 string cache_salt = 2;
138
139 // KV Connector transfer parameters
140 google.protobuf.Struct kv_transfer_params = 3;
141
142 // Encoder cache connector transfer parameters
143 google.protobuf.Struct ec_transfer_params = 4;
144 }
145
146 // Controls which extra candidate tokens at each position should be returned
147 message CandidateTokens {
148 oneof select {
149 uint32 top_n = 1;
150 TokenIds token_ids = 2;
151 bool all = 3;
152 }
153 }
154
155 // ======================================================================================
156 // Generate Response
157 // ======================================================================================
158
159 message GenerateResponse {
160 // Only present in first response
161 optional PromptInfo prompt_info = 1;
162 SequenceOutput outputs = 2;
163 }
164
165 message SequenceOutput {
166 // Index of output sequence for num_sequences > 1.
167 uint32 index = 1;
168
169 string text = 2;
170 uint32 num_tokens = 3; // Number of tokens in this chunk
171 repeated uint32 token_ids = 4; // If requested
172 repeated float logprobs = 5; // If requested
173 repeated uint32 ranks = 6; // If logprobs were requested
174 repeated CandidateTokenInfo candidate_tokens = 7; // If requested
175
176 // Only present in final output for this sequence
177 optional FinishInfo finish_info = 8;
178 // Only present in final output when sampling-mask capture is enabled.
179 repeated TokenIds sampling_mask = 10;
180 }
181
182 // Prompt info, returned in the first response
183 message PromptInfo {
184 uint32 num_prompt_tokens = 1;
185 repeated uint32 token_ids = 2; // If requested
186 repeated float logprobs = 3; // If requested
187 repeated uint32 ranks = 4; // If logprobs were requested
188 repeated CandidateTokenInfo candidate_tokens = 5;
189 }
190
191 // Finish info, returned in the final response
192 message FinishInfo {
193 uint32 num_output_tokens = 1;
194
195 enum FinishReason {
196 NOT_FINISHED = 0; // Possibly more tokens to be streamed
197 LENGTH = 1; // Finished due to length constraint
198 STOP = 2; // Stop string/token or EOS encountered
199 ABORTED = 3; // Request aborted/cancelled
200 }
201
202 FinishReason finish_reason = 2;
203 // One of these will be set when finish_reason == STOP
204 oneof stop_reason {
205 uint32 stop_token_id = 3;
206 uint32 eos_token_id = 4;
207 string stop_string = 5;
208 }
209
210 google.protobuf.Struct kv_transfer_params = 6;
211 //uint64 seed = 7;
212 google.protobuf.Struct ec_transfer_params = 8;
213 }
214
215 // Info for candidate tokens other than the input/sampled
216 // token at a given position
217 message CandidateTokenInfo {
218 message TokenInfo {
219 uint32 id = 1;
220 float logprob = 2;
221 uint32 rank = 3;
222 // string text = 4;
223 // bytes token_bytes = 5;
224 }
225 // Candidate token infos at this position
226 repeated TokenInfo tokens = 1;
227 }
228
229 // Token ids used for prompt
230 message TokenIds {
231 repeated uint32 ids = 1;
232 }
233
234 // ======================================================================================
235 // Media
236 // ======================================================================================
237
238 enum Modality {
239 MODALITY_UNSPECIFIED = 0;
240 MODALITY_IMAGE = 1;
241 MODALITY_VIDEO = 2;
242 MODALITY_AUDIO = 3;
243 }
244
245 message MediaItem {
246 Modality modality = 1;
247 oneof source {
248 string url = 2; // http:// or https://
249 string data_uri = 3; // data:
250 bytes raw_bytes = 4;
251 PreprocessedMediaFeatures features = 7;
252 }
253 string mime_type = 5;
254 string uuid = 6;
255 }
256
257 // Engine-ready multimodal features produced by a version-compatible frontend.
258 // `kwargs` is the MessagePack wire representation of one MmKwargsItem.
259 message PreprocessedMediaFeatures {
260 // Required inline payload. Cache-only references are not accepted.
261 optional bytes kwargs = 1;
262 string identifier = 2;
263 uint64 offset = 3;
264 uint64 length = 4;
265 optional string mm_hash = 5;
266 repeated bool is_embed = 6;
267 }