A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Bugfix] Fix bug when dataset is json (#15899)
Signed-off-by: Chenyaaang <chenyangli@google.com>
Chenyaaang committed
Apr 10, 2025 at 11:35 UTC
5fbab20e021a4febbeda830eaa15a2deb598c01e
1 file changed
+3
-2
benchmarks/benchmark_serving_structured_output.py
+3
-2
@@ -130,10 +130,11 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
130
"description":
131
"An unique optional field to avoid cached schemas"
132
}
133
+ else:
134
+ json_schemas = [schema] * args.num_prompts
135
136
def gen_prompt(index: int):
135
- schema = json_schemas[index % len(json_schemas)]
136
- return f"Generate an example of a user profile given the following schema: {json.dumps(schema)}" # noqa: E501
137
+ return f"Generate an example of a user profile given the following schema: {json.dumps(get_schema(index))}" # noqa: E501
138
139
def get_schema(index: int):
140
return json_schemas[index % len(json_schemas)]