@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0064f697d3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[CI] Add test case with JSON schema using references + use xgrammar by default with OpenAI parse (#10935)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 16, 2024 at 22:39 UTC 0064f697d318a2ce38342f7c20754cf229311b8b
3 files changed +68 -1
tests/entrypoints/conftest.py
+39
@@ -100,6 +100,45 @@ def sample_complex_json_schema():
100 }
101
102
103 +@pytest.fixture
104 +def sample_definition_json_schema():
105 + return {
106 + '$defs': {
107 + 'Step': {
108 + 'properties': {
109 + 'explanation': {
110 + 'title': 'Explanation',
111 + 'type': 'string'
112 + },
113 + 'output': {
114 + 'title': 'Output',
115 + 'type': 'string'
116 + }
117 + },
118 + 'required': ['explanation', 'output'],
119 + 'title': 'Step',
120 + 'type': 'object'
121 + }
122 + },
123 + 'properties': {
124 + 'steps': {
125 + 'items': {
126 + '$ref': '#/$defs/Step'
127 + },
128 + 'title': 'Steps',
129 + 'type': 'array'
130 + },
131 + 'final_answer': {
132 + 'title': 'Final Answer',
133 + 'type': 'string'
134 + }
135 + },
136 + 'required': ['steps', 'final_answer'],
137 + 'title': 'MathReasoning',
138 + 'type': 'object'
139 + }
140 +
141 +
142 @pytest.fixture
143 def sample_guided_choice():
144 return [
tests/entrypoints/llm/test_guided_generate.py
+28
@@ -104,6 +104,34 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm):
104 schema=sample_complex_json_schema)
105
106
107 +@pytest.mark.skip_global_cleanup
108 +def test_guided_definition_json_completion(sample_definition_json_schema, llm):
109 + sampling_params = SamplingParams(temperature=1.0,
110 + max_tokens=1000,
111 + guided_decoding=GuidedDecodingParams(
112 + json=sample_definition_json_schema))
113 + outputs = llm.generate(prompts=[
114 + f"Give an example JSON for solving 8x + 7 = -23 "
115 + f"that fits this schema: {sample_definition_json_schema}"
116 + ] * 2,
117 + sampling_params=sampling_params,
118 + use_tqdm=True)
119 +
120 + assert outputs is not None
121 +
122 + for output in outputs:
123 + assert output is not None
124 + assert isinstance(output, RequestOutput)
125 + prompt = output.prompt
126 +
127 + generated_text = output.outputs[0].text
128 + assert generated_text is not None
129 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
130 + output_json = json.loads(generated_text)
131 + jsonschema.validate(instance=output_json,
132 + schema=sample_definition_json_schema)
133 +
134 +
135 @pytest.mark.skip_global_cleanup
136 def test_guided_choice_completion(sample_guided_choice, llm):
137 sampling_params = SamplingParams(
vllm/entrypoints/openai/protocol.py
+1 -1
@@ -387,7 +387,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
387 assert json_schema is not None
388 self.guided_json = json_schema.json_schema
389 if self.guided_decoding_backend is None:
390 - self.guided_decoding_backend = "lm-format-enforcer"
390 + self.guided_decoding_backend = "xgrammar"
391
392 guided_decoding = GuidedDecodingParams.from_optional(
393 json=self._get_guided_json_from_tool() or self.guided_json,