A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Add fix for JSON whitespace (#4189)
Co-authored-by: Ubuntu <ubuntu@ip-172-31-13-147.ec2.internal>
Ayush Rautwar committed
Apr 19, 2024 at 23:49 UTC
138485a82de50f90536ea0a650dd2f6bba1927e9
2 files changed
+19
-13
tests/entrypoints/test_openai_server.py
+14
-13
@@ -754,19 +754,20 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
754
755
756
async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
757
- resp = await client.chat.completions.create(
758
- model=MODEL_NAME,
759
- messages=[{
760
- "role":
761
- "user",
762
- "content": ('what is 1+1? please respond with a JSON object, '
763
- 'the format is {"result": 2}')
764
- }],
765
- response_format={"type": "json_object"})
766
-
767
- content = resp.choices[0].message.content
768
- loaded = json.loads(content)
769
- assert loaded == {"result": 2}, loaded
757
+ for _ in range(2):
758
+ resp = await client.chat.completions.create(
759
+ model=MODEL_NAME,
760
+ messages=[{
761
+ "role":
762
+ "user",
763
+ "content": ('what is 1+1? please respond with a JSON object, '
764
+ 'the format is {"result": 2}')
765
+ }],
766
+ response_format={"type": "json_object"})
767
+
768
+ content = resp.choices[0].message.content
769
+ loaded = json.loads(content)
770
+ assert loaded == {"result": 2}, loaded
771
772
773
async def test_guided_grammar(server, client: openai.AsyncOpenAI):
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+5
@@ -131,6 +131,11 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
131
fsm = CFGFSM(cfg, tokenizer)
132
self.fsm = fsm
133
134
+ def init_state(self):
135
+ """Initialize state with a CFGFSM copy."""
136
+ super().init_state()
137
+ self.fsm = self.fsm.copy()
138
+
139
140
@lru_cache
141
def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):