@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 138485a82d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Add fix for JSON whitespace (#4189)

Co-authored-by: Ubuntu <ubuntu@ip-172-31-13-147.ec2.internal>

Ayush Rautwar committed Apr 19, 2024 at 23:49 UTC 138485a82de50f90536ea0a650dd2f6bba1927e9
2 files changed +19 -13
tests/entrypoints/test_openai_server.py
+14 -13
@@ -754,19 +754,20 @@ async def test_guided_choice_chat_logprobs(server, client: openai.AsyncOpenAI,
754
755
756 async def test_response_format_json_object(server, client: openai.AsyncOpenAI):
757 - resp = await client.chat.completions.create(
758 - model=MODEL_NAME,
759 - messages=[{
760 - "role":
761 - "user",
762 - "content": ('what is 1+1? please respond with a JSON object, '
763 - 'the format is {"result": 2}')
764 - }],
765 - response_format={"type": "json_object"})
766 -
767 - content = resp.choices[0].message.content
768 - loaded = json.loads(content)
769 - assert loaded == {"result": 2}, loaded
757 + for _ in range(2):
758 + resp = await client.chat.completions.create(
759 + model=MODEL_NAME,
760 + messages=[{
761 + "role":
762 + "user",
763 + "content": ('what is 1+1? please respond with a JSON object, '
764 + 'the format is {"result": 2}')
765 + }],
766 + response_format={"type": "json_object"})
767 +
768 + content = resp.choices[0].message.content
769 + loaded = json.loads(content)
770 + assert loaded == {"result": 2}, loaded
771
772
773 async def test_guided_grammar(server, client: openai.AsyncOpenAI):
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+5
@@ -131,6 +131,11 @@ class CFGLogitsProcessor(BaseLogitsProcessor):
131 fsm = CFGFSM(cfg, tokenizer)
132 self.fsm = fsm
133
134 + def init_state(self):
135 + """Initialize state with a CFGFSM copy."""
136 + super().init_state()
137 + self.fsm = self.fsm.copy()
138 +
139
140 @lru_cache
141 def _adapt_tokenizer(tokenizer: PreTrainedTokenizerBase):