@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 74fa1d123c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix OpenAI parallel sampling when using xgrammar (#11637)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Dec 30, 2024 at 22:43 UTC 74fa1d123c2818065d862d2ceb2338468914fa79
4 files changed +17 -13
tests/entrypoints/openai/test_completion.py
+6 -8
@@ -28,6 +28,8 @@ PA_NAME = "swapnilbp/llama_tweet_ptune"
28 # need to change to match the prompt adapter
29 PA_NUM_VIRTUAL_TOKENS = 8
30
31 +GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
32 +
33
34 @pytest.fixture(scope="module")
35 def zephyr_lora_files():
@@ -635,8 +637,7 @@ async def test_allowed_token_ids(client: openai.AsyncOpenAI):
637
638
639 @pytest.mark.asyncio
638 -@pytest.mark.parametrize("guided_decoding_backend",
639 - ["outlines", "lm-format-enforcer"])
640 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
641 async def test_guided_json_completion(client: openai.AsyncOpenAI,
642 guided_decoding_backend: str,
643 sample_json_schema):
@@ -658,8 +659,7 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
659
660
661 @pytest.mark.asyncio
661 -@pytest.mark.parametrize("guided_decoding_backend",
662 - ["outlines", "lm-format-enforcer"])
662 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
663 async def test_guided_regex_completion(client: openai.AsyncOpenAI,
664 guided_decoding_backend: str,
665 sample_regex):
@@ -680,8 +680,7 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
680
681
682 @pytest.mark.asyncio
683 -@pytest.mark.parametrize("guided_decoding_backend",
684 - ["outlines", "lm-format-enforcer"])
683 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
684 async def test_guided_choice_completion(client: openai.AsyncOpenAI,
685 guided_decoding_backend: str,
686 sample_guided_choice):
@@ -761,8 +760,7 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
760
761
762 @pytest.mark.asyncio
764 -@pytest.mark.parametrize("guided_decoding_backend",
765 - ["outlines", "lm-format-enforcer"])
763 +@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
764 async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
765 guided_decoding_backend: str,
766 sample_json_schema, sample_regex):
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+5
@@ -1,6 +1,7 @@
1 # noqa: UP007
2 from __future__ import annotations
3
4 +import copy
5 import json
6 from dataclasses import dataclass, field
7 from typing import TYPE_CHECKING, Any
@@ -309,3 +310,7 @@ class XGrammarLogitsProcessor:
310 scores = scores.to(device_type).squeeze()
311
312 return scores
313 +
314 + def clone(self) -> XGrammarLogitsProcessor:
315 + """Deepcopy due to per-sequence state in the matchers"""
316 + return copy.deepcopy(self)
vllm/sampling_params.py
+5 -4
@@ -450,15 +450,16 @@ class SamplingParams(
450 return self._all_stop_token_ids
451
452 def clone(self) -> "SamplingParams":
453 - """Deep copy excluding LogitsProcessor objects.
453 + """Deep copy, but maybe not the LogitsProcessor objects.
454
455 - LogitsProcessor objects are excluded because they may contain an
456 - arbitrary, nontrivial amount of data.
455 + LogitsProcessor objects may contain an arbitrary, nontrivial amount of
456 + data that is expensive to copy. However, if not copied, the processor
457 + needs to support parallel decoding for multiple sequences
458 See https://github.com/vllm-project/vllm/issues/3087
459 """
460
461 logit_processor_refs = None if self.logits_processors is None else {
461 - id(lp): lp
462 + id(lp): lp.clone() if hasattr(lp, 'clone') else lp
463 for lp in self.logits_processors
464 }
465 return copy.deepcopy(self, memo=logit_processor_refs)
vllm/sequence.py
+1 -1
@@ -1372,7 +1372,7 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1372 @staticmethod
1373 def add_request(request_id: str, engine, params, **kwargs):
1374 original_params = params
1375 - params = copy.deepcopy(original_params)
1375 + params = original_params.clone()
1376 params.n = 1
1377 group = ParallelSampleSequenceGroup(request_id)
1378 seqs = []