A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix OpenAI parallel sampling when using xgrammar (#11637)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Dec 30, 2024 at 22:43 UTC
74fa1d123c2818065d862d2ceb2338468914fa79
4 files changed
+17
-13
tests/entrypoints/openai/test_completion.py
+6
-8
@@ -28,6 +28,8 @@ PA_NAME = "swapnilbp/llama_tweet_ptune"
28
# need to change to match the prompt adapter
29
PA_NUM_VIRTUAL_TOKENS = 8
30
31
+GUIDED_DECODING_BACKENDS = ["outlines", "lm-format-enforcer", "xgrammar"]
32
+
33
34
@pytest.fixture(scope="module")
35
def zephyr_lora_files():
@@ -635,8 +637,7 @@ async def test_allowed_token_ids(client: openai.AsyncOpenAI):
637
638
639
@pytest.mark.asyncio
638
-@pytest.mark.parametrize("guided_decoding_backend",
639
- ["outlines", "lm-format-enforcer"])
640
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
641
async def test_guided_json_completion(client: openai.AsyncOpenAI,
642
guided_decoding_backend: str,
643
sample_json_schema):
@@ -658,8 +659,7 @@ async def test_guided_json_completion(client: openai.AsyncOpenAI,
659
660
661
@pytest.mark.asyncio
661
-@pytest.mark.parametrize("guided_decoding_backend",
662
- ["outlines", "lm-format-enforcer"])
662
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
663
async def test_guided_regex_completion(client: openai.AsyncOpenAI,
664
guided_decoding_backend: str,
665
sample_regex):
@@ -680,8 +680,7 @@ async def test_guided_regex_completion(client: openai.AsyncOpenAI,
680
681
682
@pytest.mark.asyncio
683
-@pytest.mark.parametrize("guided_decoding_backend",
684
- ["outlines", "lm-format-enforcer"])
683
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
684
async def test_guided_choice_completion(client: openai.AsyncOpenAI,
685
guided_decoding_backend: str,
686
sample_guided_choice):
@@ -761,8 +760,7 @@ async def test_echo_logprob_completion(client: openai.AsyncOpenAI,
760
761
762
@pytest.mark.asyncio
764
-@pytest.mark.parametrize("guided_decoding_backend",
765
- ["outlines", "lm-format-enforcer"])
763
+@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS)
764
async def test_guided_decoding_type_error(client: openai.AsyncOpenAI,
765
guided_decoding_backend: str,
766
sample_json_schema, sample_regex):
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+5
@@ -1,6 +1,7 @@
1
# noqa: UP007
2
from __future__ import annotations
3
4
+import copy
5
import json
6
from dataclasses import dataclass, field
7
from typing import TYPE_CHECKING, Any
@@ -309,3 +310,7 @@ class XGrammarLogitsProcessor:
310
scores = scores.to(device_type).squeeze()
311
312
return scores
313
+
314
+ def clone(self) -> XGrammarLogitsProcessor:
315
+ """Deepcopy due to per-sequence state in the matchers"""
316
+ return copy.deepcopy(self)
vllm/sampling_params.py
+5
-4
@@ -450,15 +450,16 @@ class SamplingParams(
450
return self._all_stop_token_ids
451
452
def clone(self) -> "SamplingParams":
453
- """Deep copy excluding LogitsProcessor objects.
453
+ """Deep copy, but maybe not the LogitsProcessor objects.
454
455
- LogitsProcessor objects are excluded because they may contain an
456
- arbitrary, nontrivial amount of data.
455
+ LogitsProcessor objects may contain an arbitrary, nontrivial amount of
456
+ data that is expensive to copy. However, if not copied, the processor
457
+ needs to support parallel decoding for multiple sequences
458
See https://github.com/vllm-project/vllm/issues/3087
459
"""
460
461
logit_processor_refs = None if self.logits_processors is None else {
461
- id(lp): lp
462
+ id(lp): lp.clone() if hasattr(lp, 'clone') else lp
463
for lp in self.logits_processors
464
}
465
return copy.deepcopy(self, memo=logit_processor_refs)
vllm/sequence.py
+1
-1
@@ -1372,7 +1372,7 @@ class ParallelSampleSequenceGroup(SequenceGroupBase):
1372
@staticmethod
1373
def add_request(request_id: str, engine, params, **kwargs):
1374
original_params = params
1375
- params = copy.deepcopy(original_params)
1375
+ params = original_params.clone()
1376
params.n = 1
1377
group = ParallelSampleSequenceGroup(request_id)
1378
seqs = []