@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 031a7995f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][Frontend] Reject guided decoding in multistep mode (#9892)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Oct 31, 2024 at 19:09 UTC 031a7995f38d3c73b0790280cc0fa1fe25d33bff
4 files changed +30 -3
docs/source/serving/compatibility_matrix.rst
+1 -1
@@ -283,7 +283,7 @@ Feature x Feature
283 - ✅
284 - ✅
285 - ✅
286 - - `✗ <https://github.com/vllm-project/vllm/issues/8985>`__
286 + - `✗ <https://github.com/vllm-project/vllm/issues/9893>`__
287 - ?
288 - ✅
289 - ✅
tests/entrypoints/openai/test_prompt_validation.py
+20
@@ -35,3 +35,23 @@ async def test_out_of_vocab_token_ids():
35 prompt=[999999],
36 max_tokens=5,
37 temperature=0.0)
38 +
39 +
40 +@pytest.mark.asyncio
41 +async def test_reject_multistep_with_guided_decoding():
42 + model_name = "gpt2"
43 + server_args = ["--enforce-eager", "--num-scheduler-steps", "8"]
44 + with RemoteOpenAIServer(model_name, server_args) as remote_server:
45 + client = remote_server.get_async_client()
46 +
47 + with pytest.raises(openai.BadRequestError,
48 + match=re.compile(
49 + '.*Guided decoding .* multi-step decoding.*')):
50 + await client.completions.create(
51 + model=model_name,
52 + prompt="Hello",
53 + max_tokens=5,
54 + temperature=0.0,
55 + extra_body={"response_format": {
56 + "type": "json_object"
57 + }})
vllm/engine/llm_engine.py
+7
@@ -829,6 +829,13 @@ class LLMEngine:
829 raise ValueError(f"Got priority {priority} but "
830 "Priority scheduling is not enabled.")
831
832 + if isinstance(params, SamplingParams) \
833 + and (params.guided_decoding or params.logits_processors) \
834 + and self.scheduler_config.num_scheduler_steps > 1:
835 + raise ValueError(
836 + "Guided decoding and logits processors are not supported "
837 + "in multi-step decoding")
838 +
839 if arrival_time is None:
840 arrival_time = time.time()
841
vllm/sampling_params.py
+2 -2
@@ -485,8 +485,8 @@ class SamplingParams(
485 f"skip_special_tokens={self.skip_special_tokens}, "
486 "spaces_between_special_tokens="
487 f"{self.spaces_between_special_tokens}, "
488 - f"truncate_prompt_tokens={self.truncate_prompt_tokens}), "
489 - f"guided_decoding={self.guided_decoding}")
488 + f"truncate_prompt_tokens={self.truncate_prompt_tokens}, "
489 + f"guided_decoding={self.guided_decoding})")
490
491
492 class BeamSearchParams(