A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Bugfix][Frontend] Reject guided decoding in multistep mode (#9892)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Oct 31, 2024 at 19:09 UTC
031a7995f38d3c73b0790280cc0fa1fe25d33bff
4 files changed
+30
-3
docs/source/serving/compatibility_matrix.rst
+1
-1
@@ -283,7 +283,7 @@ Feature x Feature
283
- ✅
284
- ✅
285
- ✅
286
- - `✗ <https://github.com/vllm-project/vllm/issues/8985>`__
286
+ - `✗ <https://github.com/vllm-project/vllm/issues/9893>`__
287
- ?
288
- ✅
289
- ✅
tests/entrypoints/openai/test_prompt_validation.py
+20
@@ -35,3 +35,23 @@ async def test_out_of_vocab_token_ids():
35
prompt=[999999],
36
max_tokens=5,
37
temperature=0.0)
38
+
39
+
40
+@pytest.mark.asyncio
41
+async def test_reject_multistep_with_guided_decoding():
42
+ model_name = "gpt2"
43
+ server_args = ["--enforce-eager", "--num-scheduler-steps", "8"]
44
+ with RemoteOpenAIServer(model_name, server_args) as remote_server:
45
+ client = remote_server.get_async_client()
46
+
47
+ with pytest.raises(openai.BadRequestError,
48
+ match=re.compile(
49
+ '.*Guided decoding .* multi-step decoding.*')):
50
+ await client.completions.create(
51
+ model=model_name,
52
+ prompt="Hello",
53
+ max_tokens=5,
54
+ temperature=0.0,
55
+ extra_body={"response_format": {
56
+ "type": "json_object"
57
+ }})
vllm/engine/llm_engine.py
+7
@@ -829,6 +829,13 @@ class LLMEngine:
829
raise ValueError(f"Got priority {priority} but "
830
"Priority scheduling is not enabled.")
831
832
+ if isinstance(params, SamplingParams) \
833
+ and (params.guided_decoding or params.logits_processors) \
834
+ and self.scheduler_config.num_scheduler_steps > 1:
835
+ raise ValueError(
836
+ "Guided decoding and logits processors are not supported "
837
+ "in multi-step decoding")
838
+
839
if arrival_time is None:
840
arrival_time = time.time()
841
vllm/sampling_params.py
+2
-2
@@ -485,8 +485,8 @@ class SamplingParams(
485
f"skip_special_tokens={self.skip_special_tokens}, "
486
"spaces_between_special_tokens="
487
f"{self.spaces_between_special_tokens}, "
488
- f"truncate_prompt_tokens={self.truncate_prompt_tokens}), "
489
- f"guided_decoding={self.guided_decoding}")
488
+ f"truncate_prompt_tokens={self.truncate_prompt_tokens}, "
489
+ f"guided_decoding={self.guided_decoding})")
490
491
492
class BeamSearchParams(