@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f6778620a9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Disable spec-decode + chunked-prefill for draft models with tensor parallelism > 1 (#10136)

Signed-off-by: Sourashis Roy <sroy@roblox.com>

sroy745 committed Nov 8, 2024 at 07:56 UTC f6778620a95baf925eb54694ab4666524d0d8584
2 files changed +83 -8
tests/spec_decode/e2e/test_compatibility.py
+46
@@ -50,3 +50,49 @@ def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
50 with pytest.raises(ValueError, match="cannot be larger than"):
51 get_output_from_llm_generator(test_llm_generator, prompts,
52 sampling_params)
53 +
54 +
55 +@pytest.mark.parametrize("common_llm_kwargs",
56 + [{
57 + "model": "meta-llama/Llama-2-7b-chat-hf",
58 + "speculative_model": "JackFram/llama-68m",
59 + "num_speculative_tokens": 5,
60 + "enable_chunked_prefill": "True",
61 + }])
62 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
63 + {
64 + "tensor_parallel_size": 2,
65 + "speculative_draft_tensor_parallel_size": 2,
66 + },
67 + {
68 + "tensor_parallel_size": 4,
69 + "speculative_draft_tensor_parallel_size": 4,
70 + },
71 + {
72 + "tensor_parallel_size": 8,
73 + "speculative_draft_tensor_parallel_size": 8,
74 + },
75 +])
76 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
77 +@pytest.mark.parametrize("seed", [1])
78 +def test_spec_decode_xfail_chunked_prefill_draft_model_tp_not_one(
79 + test_llm_generator):
80 + """Verify that speculative decoding fails if chunked prefill is enabled for
81 + draft model with tensor parallelism of more than 1.
82 + """
83 + output_len = 128
84 + temperature = 0.0
85 +
86 + prompts = [
87 + "Hello, my name is",
88 + ]
89 +
90 + sampling_params = SamplingParams(
91 + max_tokens=output_len,
92 + ignore_eos=True,
93 + temperature=temperature,
94 + )
95 +
96 + with pytest.raises(ValueError, match="with tensor parallel size 1"):
97 + get_output_from_llm_generator(test_llm_generator, prompts,
98 + sampling_params)
vllm/config.py
+37 -8
@@ -1388,6 +1388,23 @@ class SpeculativeConfig:
1388 "Chunked prefill and hidden-state based draft models are "
1389 "not compatible.")
1390
1391 + speculative_draft_tensor_parallel_size = \
1392 + SpeculativeConfig._verify_and_get_draft_model_tensor_parallel_size(
1393 + target_parallel_config,
1394 + speculative_draft_tensor_parallel_size,
1395 + draft_hf_config
1396 + )
1397 +
1398 + if (enable_chunked_prefill and \
1399 + speculative_draft_tensor_parallel_size != 1):
1400 + # TODO - Investigate why the error reported in
1401 + # https://github.com/vllm-project/vllm/pull/9291#issuecomment-2463266258
1402 + # is happening and re-enable it.
1403 + raise ValueError(
1404 + "Chunked prefill and speculative decoding can be enabled "
1405 + "simultaneously only for draft models with tensor "
1406 + "parallel size 1.")
1407 +
1408 draft_model_config.max_model_len = (
1409 SpeculativeConfig._maybe_override_draft_max_model_len(
1410 speculative_max_model_len,
@@ -1466,15 +1483,16 @@ class SpeculativeConfig:
1483 )
1484
1485 @staticmethod
1469 - def create_draft_parallel_config(
1470 - target_parallel_config: ParallelConfig,
1471 - speculative_draft_tensor_parallel_size: Optional[int],
1472 - draft_hf_config: PretrainedConfig,
1473 - ) -> ParallelConfig:
1474 - """Create a parallel config for use by the draft worker.
1475 -
1476 - This is mostly a copy of the target parallel config, except the tp_size.
1486 + def _verify_and_get_draft_model_tensor_parallel_size(
1487 + target_parallel_config: ParallelConfig,
1488 + speculative_draft_tensor_parallel_size: Optional[int],
1489 + draft_hf_config: PretrainedConfig) -> int:
1490 + """
1491 + Verifies and adjusts the tensor parallel size for a draft model
1492 + specified using speculative_draft_tensor_parallel_size.
1493 """
1494 + # If speculative_draft_tensor_parallel_size is unset then set it
1495 + # appropriately else verify that it is set correctly.
1496 if speculative_draft_tensor_parallel_size is None:
1497 if draft_hf_config.model_type == "mlp_speculator":
1498 speculative_draft_tensor_parallel_size = 1
@@ -1490,7 +1508,18 @@ class SpeculativeConfig:
1508 raise ValueError(
1509 f"{speculative_draft_tensor_parallel_size=} cannot be "
1510 f"other value than 1 or target model tensor_parallel_size")
1511 + return speculative_draft_tensor_parallel_size
1512
1513 + @staticmethod
1514 + def create_draft_parallel_config(
1515 + target_parallel_config: ParallelConfig,
1516 + speculative_draft_tensor_parallel_size: int,
1517 + draft_hf_config: PretrainedConfig,
1518 + ) -> ParallelConfig:
1519 + """Create a parallel config for use by the draft worker.
1520 +
1521 + This is mostly a copy of the target parallel config, except the tp_size.
1522 + """
1523 draft_parallel_config = ParallelConfig(
1524 pipeline_parallel_size=target_parallel_config.
1525 pipeline_parallel_size,