A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Disable spec-decode + chunked-prefill for draft models with tensor parallelism > 1 (#10136)
Signed-off-by: Sourashis Roy <sroy@roblox.com>
sroy745 committed
Nov 8, 2024 at 07:56 UTC
f6778620a95baf925eb54694ab4666524d0d8584
2 files changed
+83
-8
tests/spec_decode/e2e/test_compatibility.py
+46
@@ -50,3 +50,49 @@ def test_spec_decode_xfail_spec_max_model_len(test_llm_generator):
50
with pytest.raises(ValueError, match="cannot be larger than"):
51
get_output_from_llm_generator(test_llm_generator, prompts,
52
sampling_params)
53
+
54
+
55
+@pytest.mark.parametrize("common_llm_kwargs",
56
+ [{
57
+ "model": "meta-llama/Llama-2-7b-chat-hf",
58
+ "speculative_model": "JackFram/llama-68m",
59
+ "num_speculative_tokens": 5,
60
+ "enable_chunked_prefill": "True",
61
+ }])
62
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [
63
+ {
64
+ "tensor_parallel_size": 2,
65
+ "speculative_draft_tensor_parallel_size": 2,
66
+ },
67
+ {
68
+ "tensor_parallel_size": 4,
69
+ "speculative_draft_tensor_parallel_size": 4,
70
+ },
71
+ {
72
+ "tensor_parallel_size": 8,
73
+ "speculative_draft_tensor_parallel_size": 8,
74
+ },
75
+])
76
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
77
+@pytest.mark.parametrize("seed", [1])
78
+def test_spec_decode_xfail_chunked_prefill_draft_model_tp_not_one(
79
+ test_llm_generator):
80
+ """Verify that speculative decoding fails if chunked prefill is enabled for
81
+ draft model with tensor parallelism of more than 1.
82
+ """
83
+ output_len = 128
84
+ temperature = 0.0
85
+
86
+ prompts = [
87
+ "Hello, my name is",
88
+ ]
89
+
90
+ sampling_params = SamplingParams(
91
+ max_tokens=output_len,
92
+ ignore_eos=True,
93
+ temperature=temperature,
94
+ )
95
+
96
+ with pytest.raises(ValueError, match="with tensor parallel size 1"):
97
+ get_output_from_llm_generator(test_llm_generator, prompts,
98
+ sampling_params)
vllm/config.py
+37
-8
@@ -1388,6 +1388,23 @@ class SpeculativeConfig:
1388
"Chunked prefill and hidden-state based draft models are "
1389
"not compatible.")
1390
1391
+ speculative_draft_tensor_parallel_size = \
1392
+ SpeculativeConfig._verify_and_get_draft_model_tensor_parallel_size(
1393
+ target_parallel_config,
1394
+ speculative_draft_tensor_parallel_size,
1395
+ draft_hf_config
1396
+ )
1397
+
1398
+ if (enable_chunked_prefill and \
1399
+ speculative_draft_tensor_parallel_size != 1):
1400
+ # TODO - Investigate why the error reported in
1401
+ # https://github.com/vllm-project/vllm/pull/9291#issuecomment-2463266258
1402
+ # is happening and re-enable it.
1403
+ raise ValueError(
1404
+ "Chunked prefill and speculative decoding can be enabled "
1405
+ "simultaneously only for draft models with tensor "
1406
+ "parallel size 1.")
1407
+
1408
draft_model_config.max_model_len = (
1409
SpeculativeConfig._maybe_override_draft_max_model_len(
1410
speculative_max_model_len,
@@ -1466,15 +1483,16 @@ class SpeculativeConfig:
1483
)
1484
1485
@staticmethod
1469
- def create_draft_parallel_config(
1470
- target_parallel_config: ParallelConfig,
1471
- speculative_draft_tensor_parallel_size: Optional[int],
1472
- draft_hf_config: PretrainedConfig,
1473
- ) -> ParallelConfig:
1474
- """Create a parallel config for use by the draft worker.
1475
-
1476
- This is mostly a copy of the target parallel config, except the tp_size.
1486
+ def _verify_and_get_draft_model_tensor_parallel_size(
1487
+ target_parallel_config: ParallelConfig,
1488
+ speculative_draft_tensor_parallel_size: Optional[int],
1489
+ draft_hf_config: PretrainedConfig) -> int:
1490
+ """
1491
+ Verifies and adjusts the tensor parallel size for a draft model
1492
+ specified using speculative_draft_tensor_parallel_size.
1493
"""
1494
+ # If speculative_draft_tensor_parallel_size is unset then set it
1495
+ # appropriately else verify that it is set correctly.
1496
if speculative_draft_tensor_parallel_size is None:
1497
if draft_hf_config.model_type == "mlp_speculator":
1498
speculative_draft_tensor_parallel_size = 1
@@ -1490,7 +1508,18 @@ class SpeculativeConfig:
1508
raise ValueError(
1509
f"{speculative_draft_tensor_parallel_size=} cannot be "
1510
f"other value than 1 or target model tensor_parallel_size")
1511
+ return speculative_draft_tensor_parallel_size
1512
1513
+ @staticmethod
1514
+ def create_draft_parallel_config(
1515
+ target_parallel_config: ParallelConfig,
1516
+ speculative_draft_tensor_parallel_size: int,
1517
+ draft_hf_config: PretrainedConfig,
1518
+ ) -> ParallelConfig:
1519
+ """Create a parallel config for use by the draft worker.
1520
+
1521
+ This is mostly a copy of the target parallel config, except the tp_size.
1522
+ """
1523
draft_parallel_config = ParallelConfig(
1524
pipeline_parallel_size=target_parallel_config.
1525
pipeline_parallel_size,