@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b67ae00cdb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Add quantization config support for speculative model. (#7343)

shangmingc committed Aug 16, 2024 at 10:34 UTC b67ae00cdbbe1a58ffc8ff170f0c8d79044a684a
3 files changed +71 -4
tests/spec_decode/e2e/test_integration.py
+48
@@ -42,3 +42,51 @@ def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
42 max_output_len=output_len,
43 force_output_len=True,
44 )
45 +
46 +
47 +@pytest.mark.parametrize(
48 + "common_llm_kwargs",
49 + [{
50 + "model": "JackFram/llama-160m",
51 +
52 + # Skip cuda graph recording for fast test.
53 + "enforce_eager": True,
54 +
55 + # Required for spec decode.
56 + "use_v2_block_manager": True,
57 + }])
58 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [
59 + {
60 + "speculative_model": "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
61 + "num_speculative_tokens": 5,
62 + },
63 +])
64 +@pytest.mark.parametrize(
65 + "test_llm_kwargs",
66 + [
67 + # Explicitly specify draft model quantization
68 + {
69 + "speculative_model_quantization": "gptq",
70 + },
71 + # Explicitly specify GPTQ-based draft model to use marlin quantization
72 + {
73 + "speculative_model_quantization": "marlin",
74 + },
75 + # Not explicitly specify draft model quantization
76 + {
77 + "speculative_model_quantization": None,
78 + },
79 + ])
80 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
81 +@pytest.mark.parametrize("batch_size", [2])
82 +@pytest.mark.parametrize("seed", [1])
83 +def test_speculative_model_quantization_config(baseline_llm_generator,
84 + test_llm_generator,
85 + batch_size: int):
86 + """Verify spec decode works well with draft model quantization configs.
87 + """
88 + run_greedy_equality_correctness_test(baseline_llm_generator,
89 + test_llm_generator,
90 + batch_size,
91 + max_output_len=32,
92 + force_output_len=True)
vllm/config.py
+8 -4
@@ -961,6 +961,7 @@ class SpeculativeConfig:
961 target_parallel_config: ParallelConfig,
962 target_dtype: str,
963 speculative_model: Optional[str],
964 + speculative_model_quantization: Optional[str],
965 speculative_draft_tensor_parallel_size: Optional[int],
966 num_speculative_tokens: Optional[int],
967 speculative_max_model_len: Optional[int],
@@ -989,6 +990,9 @@ class SpeculativeConfig:
990 target_dtype (str): The data type used for the target model.
991 speculative_model (Optional[str]): The name of the speculative
992 model, if provided.
993 + speculative_model_quantization (Optional[str]): Quantization method
994 + that was used to quantize the speculative model weights. If
995 + None, we assume the model weights are not quantized.
996 speculative_draft_tensor_parallel_size (Optional[int]): The degree
997 of the tensor parallelism for the draft model.
998 num_speculative_tokens (Optional[int]): The number of speculative
@@ -1056,11 +1060,11 @@ class SpeculativeConfig:
1060 "Speculative decoding requires usage of the V2 "
1061 "block manager. Enable it with --use-v2-block-manager.")
1062
1059 - # TODO: The user should be able to specify revision/quantization/max
1060 - # model len for the draft model. It is not currently supported.
1063 + # TODO: The user should be able to specify revision/max model len
1064 + # for the draft model. It is not currently supported.
1065 draft_revision = None
1066 draft_code_revision = None
1063 - draft_quantization = None
1067 + draft_quantization = speculative_model_quantization
1068
1069 if speculative_model == "[ngram]":
1070 if ngram_prompt_lookup_min is None:
@@ -1217,7 +1221,7 @@ class SpeculativeConfig:
1221 elif speculative_draft_tensor_parallel_size != 1:
1222 # TODO(wooyeon): allow tp values larger than 1
1223 raise ValueError(
1220 - f"{speculative_draft_tensor_parallel_size=} cannot be"
1224 + f"{speculative_draft_tensor_parallel_size=} cannot be "
1225 f"other value than 1")
1226
1227 draft_parallel_config = ParallelConfig(
vllm/engine/arg_utils.py
+15
@@ -129,6 +129,7 @@ class EngineArgs:
129 guided_decoding_backend: str = 'outlines'
130 # Speculative decoding configuration.
131 speculative_model: Optional[str] = None
132 + speculative_model_quantization: Optional[str] = None
133 speculative_draft_tensor_parallel_size: Optional[int] = None
134 num_speculative_tokens: Optional[int] = None
135 speculative_max_model_len: Optional[int] = None
@@ -571,6 +572,18 @@ class EngineArgs:
572 default=EngineArgs.speculative_model,
573 help=
574 'The name of the draft model to be used in speculative decoding.')
575 + # Quantization settings for speculative model.
576 + parser.add_argument(
577 + '--speculative-model-quantization',
578 + type=nullable_str,
579 + choices=[*QUANTIZATION_METHODS, None],
580 + default=EngineArgs.speculative_model_quantization,
581 + help='Method used to quantize the weights of speculative model.'
582 + 'If None, we first check the `quantization_config` '
583 + 'attribute in the model config file. If that is '
584 + 'None, we assume the model weights are not '
585 + 'quantized and use `dtype` to determine the data '
586 + 'type of the weights.')
587 parser.add_argument(
588 '--num-speculative-tokens',
589 type=int,
@@ -844,6 +857,8 @@ class EngineArgs:
857 target_parallel_config=parallel_config,
858 target_dtype=self.dtype,
859 speculative_model=self.speculative_model,
860 + speculative_model_quantization = \
861 + self.speculative_model_quantization,
862 speculative_draft_tensor_parallel_size = \
863 self.speculative_draft_tensor_parallel_size,
864 num_speculative_tokens=self.num_speculative_tokens,