A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Doc] Improve help examples for `--compilation-config` (#16729)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Apr 18, 2025 at 12:22 UTC
c16fb5dae88969f0e8b4ee3f9f7861fe91c8b2dc
3 files changed
+17
-8
docs/source/design/v1/torch_compile.md
+1
-1
@@ -134,6 +134,6 @@ The cudagraphs are captured and managed by the compiler backend, and replayed wh
134
135
By default, vLLM will try to determine a set of sizes to capture cudagraph. You can also override it using the config `cudagraph_capture_sizes`:
136
137
-`VLLM_USE_V1=1 vllm serve meta-llama/Llama-3.2-1B --compilation_config "{'cudagraph_capture_sizes': [1, 2, 4, 8]}"`
137
+`VLLM_USE_V1=1 vllm serve meta-llama/Llama-3.2-1B --compilation-config "{'cudagraph_capture_sizes': [1, 2, 4, 8]}"`
138
139
Then it will only capture cudagraph for the specified sizes. It can be useful to have fine-grained control over the cudagraph capture.
tests/engine/test_arg_utils.py
+12
-4
@@ -53,12 +53,20 @@ def test_compilation_config():
53
assert args.compilation_config.level == 3
54
55
# set to string form of a dict
56
- args = parser.parse_args(["--compilation-config", "{'level': 3}"])
57
- assert args.compilation_config.level == 3
56
+ args = parser.parse_args([
57
+ "--compilation-config",
58
+ "{'level': 3, 'cudagraph_capture_sizes': [1, 2, 4, 8]}",
59
+ ])
60
+ assert (args.compilation_config.level == 3 and
61
+ args.compilation_config.cudagraph_capture_sizes == [1, 2, 4, 8])
62
63
# set to string form of a dict
60
- args = parser.parse_args(["--compilation-config={'level': 3}"])
61
- assert args.compilation_config.level == 3
64
+ args = parser.parse_args([
65
+ "--compilation-config="
66
+ "{'level': 3, 'cudagraph_capture_sizes': [1, 2, 4, 8]}",
67
+ ])
68
+ assert (args.compilation_config.level == 3 and
69
+ args.compilation_config.cudagraph_capture_sizes == [1, 2, 4, 8])
70
71
72
def test_prefix_cache_default():
vllm/engine/arg_utils.py
+4
-3
@@ -939,10 +939,11 @@ class EngineArgs:
939
'testing only. level 3 is the recommended level '
940
'for production.\n'
941
'To specify the full compilation config, '
942
- 'use a JSON string.\n'
942
+ 'use a JSON string, e.g. ``{"level": 3, '
943
+ '"cudagraph_capture_sizes": [1, 2, 4, 8]}``\n'
944
'Following the convention of traditional '
944
- 'compilers, using -O without space is also '
945
- 'supported. -O3 is equivalent to -O 3.')
945
+ 'compilers, using ``-O`` without space is also '
946
+ 'supported. ``-O3`` is equivalent to ``-O 3``.')
947
948
parser.add_argument('--kv-transfer-config',
949
type=KVTransferConfig.from_cli,