A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bug]change the position of cuda_graph_sizes in dataclasses (#17548)
Signed-off-by: CXIAAAAA <cxia0209@gmail.com>
Chen Xia committed
May 1, 2025 at 11:52 UTC
173daac19dbf00180f1eb0752ce33fd97f48649d
1 file changed
+7
-7
vllm/config.py
+7
-7
@@ -1865,13 +1865,6 @@ class SchedulerConfig:
1865
This config has no static default. If left unspecified by the user, it will
1866
be set in `EngineArgs.create_engine_config` based on the usage context."""
1867
1868
- cuda_graph_sizes: list[int] = field(default_factory=lambda: [512])
1869
- """Cuda graph capture sizes, default is 512.
1870
- 1. if one value is provided, then the capture list would follow the pattern:
1871
- [1, 2, 4] + [i for i in range(8, cuda_graph_sizes + 1, 8)]
1872
- 2. more than one value (e.g. 1 2 128) is provided,
1873
- then the capture list will follow the provided list."""
1874
-
1868
max_num_seqs: int = None # type: ignore
1869
"""Maximum number of sequences to be processed in a single iteration.
1870
@@ -1906,6 +1899,13 @@ class SchedulerConfig:
1899
NOTE: This will be replaced by speculative config in the future; it is
1900
present to enable correctness tests until then."""
1901
1902
+ cuda_graph_sizes: list[int] = field(default_factory=lambda: [512])
1903
+ """Cuda graph capture sizes, default is 512.
1904
+ 1. if one value is provided, then the capture list would follow the pattern:
1905
+ [1, 2, 4] + [i for i in range(8, cuda_graph_sizes + 1, 8)]
1906
+ 2. more than one value (e.g. 1 2 128) is provided,
1907
+ then the capture list will follow the provided list."""
1908
+
1909
delay_factor: float = 0.0
1910
"""Apply a delay (of delay factor multiplied by previous
1911
prompt latency) before scheduling next prompt."""