A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 00:15 UTC
[torch.compile] Rename `compile_ranges_split_points` to `compile_ranges_endpoints` (#36027)
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ProExpertProg <11367180+ProExpertProg@users.noreply.github.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
Copilot committed
Mar 9, 2026 at 18:04 UTC
4b87ffbefb3881a0a33f9c1cb7121429bddad666
5 files changed
+30
-31
tests/compile/fusions_e2e/conftest.py
+3
-3
@@ -46,10 +46,10 @@ def run_model(compile_config: int | CompilationConfig, model: str, **model_kwarg
46
generated_text = output.outputs[0].text
47
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
48
49
- # Get the compile ranges split points after vllm config post init
49
+ # Get the compile ranges endpoints after vllm config post init
50
# in order to compute compile ranges correctly
51
- compilation_config.compile_ranges_split_points = (
52
- llm.llm_engine.vllm_config.compilation_config.compile_ranges_split_points
51
+ compilation_config.compile_ranges_endpoints = (
52
+ llm.llm_engine.vllm_config.compilation_config.compile_ranges_endpoints
53
)
54
55
tests/compile/test_compile_ranges.py
+3
-3
@@ -86,7 +86,7 @@ def test_compile_ranges(use_fresh_inductor_cache):
86
),
87
compilation_config=CompilationConfig(
88
mode=CompilationMode.VLLM_COMPILE,
89
- compile_ranges_split_points=[8, 32],
89
+ compile_ranges_endpoints=[8, 32],
90
compile_sizes=[16, 64, 128],
91
inductor_compile_config={
92
"post_grad_custom_post_pass": post_grad_range_checker,
@@ -110,7 +110,7 @@ def test_compile_ranges(use_fresh_inductor_cache):
110
111
def test_compile_config_get_compile_ranges():
112
compilation_config = CompilationConfig(
113
- compile_ranges_split_points=[8, 32],
113
+ compile_ranges_endpoints=[8, 32],
114
)
115
VllmConfig(
116
scheduler_config=SchedulerConfig(
@@ -149,7 +149,7 @@ def test_inductor_cache_compile_ranges(monkeypatch, use_fresh_inductor_cache):
149
scheduler_config=scheduler_config,
150
compilation_config=CompilationConfig(
151
mode=CompilationMode.VLLM_COMPILE,
152
- compile_ranges_split_points=[8],
152
+ compile_ranges_endpoints=[8],
153
inductor_compile_config={
154
"post_grad_custom_post_pass": post_grad_range_checker,
155
},
vllm/compilation/backends.py
+2
-2
@@ -885,8 +885,8 @@ class VllmBackend:
885
"splitting_ops": list_to_str(cc.splitting_ops),
886
"cudagraph_mode": str(cc.cudagraph_mode),
887
"compile_sizes": list_to_str(cc.compile_sizes),
888
- "compile_ranges_split_points": list_to_str(
889
- cc.compile_ranges_split_points
888
+ "compile_ranges_endpoints": list_to_str(
889
+ cc.compile_ranges_endpoints
890
),
891
"use_inductor_graph_partition": cc.use_inductor_graph_partition,
892
"inductor_passes": list_to_str(list(cc.inductor_passes.keys())),
vllm/config/compilation.py
+10
-11
@@ -382,8 +382,8 @@ class CompilationConfig:
382
[vllm.config.CompilationConfig.cudagraph_copy_inputs]
383
- Inductor compilation:
384
- [`compile_sizes`][vllm.config.CompilationConfig.compile_sizes]
385
- - [`compile_ranges_split_points`]
386
- [vllm.config.CompilationConfig.compile_ranges_split_points]
385
+ - [`compile_ranges_endpoints`]
386
+ [vllm.config.CompilationConfig.compile_ranges_endpoints]
387
- [`inductor_compile_config`]
388
[vllm.config.CompilationConfig.inductor_compile_config]
389
- [`inductor_passes`][vllm.config.CompilationConfig.inductor_passes]
@@ -492,12 +492,12 @@ class CompilationConfig:
492
to integers, it also supports "cudagraph_capture_sizes" to
493
specify the sizes for cudagraph capture."""
494
495
- compile_ranges_split_points: list[int] | None = None
496
- """Split points that represent compile ranges for inductor.
495
+ compile_ranges_endpoints: list[int] | None = None
496
+ """Endpoints for Inductor compile ranges.
497
The compile ranges are
498
- [1, split_points[0]],
499
- [split_points[0] + 1, split_points[1]], ...,
500
- [split_points[-1] + 1, max_num_batched_tokens].
498
+ [1, endpoints[0]],
499
+ [endpoints[0] + 1, endpoints[1]], ...,
500
+ [endpoints[-1] + 1, max_num_batched_tokens].
501
Compile sizes are also used single element ranges,
502
the range is represented as [compile_sizes[i], compile_sizes[i]].
503
@@ -1246,10 +1246,9 @@ class CompilationConfig:
1246
1247
def get_compile_ranges(self) -> list[Range]:
1248
"""Get the compile ranges for the compilation config."""
1249
- if self.compile_ranges_split_points is None:
1249
+ if self.compile_ranges_endpoints is None:
1250
return []
1251
- split_points = sorted(set(self.compile_ranges_split_points))
1251
+ endpoints = sorted(set(self.compile_ranges_endpoints))
1252
return [
1253
- Range(start=s + 1, end=e)
1254
- for s, e in zip([0] + split_points[:-1], split_points)
1253
+ Range(start=s + 1, end=e) for s, e in zip([0] + endpoints[:-1], endpoints)
1254
]
vllm/config/vllm.py
+12
-12
@@ -1451,12 +1451,12 @@ class VllmConfig:
1451
Set the compile ranges for the compilation config.
1452
"""
1453
compilation_config = self.compilation_config
1454
- computed_compile_ranges_split_points = []
1454
+ computed_compile_ranges_endpoints = []
1455
1456
# The upper bound of the compile ranges is the max_num_batched_tokens.
1457
compile_range_end = self.scheduler_config.max_num_batched_tokens
1458
if compile_range_end is not None:
1459
- computed_compile_ranges_split_points.append(compile_range_end)
1459
+ computed_compile_ranges_endpoints.append(compile_range_end)
1460
1461
# Add the compile ranges for flashinfer
1462
if compilation_config.pass_config.fuse_allreduce_rms:
@@ -1468,7 +1468,7 @@ class VllmConfig:
1468
* self.model_config.dtype.itemsize
1469
)
1470
if compile_range_end is not None and max_token_num < compile_range_end:
1471
- computed_compile_ranges_split_points.append(max_token_num)
1471
+ computed_compile_ranges_endpoints.append(max_token_num)
1472
else:
1473
logger.debug(
1474
"Max num batched tokens below allreduce-rms fusion threshold, "
@@ -1500,10 +1500,10 @@ class VllmConfig:
1500
and min_token_num < max_num_batched_tokens
1501
and min_token_num > 1
1502
):
1503
- # Add split point at min_token_num - 1 to ensure SP applies
1503
+ # Add endpoint at min_token_num - 1 to ensure SP applies
1504
# starting from min_token_num
1505
# This creates ranges: [1, min-1] (no SP), [min, max] (SP applies)
1506
- computed_compile_ranges_split_points.append(min_token_num - 1)
1506
+ computed_compile_ranges_endpoints.append(min_token_num - 1)
1507
1508
if compilation_config.pass_config.fuse_rope_kvcache:
1509
max_token_num = (
@@ -1511,7 +1511,7 @@ class VllmConfig:
1511
)
1512
if max_token_num is not None:
1513
if compile_range_end is not None and max_token_num < compile_range_end:
1514
- computed_compile_ranges_split_points.append(max_token_num)
1514
+ computed_compile_ranges_endpoints.append(max_token_num)
1515
else:
1516
logger.debug(
1517
"Max num batched tokens below rope+kvcache fusion threshold, "
@@ -1519,14 +1519,14 @@ class VllmConfig:
1519
compile_range_end,
1520
)
1521
1522
- if compilation_config.compile_ranges_split_points is not None:
1523
- for x in compilation_config.compile_ranges_split_points:
1522
+ if compilation_config.compile_ranges_endpoints is not None:
1523
+ for x in compilation_config.compile_ranges_endpoints:
1524
assert isinstance(x, int)
1525
- assert x > 0, f"Invalid compile range split point: {x}"
1525
+ assert x > 0, f"Invalid compile range endpoint: {x}"
1526
if compile_range_end is not None and x < compile_range_end and x > 1:
1527
- computed_compile_ranges_split_points.append(x)
1528
- compilation_config.compile_ranges_split_points = sorted(
1529
- computed_compile_ranges_split_points
1527
+ computed_compile_ranges_endpoints.append(x)
1528
+ compilation_config.compile_ranges_endpoints = sorted(
1529
+ computed_compile_ranges_endpoints
1530
)
1531
1532
def try_verify_and_update_config(self):