@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c411def234
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[torch.compile] fix shape specialization (#10722)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 27, 2024 at 10:16 UTC c411def234b0e85a349c8d95b5f32eade4aa1ed6
1 file changed +3 -4
vllm/config.py
+3 -4
@@ -2151,7 +2151,7 @@ class CompilationConfig(BaseModel):
2151
2152 use_inductor: bool = True
2153 inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
2154 - inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
2154 + inductor_compile_sizes: Optional[List[int]] = Field(default=None)
2155 inductor_compile_config: Dict = Field(default_factory=dict)
2156 inductor_passes: Dict[str, str] = Field(default_factory=dict)
2157
@@ -2290,9 +2290,8 @@ class CompilationConfig(BaseModel):
2290 if x <= self.inductor_specialize_for_cudagraph_no_more_than
2291 ]
2292 else:
2293 - assert self.inductor_compile_sizes is not None, (
2294 - "inductor_compile_sizes should not be None when "
2295 - "inductor_specialize_for_cudagraph_no_more_than is None")
2293 + if self.inductor_compile_sizes is None:
2294 + self.inductor_compile_sizes = []
2295 self.compile_sizes = self.inductor_compile_sizes
2296
2297