A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[torch.compile] fix shape specialization (#10722)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 27, 2024 at 10:16 UTC
c411def234b0e85a349c8d95b5f32eade4aa1ed6
1 file changed
+3
-4
vllm/config.py
+3
-4
@@ -2151,7 +2151,7 @@ class CompilationConfig(BaseModel):
2151
2152
use_inductor: bool = True
2153
inductor_specialize_for_cudagraph_no_more_than: Optional[int] = None
2154
- inductor_compile_sizes: Optional[List[int]] = Field(default_factory=dict)
2154
+ inductor_compile_sizes: Optional[List[int]] = Field(default=None)
2155
inductor_compile_config: Dict = Field(default_factory=dict)
2156
inductor_passes: Dict[str, str] = Field(default_factory=dict)
2157
@@ -2290,9 +2290,8 @@ class CompilationConfig(BaseModel):
2290
if x <= self.inductor_specialize_for_cudagraph_no_more_than
2291
]
2292
else:
2293
- assert self.inductor_compile_sizes is not None, (
2294
- "inductor_compile_sizes should not be None when "
2295
- "inductor_specialize_for_cudagraph_no_more_than is None")
2293
+ if self.inductor_compile_sizes is None:
2294
+ self.inductor_compile_sizes = []
2295
self.compile_sizes = self.inductor_compile_sizes
2296
2297