A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[bugfix] fix inductor cache on max_position_embeddings (#15436)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Mar 25, 2025 at 22:05 UTC
d0cfec7ab919bfab261db419e17c768f08a24dc8
1 file changed
+3
vllm/config.py
+3
@@ -221,6 +221,9 @@ class ModelConfig:
221
factors.append(self.trust_remote_code)
222
factors.append(self.rope_scaling)
223
factors.append(self.rope_theta)
224
+ # rope cos/sin cache depends on the max_position_embeddings
225
+ factors.append(
226
+ getattr(self.hf_config, "max_position_embeddings", "None"))
227
return hashlib.sha256(str(factors).encode()).hexdigest()
228
229
def __init__(