A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
Fix vLLM x torch.compile config caching (#16491)
Signed-off-by: rzou <zou3519@gmail.com>
Richard Zou committed
Apr 15, 2025 at 02:11 UTC
b590adfdc15fc716f6d120aeefeb587f491f8fce
1 file changed
+9
-3
vllm/config.py
+9
-3
@@ -298,12 +298,18 @@ class ModelConfig:
298
factors.append(self.quantization)
299
factors.append(self.revision)
300
factors.append(self.code_revision)
301
+ factors.append(self.max_model_len)
302
+ factors.append(self.max_logprobs)
303
+ factors.append(self.disable_sliding_window)
304
factors.append(self.trust_remote_code)
305
+ factors.append(self.mm_processor_kwargs)
306
+ factors.append(self.generation_config)
307
+ factors.append(self.model_impl)
308
+ factors.append(self.override_generation_config)
309
factors.append(self.rope_scaling)
310
factors.append(self.rope_theta)
304
- # rope cos/sin cache depends on the max_position_embeddings
305
- factors.append(
306
- getattr(self.hf_config, "max_position_embeddings", "None"))
311
+ # hf_config can control how the model looks!
312
+ factors.append(self.hf_config.to_json_string())
313
return hashlib.sha256(str(factors).encode()).hexdigest()
314
315
def __init__(