[Bugfix][KV Offload] Namespace auto cache dtype by effective dtype (#49438)
Signed-off-by: Jonguk Cheong <jdal3031@snu.ac.kr> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
Jonguk Cheong committed
Jul 27, 2026 at 02:59 UTC
b68d7ef2622d2d22e964dd842381021865e942b8
1 file changed
+7
-1
vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py
+7
-1
@@ -138,6 +138,12 @@ def build_offloading_config(
138
)
139
140
kv_events_config = vllm_config.kv_events_config
141
+ cache_dtype = (
142
+ vllm_config.model_config.dtype
143
+ if vllm_config.cache_config.cache_dtype == "auto"
144
+ else vllm_config.cache_config.cache_dtype
145
+ )
146
+
147
return OffloadingConfig(
148
groups=groups,
149
worker_kv_bytes_per_block=worker_kv_bytes_per_block,
@@ -148,7 +154,7 @@ def build_offloading_config(
154
engine_id=engine_id,
155
model=OffloadingModelConfig(
156
name=vllm_config.model_config.model,
151
- dtype=str(vllm_config.cache_config.cache_dtype).replace("torch.", ""),
157
+ dtype=str(cache_dtype).removeprefix("torch."),
158
),
159
cache=OffloadingCacheConfig(
160
tokens_per_hash=tokens_per_hash,