feat(settings): add LiteLLM global params
linuztx committed
Aug 17, 2025 at 22:02 UTC
7f66fadb9d78ad7a464e802678ae651df039a7f7
3 files changed
+76
-1
models.py
+38
-1
@@ -18,6 +18,7 @@ from litellm import completion, acompletion, embedding
18
import litellm
19
20
from python.helpers import dotenv
21
+from python.helpers import settings
22
from python.helpers.dotenv import load_dotenv
23
from python.helpers.providers import get_provider_config
24
from python.helpers.rate_limiter import RateLimiter
@@ -434,7 +435,18 @@ class LocalSentenceTransformerWrapper(Embeddings):
435
if model.startswith("sentence-transformers/"):
436
model = model[len("sentence-transformers/") :]
437
437
- self.model = SentenceTransformer(model, **kwargs)
438
+ # Filter kwargs for SentenceTransformer only (no LiteLLM params like 'stream_timeout')
439
+ st_allowed_keys = {
440
+ "device",
441
+ "cache_folder",
442
+ "use_auth_token",
443
+ "revision",
444
+ "trust_remote_code",
445
+ "model_kwargs",
446
+ }
447
+ st_kwargs = {k: v for k, v in (kwargs or {}).items() if k in st_allowed_keys}
448
+
449
+ self.model = SentenceTransformer(model, **st_kwargs)
450
self.model_name = model
451
self.a0_model_conf = model_config
452
@@ -542,6 +554,22 @@ def _adjust_call_args(provider_name: str, model_name: str, kwargs: dict):
554
def _merge_provider_defaults(
555
provider_type: str, original_provider: str, kwargs: dict
556
) -> tuple[str, dict]:
557
+ # Normalize .env-style numeric strings (e.g., "timeout=30") into ints/floats for LiteLLM
558
+ def _normalize_values(values: dict) -> dict:
559
+ result: dict[str, Any] = {}
560
+ for k, v in values.items():
561
+ if isinstance(v, str):
562
+ try:
563
+ result[k] = int(v)
564
+ except ValueError:
565
+ try:
566
+ result[k] = float(v)
567
+ except ValueError:
568
+ result[k] = v
569
+ else:
570
+ result[k] = v
571
+ return result
572
+
573
provider_name = original_provider # default: unchanged
574
cfg = get_provider_config(provider_type, original_provider)
575
if cfg:
@@ -559,6 +587,15 @@ def _merge_provider_defaults(
587
if key and key not in ("None", "NA"):
588
kwargs["api_key"] = key
589
590
+ # Merge LiteLLM global kwargs (timeouts, stream_timeout, etc.)
591
+ try:
592
+ global_kwargs = settings.get_settings().get("litellm_global_kwargs", {}) # type: ignore[union-attr]
593
+ except Exception:
594
+ global_kwargs = {}
595
+ if isinstance(global_kwargs, dict):
596
+ for k, v in _normalize_values(global_kwargs).items():
597
+ kwargs.setdefault(k, v)
598
+
599
return provider_name, kwargs
600
601
python/helpers/settings.py
+27
@@ -105,6 +105,9 @@ class Settings(TypedDict):
105
106
secrets: str
107
108
+ # LiteLLM global kwargs applied to all model calls
109
+ litellm_global_kwargs: dict[str, str]
110
+
111
class PartialSettings(Settings, total=False):
112
pass
113
@@ -582,6 +585,28 @@ def convert_out(settings: Settings) -> SettingsOutput:
585
"tab": "external",
586
}
587
588
+ # LiteLLM global config section
589
+ litellm_fields: list[SettingsField] = []
590
+
591
+ litellm_fields.append(
592
+ {
593
+ "id": "litellm_global_kwargs",
594
+ "title": "LiteLLM global parameters",
595
+ "description": "Global LiteLLM params (e.g. timeout, stream_timeout) in .env format: one KEY=VALUE per line. Example: <code>stream_timeout=30</code>. Applied to all LiteLLM calls unless overridden. See <a href='https://docs.litellm.ai/docs/set_keys' target='_blank'>LiteLLM</a> and <a href='https://docs.litellm.ai/docs/proxy/timeout' target='_blank'>timeouts</a>.",
596
+ "type": "textarea",
597
+ "value": _dict_to_env(settings["litellm_global_kwargs"]),
598
+ "style": "height: 12em",
599
+ }
600
+ )
601
+
602
+ litellm_section: SettingsSection = {
603
+ "id": "litellm",
604
+ "title": "LiteLLM Global Settings",
605
+ "description": "Configure global parameters passed to LiteLLM for all providers.",
606
+ "fields": litellm_fields,
607
+ "tab": "external",
608
+ }
609
+
610
# Agent config section
611
agent_fields: list[SettingsField] = []
612
@@ -1205,6 +1230,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
1230
memory_section,
1231
speech_section,
1232
api_keys_section,
1233
+ litellm_section,
1234
secrets_section,
1235
auth_section,
1236
mcp_client_section,
@@ -1441,6 +1467,7 @@ def get_default_settings() -> Settings:
1467
mcp_server_token=create_auth_token(),
1468
a2a_server_enabled=False,
1469
secrets="",
1470
+ litellm_global_kwargs={},
1471
)
1472
1473
webui/public/litellm.svg
new
+11
@@ -0,0 +1,11 @@
1
+
2
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512">
3
+ <g stroke="#333" stroke-width="7" stroke-miterlimit="10" fill="#000" fill-opacity="0.08">
4
+ <path d="M511.9 82.5c-.8-.1-1.4-.1-1.9-.1-81.6 0-163.2 0-244.8.1-8.5 0-16.6 2-24.4 5.4-15.7 6.9-27.4 18.8-39 30.7 10.8 0 21.6.1 32.4.1h126.3c1.5 0 2.9 0 4.9 1l-11.7 11.7c-18.7 18.6-37.5 37.1-56.1 55.8-3.4 3.4-6.8 4.9-11.7 4.9-50.7-.2-101.3-.1-152-.1-1.8 0-3.6-.1-5.5-.1-18.4 9.3-36.9 18.7-55.3 28 0 49.8 0 99.7-.1 149.5 0 1.8-.1 3.6-.2 5.5h110.3l69.5-69.8c2.3-2.4 4.7-3.4 8.1-3.4h251c2.1 0 4.2-.1 6.3-.1V85c.2-.8.1-1.6.1-2.5Z"/>
5
+ <path d="M183.2 374.8 252.9 305c2.3-2.4 4.7-3.4 8.1-3.4h251c2.1 0 4.2-.1 6.3-.1v73.2H296.7H237.3"/>
6
+ <path d="M30.5 243.1C4.4 260.5-4 286.2 1.7 316.2c6.5 34.3 36.1 58 71.2 58.7V225.4l-42.4 17.7Z"/>
7
+ <path d="M184.8 171.6h164.5c17.4-17.3 34.9-34.5 52.1-51.9"/>
8
+ <path d="M511.8 429.6H19c18-18.2 35.9-36.4 53.9-54.6h224.1 209.7v36.5Z"/>
9
+ <path d="M365.4 119.6H234.2c-10.8 0-21.6-.1-32.4-.1l-17 16.8h164.8c5.4-5.4 9.4-9.4 16.8-16.7Z"/>
10
+ </g>
11
+</svg>