@devries-cameron20 / OCIU-npo Mirror · read-only
Fork 0

Mirrored from https://github.com/Camo67/OCIU-npo.git

Synced Nov 21, 06:00 UTC

1 import psutil
2 import os
3 from enum import Enum
4 from typing import Optional
5
6 class TaskComplexity(Enum):
7 LIGHT = "light"
8 MID = "mid"
9 HEAVY = "heavy"
10
11 class ModelRouter:
12 def __init__(self):
13 self.ram_threshold = 0.8
14
15 def get_current_ram_usage(self):
16 # Allow override for testing
17 force_usage = os.getenv("FORCE_RAM_USAGE")
18 if force_usage:
19 return float(force_usage)
20 return psutil.virtual_memory().percent / 100
21
22 def select_model(self, complexity: TaskComplexity) -> str:
23 ram_usage = self.get_current_ram_usage()
24
25 # Check if high complexity task under high-ish load
26 if ram_usage > 0.75 and complexity == TaskComplexity.HEAVY:
27 return "qwen3:4b" # Fallback to mid model to save RAM
28
29 if ram_usage > self.ram_threshold:
30 return "cloud-fallback"
31
32 if complexity == TaskComplexity.LIGHT:
33 return "qwen3:0.5b"
34 elif complexity == TaskComplexity.MID:
35 return "qwen3:4b"
36 elif complexity == TaskComplexity.HEAVY:
37 return "qwen3:8b"
38
39 return "qwen3:1.8b"
40
41 def get_status(self):
42 return {
43 "ram_usage": self.get_current_ram_usage(),
44 "fallback_active": self.get_current_ram_usage() > self.ram_threshold
45 }
46
47 router = ModelRouter()