Free & local
Bugfixes for free and local models
frdel committed
Sep 13, 2024 at 19:41 UTC
b73f29a3484dcb11560984e5bdc4414090644ca8
4 files changed
+47
-33
example.env
+1
-1
@@ -1,4 +1,4 @@
1
-API_KEY_OPENAI=sk-hyBlbkFJCJjaYGCbqPTyT3uaYGCbqFBlbkFJCyJCyuPhYGCb
1
+API_KEY_OPENAI=
2
API_KEY_ANTHROPIC=
3
API_KEY_GROQ=
4
API_KEY_PERPLEXITY=
initialize.py
+3
-2
@@ -6,8 +6,8 @@ def initialize():
6
# main chat model used by agents (smarter, more accurate)
7
chat_llm = models.get_openai_chat(model_name="gpt-4o-mini", temperature=0)
8
# chat_llm = models.get_ollama_chat(model_name="gemma2:latest", temperature=0)
9
- # chat_llm = models.get_lmstudio_chat(model_name="TheBloke/Mistral-7B-Instruct-v0.2-GGUF", temperature=0)
10
- # chat_llm = models.get_openrouter_chat(model_name="nousresearch/hermes-3-llama-3.1-405b")
9
+ # chat_llm = models.get_lmstudio_chat(model_name="lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF", temperature=0)
10
+ # chat_llm = models.get_openrouter_chat(model_name="mattshumer/reflection-70b:free")
11
# chat_llm = models.get_azure_openai_chat(deployment_name="gpt-4o-mini", temperature=0)
12
# chat_llm = models.get_anthropic_chat(model_name="claude-3-5-sonnet-20240620", temperature=0)
13
# chat_llm = models.get_google_chat(model_name="gemini-1.5-flash", temperature=0)
@@ -20,6 +20,7 @@ def initialize():
20
embedding_llm = models.get_openai_embedding(model_name="text-embedding-3-small")
21
# embedding_llm = models.get_ollama_embedding(model_name="nomic-embed-text")
22
# embedding_llm = models.get_huggingface_embedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
23
+ # embedding_llm = models.get_lmstudio_embedding(model_name="nomic-ai/nomic-embed-text-v1.5-GGUF")
24
25
# agent configuration
26
config = AgentConfig(
models.py
+1
-1
@@ -40,7 +40,7 @@ def get_lmstudio_chat(model_name:str, temperature=DEFAULT_TEMPERATURE, base_url=
40
return ChatOpenAI(model_name=model_name, base_url=base_url, temperature=temperature, api_key="none") # type: ignore
41
42
def get_lmstudio_embedding(model_name:str, base_url=os.getenv("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1"):
43
- return OpenAIEmbeddings(model_name=model_name, base_url=base_url) # type: ignore
43
+ return OpenAIEmbeddings(model=model_name, api_key="none", base_url=base_url, check_embedding_ctx_length=False) # type: ignore
44
45
# Anthropic models
46
def get_anthropic_chat(model_name:str, api_key=get_api_key("anthropic"), temperature=DEFAULT_TEMPERATURE):
python/helpers/defer.py
+42
-29
@@ -7,55 +7,68 @@ class EventLoopThread:
7
_instance = None
8
_lock = threading.Lock()
9
10
- def __init__(self) -> None:
11
- self.loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
12
- self.thread: threading.Thread = threading.Thread(target=self._run_event_loop, daemon=True)
13
- self.thread.start()
14
-
15
- def __new__(cls) -> 'EventLoopThread':
10
+ def __new__(cls):
11
with cls._lock:
12
if cls._instance is None:
18
- cls._instance = super().__new__(cls)
19
- cls._instance.__init__()
20
- return cls._instance
13
+ cls._instance = super(EventLoopThread, cls).__new__(cls)
14
+ cls._instance.loop = asyncio.new_event_loop() # type: ignore
15
+ cls._instance.thread = threading.Thread(target=cls._instance._run_event_loop, daemon=True) # type: ignore
16
+ cls._instance.thread.start() # type: ignore
17
+ return cls._instance
18
19
def _run_event_loop(self):
23
- asyncio.set_event_loop(self.loop)
24
- self.loop.run_forever()
20
+ asyncio.set_event_loop(self.loop) # type: ignore
21
+ self.loop.run_forever() # type: ignore
22
23
def run_coroutine(self, coro):
27
- return asyncio.run_coroutine_threadsafe(coro, self.loop)
24
+ return asyncio.run_coroutine_threadsafe(coro, self.loop) # type: ignore
25
26
class DeferredTask:
30
- def __init__(self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any) -> None:
31
- self._event_loop_thread = EventLoopThread()
32
- self._future: Future[Any] = self._event_loop_thread.run_coroutine(self._run(func, *args, **kwargs))
27
+ def __init__(self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any):
28
+ self.func = func
29
+ self.args = args
30
+ self.kwargs = kwargs
31
+ self.event_loop_thread = EventLoopThread()
32
+ self._future: Optional[Future] = None
33
+ self._start_task()
34
34
- async def _run(self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any) -> Any:
35
- return await func(*args, **kwargs)
35
+ def _start_task(self):
36
+ self._future = self.event_loop_thread.run_coroutine(self._run())
37
37
- def is_ready(self) -> bool:
38
- return self._future.done()
38
+ async def _run(self):
39
+ return await self.func(*self.args, **self.kwargs)
40
40
- async def result(self, timeout: Optional[float] = None) -> Any:
41
- try:
42
- return await asyncio.wait_for(asyncio.wrap_future(self._future), timeout)
43
- except asyncio.TimeoutError:
44
- raise TimeoutError("The task did not complete within the specified timeout.")
41
+ def is_ready(self) -> bool:
42
+ return self._future.done() if self._future else False
43
44
def result_sync(self, timeout: Optional[float] = None) -> Any:
45
+ if not self._future:
46
+ raise RuntimeError("Task hasn't been started")
47
try:
48
return self._future.result(timeout)
49
except TimeoutError:
50
raise TimeoutError("The task did not complete within the specified timeout.")
51
52
+ async def result(self, timeout: Optional[float] = None) -> Any:
53
+ if not self._future:
54
+ raise RuntimeError("Task hasn't been started")
55
+
56
+ loop = asyncio.get_running_loop()
57
+
58
+ def _get_result():
59
+ try:
60
+ return self._future.result(timeout) # type: ignore
61
+ except TimeoutError:
62
+ raise TimeoutError("The task did not complete within the specified timeout.")
63
+
64
+ return await loop.run_in_executor(None, _get_result)
65
+
66
def kill(self) -> None:
53
- if not self._future.done():
67
+ if self._future and not self._future.done():
68
self._future.cancel()
69
70
def is_alive(self) -> bool:
57
- return not self._future.done()
71
+ return self._future and not self._future.done() # type: ignore
72
59
-# Helper function to run async code
60
-async def run_async(func, *args, **kwargs):
61
- return await func(*args, **kwargs)
\ No newline at end of file
73
+ def restart(self) -> None:
74
+ self._start_task()
\ No newline at end of file