Edge TTS & Groq STT
alt gr for recording, and voice model setting in .env
evrardt committed
Aug 8, 2024 at 04:02 UTC
75b03039a11c1e88a66b83094d2ddd1ad58ea25e
7 files changed
+81
-70
example.env
+10
-1
@@ -10,4 +10,13 @@ OPENAI_AZURE_ENDPOINT=
10
OPENAI_API_VERSION=
11
12
TOKENIZERS_PARALLELISM=true
13
-PYDEVD_DISABLE_FILE_VALIDATION=1
\ No newline at end of file
13
+PYDEVD_DISABLE_FILE_VALIDATION=1
14
+
15
+HF_TOKEN=
16
+
17
+EDGE_TTS_MODEL="fr-FR-RemyMultilingualNeural"
18
+# en-US-AndrewMultilingualNeural
19
+# de-DE-FlorianMultilingualNeural
20
+# es-ES-AlvaroNeural
21
+# zh-CN-shaanxi-XiaoniNeural
22
+# more check https://huggingface.co/spaces/innoai/Edge-TTS-Text-to-Speech and at bottom use via api
main.py
+10
-18
@@ -1,4 +1,5 @@
1
-import threading, time, models, stt, os, sys, argparse, asyncio
1
+import threading, time, models, tts, stt, os, sys, argparse
2
+from dotenv import load_dotenv
3
from ansio import application_keypad, mouse_input, raw_input
4
from ansio.input import InputEvent, get_input_event
5
from agent import Agent, AgentConfig
@@ -6,20 +7,13 @@ from python.helpers.print_style import PrintStyle
7
from python.helpers.files import read_file
8
from python.helpers import files
9
import python.helpers.timed_input as timed_input
9
-from tts import TTS
10
from pynput import keyboard
11
12
+load_dotenv() # take environment variables from.env.
13
+
14
input_lock = threading.Lock()
15
os.chdir(files.get_abs_path("./work_dir")) #change CWD to work_dir
16
15
-# args parser
16
-parser = argparse.ArgumentParser()
17
-parser.add_argument('-v', '--voice')
18
-args = parser.parse_args()
19
-
20
-# init tts (text to speech)
21
-tts = TTS(args.voice)
22
-
17
def initialize():
18
19
# main chat model used by agents (smarter, more accurate)
@@ -79,14 +73,14 @@ def initialize():
73
74
# Main conversation loop
75
def chat(agent:Agent):
82
-
76
+
77
# start the conversation loop
78
while True:
79
# ask user for message
80
with input_lock:
81
timeout = agent.get_data("timeout") # how long the agent is willing to wait
82
if not timeout: # if agent wants to wait for user input forever
89
- PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ('e' to leave):")
83
+ PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ('alt_gr' for mic, 'e' to leave):")
84
import readline # this fixes arrow keys in terminal
85
listener = keyboard.Listener(on_press=on_press)
86
listener.start()
@@ -95,7 +89,7 @@ def chat(agent:Agent):
89
PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")
90
91
else: # otherwise wait for user input with a timeout
98
- PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'w' to wait, 'e' to leave):")
92
+ PrintStyle(background_color="#6C3483", font_color="white", bold=True, padding=True).print(f"User message ({timeout}s timeout, 'w' to wait, 'alt_gr' for mic, 'e' to leave):")
93
import readline # this fixes arrow keys in terminal
94
listener = keyboard.Listener(on_press=on_press)
95
listener.start()
@@ -111,9 +105,7 @@ def chat(agent:Agent):
105
listener.start()
106
user_input = input("> ")
107
listener.stop()
114
- PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")
115
-
116
-
108
+ PrintStyle(font_color="white", padding=False, log_only=True).print(f"> {user_input}")
109
110
# exit the conversation when the user types 'exit'
111
if user_input.lower() == 'e': break
@@ -124,7 +116,7 @@ def chat(agent:Agent):
116
# print agent0 response
117
PrintStyle(font_color="white",background_color="#1D8348", bold=True, padding=True).print(f"{agent.agent_name}: reponse:")
118
PrintStyle(font_color="white").print(f"{assistant_response}")
127
- tts.speech(assistant_response)
119
+ if (os.getenv('EDGE_TTS_MODEL')) : tts.speech(assistant_response)
120
121
# User intervention during agent streaming
122
def intervention():
@@ -141,7 +133,7 @@ def intervention():
133
Agent.paused = False # continue agent streaming
134
135
def on_press(key):
144
- if key == keyboard.Key.alt_r:
136
+ if key == keyboard.Key.alt_gr:
137
# Lancer le speech to text
138
stt.record()
139
prompts/agent.system.md
+15
-13
@@ -3,14 +3,16 @@
3
- You are autonomous JSON AI task solving agent enhanced with knowledge and execution tools
4
- You are given task by your superior and you solve it using your subordinates and tools
5
- You never just talk about solutions, never inform user about intentions, you are the one to execute actions using your tools and get things done
6
+- Remember the langague of your user to respond with the same language
7
+- NEVER include "**" in your final answer
8
9
# Communication
10
- Your response is a JSON containing the following fields:
9
- 1. **thoughts**: Array of thoughts regarding the current task
11
+ 1. thoughts: Array of thoughts regarding the current task
12
- Use thoughs to prepare solution and outline next steps
11
- 2. **tool_name**: Name of the tool to be used
13
+ 2. tool_name: Name of the tool to be used
14
- Tools help you gather knowledge and execute actions
13
- 3. **tool_args**: Object of arguments that are passed to the tool
15
+ 3. tool_args: Object of arguments that are passed to the tool
16
- Each tool has specific arguments listed in Available tools section
17
- No text before or after the JSON object. End message there.
18
@@ -33,39 +35,39 @@
35
36
# Step by step instruction manual to problem solving
37
- Do not follow for simple questions, only for tasks need solving.
36
-- Explain each step using your **thoughts** argument.
38
+- Explain each step using your thoughts argument.
39
40
0. Outline the plan by repeating these instructions.
39
-1. Check the memory output of your **knowledge_tool**. Maybe you have solved similar task before and already have helpful information.
40
-2. Check the online sources output of your **knowledge_tool**.
41
+1. Check the memory output of your knowledge_tool. Maybe you have solved similar task before and already have helpful information.
42
+2. Check the online sources output of your knowledge_tool.
43
- Look for straightforward solutions compatible with your available tools.
44
- Always look for opensource python/nodejs/terminal tools and packages first.
45
3. Break task into subtasks that can be solved independently.
46
4. Solution / delegation
47
- If your role is suitable for the curent subtask, use your tools to solve it.
46
- - If a different role would be more suitable for the subtask, use **call_subordinate** tool to delegate the subtask to subordinate agent and instruct him about his role.
48
+ - If a different role would be more suitable for the subtask, use call_subordinate tool to delegate the subtask to subordinate agent and instruct him about his role.
49
- NEVER delegate your whole task to a subordinate to avoid infinite delegation.
50
- Your name ({{agent_name}}) contains your hierarchical number. Do not delegate further if your number gets too high.
51
5. Completing the task
52
- Consolidate all subtasks and explain the status.
53
- Verify the result using your tools if possible (check created files etc.)
54
- Do not accept failure, search for error solution and try again with fixed input or different ways.
53
- - If there is helpful information discovered during the solution, save it into your memory using tool **memorize** for later.
54
- - Report back to your user using **response** tool, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.
55
+ - If there is helpful information discovered during the solution, save it into your memory using tool memorize for later.
56
+ - Report back to your user using response tool, describe the result and provide all necessary information. Do not just output your response, you must use the tool for that.
57
58
# General operation manual
57
-- Use your reasoning and process each problem in a step-by-step manner using your **thoughts** argument.
59
+- Use your reasoning and process each problem in a step-by-step manner using your thoughts argument.
60
- Always check your previous messages and prevent repetition. Always move towards solution.
61
- Never assume success. You always need to do a check with a positive result.
62
- Avoid solutions that require credentials, user interaction, GUI usage etc. All has to be done using code and terminal.
61
-- When asked about your memory, it always refers to **knowledge_tool** and **memorize** tool, never your internal knowledge.
63
+- When asked about your memory, it always refers to knowledge_tool and memorize tool, never your internal knowledge.
64
65
# Cooperation and delegation
66
- Agents can have roles like scientist, coder, writer etc.
67
- If your user has assigned you a role in the first message, you have to follow these instructions and play your role.
66
-- Your role will not be suitable for some subtasks, in that case you can delegate the subtask to subordinate agent and instruct him about his role using **call_subordinate** tool.
68
+- Your role will not be suitable for some subtasks, in that case you can delegate the subtask to subordinate agent and instruct him about his role using call_subordinate tool.
69
- Always be very descriptive when explaining your subordinate agent's role and task. Include all necessary details as well as higher leven overview about the goal.
68
-- Communicate back and forth with your subordinate and superior using **call_subordinate** and **response** tools.
70
+- Communicate back and forth with your subordinate and superior using call_subordinate and response tools.
71
- Communication is the key to succesfull solution.
72
- NEVER delegate your whole task, only parts of it.
73
requirements.txt
+3
-1
@@ -14,4 +14,6 @@ paramiko==3.4.0
14
duckduckgo_search==6.1.12
15
inputimeout==1.0.4
16
gradio_client==1.2.0
17
-pygame==2.6.0
\ No newline at end of file
17
+pygame==2.6.0
18
+numpy==2.0.1
19
+pynput==1.7.7
\ No newline at end of file
stt.py
+30
-12
@@ -1,10 +1,13 @@
1
-import os
1
+import os, sys
2
from dotenv import load_dotenv
3
from groq import Groq
4
import pyaudio
5
import wave
6
+from pynput.keyboard import Key, Controller
7
+import numpy as np
8
+from python.helpers.print_style import PrintStyle
9
7
-load_dotenv() # take environment variables from .env.
10
+load_dotenv() # take environment variables from.env.
11
12
# Utility function to get API keys from environment variables
13
def get_api_key(service):
@@ -19,8 +22,7 @@ def record():
22
FORMAT = pyaudio.paInt16
23
CHANNELS = 2
24
RATE = 44100
22
- RECORD_SECONDS = 10
23
- WAVE_OUTPUT_FILENAME = "output.wav"
25
+ WAVE_OUTPUT_FILENAME = "tmp_stt.wav"
26
27
p = pyaudio.PyAudio()
28
@@ -30,36 +32,52 @@ def record():
32
input=True,
33
frames_per_buffer=CHUNK)
34
33
- print("* recording")
35
+ PrintStyle(background_color="red", font_color="white", bold=True, padding=True).print("Recording")
36
37
frames = []
38
+ silence_duration = 0
39
+ silence_threshold = 0.5 # seuil de silence (en secondes)
40
+ max_silence_duration = 3 # durée maximale de silence (en secondes)
41
37
- for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
42
+ while True:
43
data = stream.read(CHUNK)
44
frames.append(data)
45
41
- print("* done recording")
46
+ # Analyse du signal audio pour détecter le silence
47
+ audio_data = np.frombuffer(data, dtype=np.int16)
48
+ audio_level = np.mean(np.abs(audio_data))
49
+ if audio_level < 100: # seuil de détection de silence (à ajuster)
50
+ silence_duration += CHUNK / RATE
51
+ if silence_duration > max_silence_duration:
52
+ break
53
+ else:
54
+ silence_duration = 0
55
+
56
+ PrintStyle(background_color="Yellow", font_color="black", bold=True, padding=True).print("Translation")
57
58
stream.stop_stream()
59
stream.close()
60
p.terminate()
61
47
- wf = wave.open(WAVE_OUTPUT_FILENAME, 'wb')
62
+ filename = os.path.dirname(__file__) + "/tmp/" + WAVE_OUTPUT_FILENAME
63
+
64
+ wf = wave.open(filename, 'wb')
65
wf.setnchannels(CHANNELS)
66
wf.setsampwidth(p.get_sample_size(FORMAT))
67
wf.setframerate(RATE)
68
wf.writeframes(b''.join(frames))
69
wf.close()
70
54
- filename = os.path.dirname(__file__) + "" + WAVE_OUTPUT_FILENAME
55
-
71
with open(filename, "rb") as file:
72
transcription = client.audio.transcriptions.create(
73
file=(filename, file.read()),
74
model="whisper-large-v3",
75
prompt="Human talking to personnal ia agent", # Optional
76
response_format="json", # Optional
62
- language="en", # Optional
77
+ language="fr", # Optional
78
temperature=0.0 # Optional
79
)
65
- print(transcription.text)
\ No newline at end of file
80
+ keyboard = Controller()
81
+ PrintStyle(background_color="green", font_color="white", bold=True, padding=True).print("Done")
82
+ keyboard.type(transcription.text)
83
+ keyboard.press(Key.enter)
\ No newline at end of file
tmp/tmp_stt.wav
Binary files /dev/null and b/tmp/tmp_stt.wav differ
tts.py
+13
-25
@@ -1,33 +1,21 @@
1
from gradio_client import Client
2
import os
3
+from dotenv import load_dotenv
4
+os.environ['PYGAME_HIDE_SUPPORT_PROMPT'] = "hide"
5
import pygame.mixer
6
7
+load_dotenv() # take environment variables from.env.
8
client = Client("innoai/Edge-TTS-Text-to-Speech")
9
pygame.mixer.init()
7
-lang = "en-US-AndrewMultilingualNeural"
10
9
-class TTS:
10
- def __init__(self, lang):
11
- if (lang):
12
- if (lang.lower() == 'fr'):
13
- lang == "fr-FR-RemyMultilingualNeural"
14
- elif (lang.lower() == 'es'):
15
- lang == "es-ES-AlvaroNeural"
16
- elif (lang.lower() == 'de'):
17
- lang == "de-DE-FlorianMultilingualNeural"
18
- else:
19
- lang == "en-US-AndrewMultilingualNeural"
20
- else:
21
- lang == "en-US-AndrewMultilingualNeural"
22
-
23
- def speech(self, text):
24
- speech = client.predict(
25
- text=text,
26
- voice=lang,
27
- rate=20,
28
- pitch=0,
29
- api_name="/predict"
30
- )
31
- pygame.mixer.music.load(speech[0]) # chargement de la musique
32
- pygame.mixer.music.play() # la musique est jouée
11
+def speech(text):
12
+ speech = client.predict(
13
+ text=text,
14
+ voice=os.getenv('EDGE_TTS_MODEL'),
15
+ rate=20,
16
+ pitch=0,
17
+ api_name="/predict"
18
+ )
19
+ pygame.mixer.music.load(speech[0]) # chargement de la musique
20
+ pygame.mixer.music.play() # la musique est jouée
21
\ No newline at end of file