@samitouri / QOSami_Agents_Zero-2 / commits / 709b8653

Add Telegram speech mode

Anmol Malik committed May 28, 2026 at 23:35 UTC 709b8653996508eeda42ebfbdc54a282d5d3c92e
7 files changed +248 -4
helpers/integration_commands.py
+8
@@ -70,6 +70,12 @@ COMMAND_REGISTRY: tuple[IntegrationCommandDef, ...] = (
70 "Configuration",
71 args_hint="[on|off]",
72 ),
73 + IntegrationCommandDef(
74 + "speech",
75 + "Enable or disable Telegram spoken replies.",
76 + "Configuration",
77 + args_hint="[on|off]",
78 + ),
79 IntegrationCommandDef(
80 "project",
81 "Show or switch the active project.",
@@ -194,6 +200,8 @@ def try_handle_command(context: "AgentContext", text: str) -> str | None:
200 return _handle_toggle(context, args, "telegram_stream_enabled", "Response streaming")
201 if command == "/tools":
202 return _handle_toggle(context, args, "telegram_tools_enabled", "Tool progress")
203 + if command == "/speech":
204 + return _handle_toggle(context, args, "telegram_speech_enabled", "Speech replies")
205 if command == "/project":
206 return _handle_project(context, args)
207 if command == "/model":
plugins/_telegram_integration/README.md
+3
@@ -24,6 +24,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde
24 - `/agent <profile>` switches the active agent profile when the current run is idle.
25 - `/model`, `/project`, and `/agent` show Telegram inline keyboard pickers when used without arguments.
26 - `/stream` toggles live response streaming; `/tools` toggles tool progress messages.
27 + - `/speech` toggles spoken replies. When enabled, final text replies also include Kokoro-generated Telegram audio when Kokoro TTS is available.
28 - `/send` or `/queue send` flushes queued messages for the current chat.
29 - `/steer <message>` sends an intervention to the active run.
30 - **Group support**
@@ -35,6 +36,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde
36 - **Reply delivery**
37 - Streams tool progress and response text through real Telegram messages updated with `editMessageText`.
38 - Tool progress and the AI response are separate messages; only the AI response replies to the user message.
39 + - Sends native Telegram media: photos, voice/audio files, videos, and document fallback.
40 - `tool_execute_after` intercepts the `response` tool — sends `break_loop=false` updates as separate intermediate Telegram messages.
41 - `process_chain_end` auto-sends the final response, with retry logic on failure.
42 - **Formatting**
@@ -63,6 +65,7 @@ This plugin connects one or more Telegram bots to Agent Zero. Each bot runs inde
65 - `helpers/telegram_client.py` — Low-level Telegram API wrapper: send text/file/photo, Markdown→HTML converter, keyboard builder, message splitting.
66 - `helpers/command_ui.py` — Telegram inline keyboard command pickers and callback handling.
67 - `helpers/draft_stream.py` — Editable Telegram message streaming for tool progress and response previews.
68 + - `helpers/speech.py` — Telegram speech-mode TTS synthesis and audio attachment generation.
69 - **Extensions**
70 - `extensions/python/job_loop/_10_telegram_bot.py` — Bot lifecycle manager, starts/stops bots on each tick.
71 - `extensions/python/message_loop_start/_45_telegram_draft_start.py` — Initializes Telegram response streaming.
plugins/_telegram_integration/extensions/python/process_chain_end/_55_telegram_reply.py
+10
@@ -32,7 +32,17 @@ class TelegramAutoReply(Extension):
32
33 try:
34 if response_text:
35 + from plugins._telegram_integration.helpers import speech
36 +
37 + speech_attachment, speech_warning = await speech.synthesize_attachment(
38 + context, response_text
39 + )
40 + if speech_attachment:
41 + attachments = [*attachments, speech_attachment]
42 await self._send_reply(context, response_text, attachments, keyboard)
43 + warning = speech.consume_warning(context, speech_warning)
44 + if warning:
45 + await self._send_reply(context, warning, [], None)
46 except Exception as e:
47 PrintStyle.error(f"Telegram auto-reply error: {format_error(e)}")
48 finally:
plugins/_telegram_integration/helpers/command_ui.py
+33 -4
@@ -13,6 +13,7 @@ from plugins._telegram_integration.helpers import telegram_client as tc
13 from plugins._telegram_integration.helpers.constants import (
14 CTX_TG_STREAM_ENABLED,
15 CTX_TG_TOOLS_ENABLED,
16 + CTX_TG_SPEECH_ENABLED,
17 CTX_TG_BOT,
18 CTX_TG_CHAT_ID,
19 CTX_TG_CHAT_TYPE,
@@ -85,6 +86,17 @@ async def handle_command(
86 args,
87 )
88 return True
89 + if command == "/speech":
90 + await send_toggle_picker(
91 + context,
92 + token,
93 + chat_id,
94 + reply_to_message_id,
95 + CTX_TG_SPEECH_ENABLED,
96 + "Speech replies",
97 + args,
98 + )
99 + return True
100 return False
101
102
@@ -129,9 +141,8 @@ async def handle_callback(
141 selected_context = await _select_session(context, _safe_int(value))
142 await edit_session_picker(selected_context or context, token, chat_id, message_id, 0, selected=bool(selected_context))
143 return True
132 - if kind in {"stream", "tools"} and action in {"on", "off"}:
133 - key = CTX_TG_STREAM_ENABLED if kind == "stream" else CTX_TG_TOOLS_ENABLED
134 - label = "Response streaming" if kind == "stream" else "Tool progress"
144 + if kind in {"stream", "tools", "speech"} and action in {"on", "off"}:
145 + key, label = _toggle_key_label(kind)
146 context.set_data(key, action == "on")
147 save_tmp_chat(context)
148 mark_dirty_for_context(context.id, reason=f"telegram.{kind}_toggle")
@@ -334,7 +345,7 @@ def _agent_view(
345
346 def _toggle_view(context: AgentContext, key: str, label: str) -> tuple[str, dict]:
347 enabled = _toggle_enabled(context, key)
337 - kind = "stream" if key == CTX_TG_STREAM_ENABLED else "tools"
348 + kind = _toggle_kind(key)
349 state = "enabled" if enabled else "disabled"
350 text = f"{_html(label)}: <b>{state}</b>"
351 rows = [[
@@ -542,9 +553,27 @@ def _paged_buttons(
553
554 def _toggle_enabled(context: AgentContext, key: str) -> bool:
555 value = context.get_data(key)
556 + if key == CTX_TG_SPEECH_ENABLED:
557 + return bool(value)
558 return True if value is None else bool(value)
559
560
561 +def _toggle_kind(key: str) -> str:
562 + if key == CTX_TG_STREAM_ENABLED:
563 + return "stream"
564 + if key == CTX_TG_TOOLS_ENABLED:
565 + return "tools"
566 + return "speech"
567 +
568 +
569 +def _toggle_key_label(kind: str) -> tuple[str, str]:
570 + if kind == "stream":
571 + return CTX_TG_STREAM_ENABLED, "Response streaming"
572 + if kind == "tools":
573 + return CTX_TG_TOOLS_ENABLED, "Tool progress"
574 + return CTX_TG_SPEECH_ENABLED, "Speech replies"
575 +
576 +
577 def _parse_toggle(args: str) -> bool | None:
578 value = (args or "").strip().lower()
579 if value in {"on", "enable", "enabled", "yes", "true", "1"}:
plugins/_telegram_integration/helpers/constants.py
+2
@@ -13,6 +13,7 @@ CTX_TG_TYPING_STOP = "_telegram_typing_stop"
13 CTX_TG_REPLY_TO = "_telegram_reply_to_message_id"
14 CTX_TG_STREAM_ENABLED = "telegram_stream_enabled"
15 CTX_TG_TOOLS_ENABLED = "telegram_tools_enabled"
16 +CTX_TG_SPEECH_ENABLED = "telegram_speech_enabled"
17
18 # Transient
19 CTX_TG_ATTACHMENTS = "_telegram_response_attachments"
@@ -25,3 +26,4 @@ CTX_TG_RESPONSE_LAST_UPDATE = "_telegram_response_last_update"
26 CTX_TG_ERROR_SENT = "_telegram_error_sent"
27 CTX_TG_HEARTBEAT_TASK = "_telegram_heartbeat_task"
28 CTX_TG_HEARTBEAT_STOP = "_telegram_heartbeat_stop"
29 +CTX_TG_SPEECH_WARNING_SENT = "_telegram_speech_warning_sent"
plugins/_telegram_integration/helpers/speech.py new
+80
@@ -0,0 +1,80 @@
1 +from __future__ import annotations
2 +
3 +import re
4 +import uuid
5 +
6 +from agent import AgentContext
7 +from helpers import files
8 +from plugins._telegram_integration.helpers.constants import (
9 + CTX_TG_SPEECH_ENABLED,
10 + CTX_TG_SPEECH_WARNING_SENT,
11 + DOWNLOAD_FOLDER,
12 +)
13 +
14 +
15 +MAX_SPEECH_CHARS = 1800
16 +
17 +
18 +def is_enabled(context: AgentContext) -> bool:
19 + return bool(context.get_data(CTX_TG_SPEECH_ENABLED))
20 +
21 +
22 +async def synthesize_attachment(context: AgentContext, text: str) -> tuple[str | None, str | None]:
23 + if not is_enabled(context):
24 + return None, None
25 +
26 + speech_text = _speech_text(text)
27 + if not speech_text:
28 + return None, "Speech mode is enabled, but there was no readable text to speak."
29 +
30 + try:
31 + from plugins._kokoro_tts.helpers import runtime
32 + except Exception:
33 + return None, _setup_message()
34 +
35 + if not runtime.is_globally_enabled():
36 + return None, _setup_message()
37 +
38 + try:
39 + audio = await runtime.synthesize_sentences([speech_text])
40 + except Exception as exc:
41 + return None, f"Speech mode is enabled, but text-to-speech failed: {exc}"
42 +
43 + if not audio:
44 + return None, "Speech mode is enabled, but text-to-speech returned no audio."
45 +
46 + name = f"telegram_speech_{context.id}_{uuid.uuid4().hex[:10]}.wav"
47 + relative_path = f"{DOWNLOAD_FOLDER}/{name}"
48 + files.write_file_base64(relative_path, audio)
49 + return files.get_abs_path_dockerized(relative_path), None
50 +
51 +
52 +def consume_warning(context: AgentContext, warning: str | None) -> str | None:
53 + if not warning:
54 + context.data.pop(CTX_TG_SPEECH_WARNING_SENT, None)
55 + return None
56 + if context.data.get(CTX_TG_SPEECH_WARNING_SENT):
57 + return None
58 + context.data[CTX_TG_SPEECH_WARNING_SENT] = True
59 + return warning
60 +
61 +
62 +def _setup_message() -> str:
63 + return (
64 + "Speech mode is enabled, but Kokoro TTS is not available. "
65 + "Enable the Kokoro TTS plugin from Agent Zero settings, then try again."
66 + )
67 +
68 +
69 +def _speech_text(text: str) -> str:
70 + value = str(text or "")
71 + value = re.sub(r"```.*?```", " ", value, flags=re.DOTALL)
72 + value = re.sub(r"`([^`]+)`", r"\1", value)
73 + value = re.sub(r"!\[([^\]]*)\]\([^)]+\)", r"\1", value)
74 + value = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", value)
75 + value = re.sub(r"<[^>]+>", " ", value)
76 + value = re.sub(r"[*_#>\-]+", " ", value)
77 + value = re.sub(r"\s+", " ", value).strip()
78 + if len(value) > MAX_SPEECH_CHARS:
79 + value = value[:MAX_SPEECH_CHARS].rstrip()
80 + return value
tests/test_telegram_speech_mode.py new
+112
@@ -0,0 +1,112 @@
1 +import asyncio
2 +
3 +from plugins._telegram_integration.extensions.python.process_chain_end import (
4 + _55_telegram_reply,
5 +)
6 +from plugins._telegram_integration.helpers import command_ui, speech
7 +from plugins._telegram_integration.helpers.constants import (
8 + CTX_TG_BOT,
9 + CTX_TG_REPLY_TO,
10 + CTX_TG_SPEECH_ENABLED,
11 +)
12 +
13 +
14 +class FakeContext:
15 + id = "ctx-speech"
16 +
17 + def __init__(self):
18 + self.data = {}
19 +
20 + def get_data(self, key):
21 + return self.data.get(key)
22 +
23 + def set_data(self, key, value):
24 + self.data[key] = value
25 +
26 +
27 +class FakeAgent:
28 + number = 0
29 +
30 + def __init__(self):
31 + self.context = FakeContext()
32 + self.context.data[CTX_TG_BOT] = "main"
33 + self.context.data[CTX_TG_REPLY_TO] = 456
34 +
35 +
36 +def test_speech_toggle_defaults_off():
37 + context = FakeContext()
38 +
39 + text, markup = command_ui._toggle_view(context, CTX_TG_SPEECH_ENABLED, "Speech replies")
40 +
41 + assert "Speech replies: <b>disabled</b>" == text
42 + assert markup["inline_keyboard"][0][0]["callback_data"] == "tg:speech:on"
43 + assert markup["inline_keyboard"][0][1]["callback_data"] == "tg:speech:off"
44 +
45 +
46 +def test_speech_text_is_cleaned_for_tts():
47 + cleaned = speech._speech_text(
48 + "**Hello** [`Agent Zero`](https://example.com)\n\n```python\nprint('skip')\n```"
49 + )
50 +
51 + assert cleaned == "Hello Agent Zero"
52 +
53 +
54 +def test_speech_synthesis_writes_audio_attachment(monkeypatch):
55 + context = FakeContext()
56 + context.data[CTX_TG_SPEECH_ENABLED] = True
57 + writes = []
58 +
59 + class FakeRuntime:
60 + @staticmethod
61 + def is_globally_enabled():
62 + return True
63 +
64 + @staticmethod
65 + async def synthesize_sentences(sentences):
66 + assert sentences == ["Hello from Agent Zero."]
67 + return "UklGRg=="
68 +
69 + monkeypatch.setattr(speech.files, "write_file_base64", lambda path, audio: writes.append((path, audio)))
70 + monkeypatch.setattr(speech.files, "get_abs_path_dockerized", lambda path: f"/a0/{path}")
71 + monkeypatch.setitem(__import__("sys").modules, "plugins._kokoro_tts.helpers.runtime", FakeRuntime)
72 +
73 + path, warning = asyncio.run(speech.synthesize_attachment(context, "Hello from Agent Zero."))
74 +
75 + assert warning is None
76 + assert path.startswith("/a0/usr/uploads/telegram_speech_ctx-speech_")
77 + assert path.endswith(".wav")
78 + assert writes[0][1] == "UklGRg=="
79 +
80 +
81 +def test_final_reply_includes_speech_attachment(monkeypatch):
82 + agent = FakeAgent()
83 + sent = []
84 +
85 + async def fake_synthesize(context, text):
86 + return "/a0/usr/uploads/reply.wav", None
87 +
88 + async def fake_send_reply(context, response_text, attachments, keyboard):
89 + sent.append(
90 + {
91 + "text": response_text,
92 + "attachments": attachments,
93 + "keyboard": keyboard,
94 + }
95 + )
96 +
97 + monkeypatch.setattr(_55_telegram_reply, "_extract_last_response", lambda context: "Final reply.")
98 + monkeypatch.setattr(speech, "synthesize_attachment", fake_synthesize)
99 +
100 + extension = _55_telegram_reply.TelegramAutoReply(agent=agent)
101 + extension._send_reply = fake_send_reply
102 +
103 + asyncio.run(extension.execute())
104 +
105 + assert sent == [
106 + {
107 + "text": "Final reply.",
108 + "attachments": ["/a0/usr/uploads/reply.wav"],
109 + "keyboard": None,
110 + }
111 + ]
112 + assert CTX_TG_REPLY_TO not in agent.context.data