4 files changed
+228
-13
Cargo.lock
+1
@@ -5283,6 +5283,7 @@ dependencies = [
5283
"onde",
5284
"ratatui",
5285
"regex",
5286
+ "reqwest 0.12.28",
5287
"serde_json",
5288
"tokio",
5289
"tokio-util",
Cargo.toml
+1
@@ -41,4 +41,5 @@ log = "0.4"
41
tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
42
serde_json = "1"
43
regex = "1"
44
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] }
45
uuid = { version = "1", features = ["v4"] }
src/main.rs
+9
-8
@@ -109,14 +109,15 @@ Never introduce yourself unless asked. Jump straight into the answer. \
109
Keep answers short. Write idiomatic code. \
110
Fix root causes, not symptoms.
111
112
-You have access to tools that let you read files, create directories, list \
113
-directories, search code, create new files, edit existing files, delete files, \
114
-and run shell commands. You can also use git directly through shell commands, \
115
-including `git init` and normal git workflows. Use them proactively. Read the \
116
-code before answering. Prefer absolute paths when referring to files and \
117
-directories, especially in protocol-facing output and tool arguments. Create \
118
-directories when needed. Run builds, tests, and git commands after making \
119
-changes. Ground your answers in the actual code, not in guesses.
112
+You have access to tools that let you read files, read websites directly from \
113
+http and https URLs, create directories, list directories, search code, create \
114
+new files, edit existing files, delete files, and run shell commands. You can \
115
+also use git directly through shell commands, including `git init` and normal \
116
+git workflows. Use them proactively. Read the code or website before answering. \
117
+Prefer absolute paths when referring to files and directories, especially in \
118
+protocol-facing output and tool arguments. Create directories when needed. Run \
119
+builds, tests, and git commands after making changes. Ground your answers in \
120
+the actual code or fetched page content, not in guesses.
121
122
Tool-use heuristics:
123
- prefer absolute paths over relative paths when you mention, return, or pass \
src/tools.rs
+217
-5
@@ -19,6 +19,10 @@
19
//! - `edit_file` — replace an exact old-text span with new text in an existing file
20
//! - `delete_file` — delete a file or empty directory at the given path
21
//!
22
+//! # Web Tools
23
+//!
24
+//! - `read_website` — fetch a web page and return readable text content
25
+//!
26
//! # Shell Tools
27
//!
28
//! - `run_command` — run shell commands, including git porcelain and plumbing commands
@@ -29,6 +33,11 @@ use std::fs;
33
use std::path::{Path, PathBuf};
34
use std::process::Command;
35
36
+const WEBSITE_READ_CHAR_LIMIT: usize = 20_000;
37
+const WEBSITE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20);
38
+const WEBSITE_USER_AGENT: &str =
39
+ "siGit/0.1 (+https://github.com/getsigit/sigit; website-reading tool)";
40
+
41
/// Maximum characters returned from `read_file` before truncation.
42
const READ_FILE_CHAR_LIMIT: usize = 10_000;
43
@@ -127,6 +136,25 @@ pub fn all_tools() -> Vec<AgentTool> {
136
"additionalProperties": false
137
}),
138
},
139
+ AgentTool {
140
+ name: "read_website",
141
+ description: "Fetch a web page and return readable text content. \
142
+ Use this when the user gives you a URL and asks you to read, \
143
+ summarize, inspect, or extract information from the page. \
144
+ Supports normal http and https URLs. Output is truncated if the \
145
+ page is very large.",
146
+ parameters_schema: json!({
147
+ "type": "object",
148
+ "properties": {
149
+ "url": {
150
+ "type": "string",
151
+ "description": "Absolute http or https URL to fetch."
152
+ }
153
+ },
154
+ "required": ["url"],
155
+ "additionalProperties": false
156
+ }),
157
+ },
158
AgentTool {
159
name: "create_file",
160
description: "Create a new file at the given path with the provided content. \
@@ -239,6 +267,7 @@ pub fn execute_tool(name: &str, arguments: &str) -> String {
267
"read_file" => exec_read_file(arguments),
268
"list_directory" => exec_list_directory(arguments),
269
"search_files" => exec_search_files(arguments),
270
+ "read_website" => exec_read_website(arguments),
271
"create_directory" => exec_create_directory(arguments),
272
"create_file" => exec_create_file(arguments),
273
"edit_file" => exec_edit_file(arguments),
@@ -485,6 +514,122 @@ fn search_file(path: &Path, re: &Regex, matches: &mut Vec<String>) {
514
}
515
}
516
517
+/// ── read_website ─────────────────────────────────────────────────────────────
518
+
519
+fn exec_read_website(arguments: &str) -> String {
520
+ let args: Value = match serde_json::from_str(arguments) {
521
+ Ok(v) => v,
522
+ Err(err) => return format!("Error: failed to parse arguments: {err}"),
523
+ };
524
+
525
+ let url = match args.get("url").and_then(Value::as_str) {
526
+ Some(u) => u,
527
+ None => return "Error: missing required parameter \"url\"".to_string(),
528
+ };
529
+
530
+ if !(url.starts_with("http://") || url.starts_with("https://")) {
531
+ return format!("Error: url must start with http:// or https://: {url}");
532
+ }
533
+
534
+ let client = match reqwest::blocking::Client::builder()
535
+ .timeout(WEBSITE_READ_TIMEOUT)
536
+ .user_agent(WEBSITE_USER_AGENT)
537
+ .build()
538
+ {
539
+ Ok(client) => client,
540
+ Err(err) => return format!("Error: failed to build website client: {err}"),
541
+ };
542
+
543
+ let response = match client.get(url).send() {
544
+ Ok(r) => r,
545
+ Err(err) => return format!("Error: failed to fetch website: {err}"),
546
+ };
547
+
548
+ let final_url = response.url().to_string();
549
+ let status = response.status();
550
+ if !status.is_success() {
551
+ return format!("Error: website returned HTTP {status} for {final_url}");
552
+ }
553
+
554
+ let body = match response.text() {
555
+ Ok(text) => text,
556
+ Err(err) => return format!("Error: failed to read website body: {err}"),
557
+ };
558
+
559
+ let title = Regex::new(r"(?is)<title[^>]*>(.*?)</title>")
560
+ .unwrap()
561
+ .captures(&body)
562
+ .and_then(|captures| captures.get(1))
563
+ .map(|m| {
564
+ Regex::new(r"\s+")
565
+ .unwrap()
566
+ .replace_all(m.as_str(), " ")
567
+ .trim()
568
+ .to_string()
569
+ })
570
+ .filter(|title| !title.is_empty());
571
+
572
+ let with_block_breaks = Regex::new(
573
+ r"(?is)</?(?:p|div|section|article|main|aside|header|footer|nav|li|ul|ol|h1|h2|h3|h4|h5|h6|br|tr|td|th)[^>]*>",
574
+ )
575
+ .unwrap()
576
+ .replace_all(&body, "\n");
577
+ let without_scripts = Regex::new(r"(?is)<script[^>]*>.*?</script>")
578
+ .unwrap()
579
+ .replace_all(&with_block_breaks, " ");
580
+ let without_styles = Regex::new(r"(?is)<style[^>]*>.*?</style>")
581
+ .unwrap()
582
+ .replace_all(&without_scripts, " ");
583
+ let without_tags = Regex::new(r"(?is)<[^>]+>")
584
+ .unwrap()
585
+ .replace_all(&without_styles, " ");
586
+ let normalized_newlines = without_tags
587
+ .replace(" ", " ")
588
+ .replace("&", "&")
589
+ .replace("<", "<")
590
+ .replace(">", ">")
591
+ .replace(""", "\"")
592
+ .replace("'", "'");
593
+ let collapsed_lines = Regex::new(r"[ \t]+")
594
+ .unwrap()
595
+ .replace_all(&normalized_newlines, " ");
596
+ let collapsed_breaks = Regex::new(r"\n\s*\n+")
597
+ .unwrap()
598
+ .replace_all(&collapsed_lines, "\n\n");
599
+ let cleaned = collapsed_breaks
600
+ .lines()
601
+ .map(str::trim)
602
+ .filter(|line| !line.is_empty())
603
+ .collect::<Vec<_>>()
604
+ .join("\n");
605
+
606
+ if cleaned.is_empty() {
607
+ return format!("Fetched {url}, but no readable text content was found.");
608
+ }
609
+
610
+ let mut metadata = vec![format!("URL: {final_url}")];
611
+ if let Some(title) = &title {
612
+ metadata.push(format!("Title: {title}"));
613
+ }
614
+
615
+ let body_text = match title {
616
+ Some(title) if !cleaned.starts_with(&title) => cleaned,
617
+ _ => cleaned,
618
+ };
619
+
620
+ let output = format!("{}\n\n{}", metadata.join("\n"), body_text);
621
+
622
+ if output.len() > WEBSITE_READ_CHAR_LIMIT {
623
+ let truncated: String = output.chars().take(WEBSITE_READ_CHAR_LIMIT).collect();
624
+ return format!(
625
+ "{truncated}\n\n--- truncated (showing {WEBSITE_READ_CHAR_LIMIT} of {} characters) ---",
626
+ output.len()
627
+ );
628
+ }
629
+
630
+ output
631
+}
632
+
633
/// ── create_directory ─────────────────────────────────────────────────────────
634
635
/// Create a directory and any missing parent directories.
@@ -912,15 +1057,16 @@ mod tests {
1057
#[test]
1058
fn test_all_tools_count() {
1059
let tools = all_tools();
915
- assert_eq!(tools.len(), 8);
1060
+ assert_eq!(tools.len(), 9);
1061
assert_eq!(tools[0].name, "read_file");
1062
assert_eq!(tools[1].name, "create_directory");
1063
assert_eq!(tools[2].name, "list_directory");
1064
assert_eq!(tools[3].name, "search_files");
920
- assert_eq!(tools[4].name, "create_file");
921
- assert_eq!(tools[5].name, "edit_file");
922
- assert_eq!(tools[6].name, "delete_file");
923
- assert_eq!(tools[7].name, "run_command");
1065
+ assert_eq!(tools[4].name, "read_website");
1066
+ assert_eq!(tools[5].name, "create_file");
1067
+ assert_eq!(tools[6].name, "edit_file");
1068
+ assert_eq!(tools[7].name, "delete_file");
1069
+ assert_eq!(tools[8].name, "run_command");
1070
}
1071
1072
#[test]
@@ -938,6 +1084,72 @@ mod tests {
1084
}
1085
}
1086
1087
+ // ── read_website tests ───────────────────────────────────────────────
1088
+
1089
+ #[test]
1090
+ fn test_read_website_missing_url() {
1091
+ let result = exec_read_website("{}");
1092
+ assert!(result.contains("missing required parameter"));
1093
+ }
1094
+
1095
+ #[test]
1096
+ fn test_read_website_invalid_scheme() {
1097
+ let result = exec_read_website(r#"{"url": "file:///tmp/test.html"}"#);
1098
+ assert!(result.contains("url must start with http:// or https://"));
1099
+ }
1100
+
1101
+ #[test]
1102
+ fn test_read_website_extracts_title_from_html() {
1103
+ let body = r#"
1104
+ <html>
1105
+ <head>
1106
+ <title>Qwen 3.6 27B</title>
1107
+ </head>
1108
+ <body>
1109
+ <h1>Model card</h1>
1110
+ <p>Large language model.</p>
1111
+ </body>
1112
+ </html>
1113
+ "#;
1114
+
1115
+ let title = Regex::new(r"(?is)<title[^>]*>(.*?)</title>")
1116
+ .unwrap()
1117
+ .captures(body)
1118
+ .and_then(|captures| captures.get(1))
1119
+ .map(|m| {
1120
+ Regex::new(r"\s+")
1121
+ .unwrap()
1122
+ .replace_all(m.as_str(), " ")
1123
+ .trim()
1124
+ .to_string()
1125
+ })
1126
+ .filter(|title| !title.is_empty());
1127
+
1128
+ assert_eq!(title.as_deref(), Some("Qwen 3.6 27B"));
1129
+ }
1130
+
1131
+ #[test]
1132
+ fn test_read_website_metadata_includes_final_url_header() {
1133
+ let final_url = "https://huggingface.co/Qwen/Qwen3.6-27B";
1134
+ let title = Some("Qwen 3.6 27B".to_string());
1135
+ let cleaned = "Model card\nLarge language model.".to_string();
1136
+
1137
+ let mut metadata = vec![format!("URL: {final_url}")];
1138
+ if let Some(title) = &title {
1139
+ metadata.push(format!("Title: {title}"));
1140
+ }
1141
+
1142
+ let body_text = match title {
1143
+ Some(_) => cleaned,
1144
+ None => cleaned,
1145
+ };
1146
+
1147
+ let output = format!("{}\n\n{}", metadata.join("\n"), body_text);
1148
+
1149
+ assert!(output.starts_with("URL: https://huggingface.co/Qwen/Qwen3.6-27B"));
1150
+ assert!(output.contains("\nTitle: Qwen 3.6 27B\n\n"));
1151
+ }
1152
+
1153
// ── create_directory tests ───────────────────────────────────────────
1154
1155
#[test]