Add read_website tool for fetching and extracting web page text

index 322fa22..8c9ad8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5283,6 +5283,7 @@ dependencies = [ "onde", "ratatui", "regex", + "reqwest 0.12.28", "serde_json", "tokio", "tokio-util",

index 8f77947..99e2b10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,4 +41,5 @@ log = "0.4" tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] } serde_json = "1" regex = "1" +reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] } uuid = { version = "1", features = ["v4"] }

index 703515d..a100e09 100644 --- a/src/main.rs +++ b/src/main.rs @@ -109,14 +109,15 @@ Never introduce yourself unless asked. Jump straight into the answer. \ Keep answers short. Write idiomatic code. \ Fix root causes, not symptoms. -You have access to tools that let you read files, create directories, list \ -directories, search code, create new files, edit existing files, delete files, \ -and run shell commands. You can also use git directly through shell commands, \ -including `git init` and normal git workflows. Use them proactively. Read the \ -code before answering. Prefer absolute paths when referring to files and \ -directories, especially in protocol-facing output and tool arguments. Create \ -directories when needed. Run builds, tests, and git commands after making \ -changes. Ground your answers in the actual code, not in guesses. +You have access to tools that let you read files, read websites directly from \ +http and https URLs, create directories, list directories, search code, create \ +new files, edit existing files, delete files, and run shell commands. You can \ +also use git directly through shell commands, including `git init` and normal \ +git workflows. Use them proactively. Read the code or website before answering. \ +Prefer absolute paths when referring to files and directories, especially in \ +protocol-facing output and tool arguments. Create directories when needed. Run \ +builds, tests, and git commands after making changes. Ground your answers in \ +the actual code or fetched page content, not in guesses. Tool-use heuristics: - prefer absolute paths over relative paths when you mention, return, or pass \

index 789749d..743df3f 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -19,6 +19,10 @@ //! - `edit_file` — replace an exact old-text span with new text in an existing file //! - `delete_file` — delete a file or empty directory at the given path //! +//! # Web Tools +//! +//! - `read_website` — fetch a web page and return readable text content +//! //! # Shell Tools //! //! - `run_command` — run shell commands, including git porcelain and plumbing commands @@ -29,6 +33,11 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +const WEBSITE_READ_CHAR_LIMIT: usize = 20_000; +const WEBSITE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); +const WEBSITE_USER_AGENT: &str = + "siGit/0.1 (+https://github.com/getsigit/sigit; website-reading tool)"; + /// Maximum characters returned from `read_file` before truncation. const READ_FILE_CHAR_LIMIT: usize = 10_000; @@ -127,6 +136,25 @@ pub fn all_tools() -> Vec<AgentTool> { "additionalProperties": false }), }, + AgentTool { + name: "read_website", + description: "Fetch a web page and return readable text content. \ + Use this when the user gives you a URL and asks you to read, \ + summarize, inspect, or extract information from the page. \ + Supports normal http and https URLs. Output is truncated if the \ + page is very large.", + parameters_schema: json!({ + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Absolute http or https URL to fetch." + } + }, + "required": ["url"], + "additionalProperties": false + }), + }, AgentTool { name: "create_file", description: "Create a new file at the given path with the provided content. \ @@ -239,6 +267,7 @@ pub fn execute_tool(name: &str, arguments: &str) -> String { "read_file" => exec_read_file(arguments), "list_directory" => exec_list_directory(arguments), "search_files" => exec_search_files(arguments), + "read_website" => exec_read_website(arguments), "create_directory" => exec_create_directory(arguments), "create_file" => exec_create_file(arguments), "edit_file" => exec_edit_file(arguments), @@ -485,6 +514,122 @@ fn search_file(path: &Path, re: &Regex, matches: &mut Vec<String>) { } } +/// ── read_website ───────────────────────────────────────────────────────────── + +fn exec_read_website(arguments: &str) -> String { + let args: Value = match serde_json::from_str(arguments) { + Ok(v) => v, + Err(err) => return format!("Error: failed to parse arguments: {err}"), + }; + + let url = match args.get("url").and_then(Value::as_str) { + Some(u) => u, + None => return "Error: missing required parameter \"url\"".to_string(), + }; + + if !(url.starts_with("http://") || url.starts_with("https://")) { + return format!("Error: url must start with http:// or https://: {url}"); + } + + let client = match reqwest::blocking::Client::builder() + .timeout(WEBSITE_READ_TIMEOUT) + .user_agent(WEBSITE_USER_AGENT) + .build() + { + Ok(client) => client, + Err(err) => return format!("Error: failed to build website client: {err}"), + }; + + let response = match client.get(url).send() { + Ok(r) => r, + Err(err) => return format!("Error: failed to fetch website: {err}"), + }; + + let final_url = response.url().to_string(); + let status = response.status(); + if !status.is_success() { + return format!("Error: website returned HTTP {status} for {final_url}"); + } + + let body = match response.text() { + Ok(text) => text, + Err(err) => return format!("Error: failed to read website body: {err}"), + }; + + let title = Regex::new(r"(?is)<title[^>]*>(.*?)</title>") + .unwrap() + .captures(&body) + .and_then(|captures| captures.get(1)) + .map(|m| { + Regex::new(r"\s+") + .unwrap() + .replace_all(m.as_str(), " ") + .trim() + .to_string() + }) + .filter(|title| !title.is_empty()); + + let with_block_breaks = Regex::new( + r"(?is)</?(?:p|div|section|article|main|aside|header|footer|nav|li|ul|ol|h1|h2|h3|h4|h5|h6|br|tr|td|th)[^>]*>", + ) + .unwrap() + .replace_all(&body, "\n"); + let without_scripts = Regex::new(r"(?is)<script[^>]*>.*?</script>") + .unwrap() + .replace_all(&with_block_breaks, " "); + let without_styles = Regex::new(r"(?is)<style[^>]*>.*?</style>") + .unwrap() + .replace_all(&without_scripts, " "); + let without_tags = Regex::new(r"(?is)<[^>]+>") + .unwrap() + .replace_all(&without_styles, " "); + let normalized_newlines = without_tags + .replace("&nbsp;", " ") + .replace("&amp;", "&") + .replace("&lt;", "<") + .replace("&gt;", ">") + .replace("&quot;", "\"") + .replace("&#39;", "'"); + let collapsed_lines = Regex::new(r"[ \t]+") + .unwrap() + .replace_all(&normalized_newlines, " "); + let collapsed_breaks = Regex::new(r"\n\s*\n+") + .unwrap() + .replace_all(&collapsed_lines, "\n\n"); + let cleaned = collapsed_breaks + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect::<Vec<_>>() + .join("\n"); + + if cleaned.is_empty() { + return format!("Fetched {url}, but no readable text content was found."); + } + + let mut metadata = vec![format!("URL: {final_url}")]; + if let Some(title) = &title { + metadata.push(format!("Title: {title}")); + } + + let body_text = match title { + Some(title) if !cleaned.starts_with(&title) => cleaned, + _ => cleaned, + }; + + let output = format!("{}\n\n{}", metadata.join("\n"), body_text); + + if output.len() > WEBSITE_READ_CHAR_LIMIT { + let truncated: String = output.chars().take(WEBSITE_READ_CHAR_LIMIT).collect(); + return format!( + "{truncated}\n\n--- truncated (showing {WEBSITE_READ_CHAR_LIMIT} of {} characters) ---", + output.len() + ); + } + + output +} + /// ── create_directory ───────────────────────────────────────────────────────── /// Create a directory and any missing parent directories. @@ -912,15 +1057,16 @@ mod tests { #[test] fn test_all_tools_count() { let tools = all_tools(); - assert_eq!(tools.len(), 8); + assert_eq!(tools.len(), 9); assert_eq!(tools[0].name, "read_file"); assert_eq!(tools[1].name, "create_directory"); assert_eq!(tools[2].name, "list_directory"); assert_eq!(tools[3].name, "search_files"); - assert_eq!(tools[4].name, "create_file"); - assert_eq!(tools[5].name, "edit_file"); - assert_eq!(tools[6].name, "delete_file"); - assert_eq!(tools[7].name, "run_command"); + assert_eq!(tools[4].name, "read_website"); + assert_eq!(tools[5].name, "create_file"); + assert_eq!(tools[6].name, "edit_file"); + assert_eq!(tools[7].name, "delete_file"); + assert_eq!(tools[8].name, "run_command"); } #[test] @@ -938,6 +1084,72 @@ mod tests { } } + // ── read_website tests ─────────────────────────────────────────────── + + #[test] + fn test_read_website_missing_url() { + let result = exec_read_website("{}"); + assert!(result.contains("missing required parameter")); + } + + #[test] + fn test_read_website_invalid_scheme() { + let result = exec_read_website(r#"{"url": "file:///tmp/test.html"}"#); + assert!(result.contains("url must start with http:// or https://")); + } + + #[test] + fn test_read_website_extracts_title_from_html() { + let body = r#" + <html> + <head> + <title>Qwen 3.6 27B</title> + </head> + <body> + <h1>Model card</h1> + <p>Large language model.</p> + </body> + </html> + "#; + + let title = Regex::new(r"(?is)<title[^>]*>(.*?)</title>") + .unwrap() + .captures(body) + .and_then(|captures| captures.get(1)) + .map(|m| { + Regex::new(r"\s+") + .unwrap() + .replace_all(m.as_str(), " ") + .trim() + .to_string() + }) + .filter(|title| !title.is_empty()); + + assert_eq!(title.as_deref(), Some("Qwen 3.6 27B")); + } + + #[test] + fn test_read_website_metadata_includes_final_url_header() { + let final_url = "https://huggingface.co/Qwen/Qwen3.6-27B"; + let title = Some("Qwen 3.6 27B".to_string()); + let cleaned = "Model card\nLarge language model.".to_string(); + + let mut metadata = vec![format!("URL: {final_url}")]; + if let Some(title) = &title { + metadata.push(format!("Title: {title}")); + } + + let body_text = match title { + Some(_) => cleaned, + None => cleaned, + }; + + let output = format!("{}\n\n{}", metadata.join("\n"), body_text); + + assert!(output.starts_with("URL: https://huggingface.co/Qwen/Qwen3.6-27B")); + assert!(output.contains("\nTitle: Qwen 3.6 27B\n\n")); + } + // ── create_directory tests ─────────────────────────────────────────── #[test]