add webpage content tool
John Bollenbacher committed
Jul 29, 2024 at 15:28 UTC
5e7ab53b5a4e8e66e7bde9fe88f939e27ceee248
1 file changed
+39
python/tools/webpage_content_tool.py
new
+39
@@ -0,0 +1,39 @@
1
+import requests
2
+from bs4 import BeautifulSoup
3
+from urllib.parse import urlparse
4
+from newspaper import Article
5
+from python.helpers.tool import Tool, Response
6
+
7
+class WebpageContentTool(Tool):
8
+ def execute(self, url="", **kwargs):
9
+ if not url:
10
+ return Response(message="Error: No URL provided.", break_loop=False)
11
+
12
+ try:
13
+ # Validate URL
14
+ parsed_url = urlparse(url)
15
+ if not all([parsed_url.scheme, parsed_url.netloc]):
16
+ return Response(message="Error: Invalid URL format.", break_loop=False)
17
+
18
+ # Fetch webpage content
19
+ response = requests.get(url, timeout=10)
20
+ response.raise_for_status()
21
+
22
+ # Use newspaper3k for article extraction
23
+ article = Article(url)
24
+ article.download()
25
+ article.parse()
26
+
27
+ # If it's not an article, fall back to BeautifulSoup
28
+ if not article.text:
29
+ soup = BeautifulSoup(response.content, 'html.parser')
30
+ text_content = ' '.join(soup.stripped_strings)
31
+ else:
32
+ text_content = article.text
33
+
34
+ return Response(message=f"Webpage content:\n\n{text_content}", break_loop=False)
35
+
36
+ except requests.RequestException as e:
37
+ return Response(message=f"Error fetching webpage: {str(e)}", break_loop=False)
38
+ except Exception as e:
39
+ return Response(message=f"An error occurred: {str(e)}", break_loop=False)
\ No newline at end of file