diff --git a/AutonomousAI/browse.py b/AutonomousAI/browse.py index 361f0cb8..4cc45e79 100644 --- a/AutonomousAI/browse.py +++ b/AutonomousAI/browse.py @@ -24,23 +24,6 @@ def scrape_text(url): return text -def scrape_main_content(url): - response = requests.get(url) - - # Try using Readability - doc = Document(response.text) - content = doc.summary() - soup = BeautifulSoup(content, "html.parser") - text = soup.get_text('\n', strip=True) - - # Check if Readability provided a satisfactory result (e.g., a minimum length) - # min_length = 50 - # if len(text) < min_length: - # # Fallback to the custom function - # text = scrape_main_content_custom(response.text) - - return text - def split_text(text, max_length=8192): paragraphs = text.split("\n") current_length = 0 diff --git a/AutonomousAI/commands.py b/AutonomousAI/commands.py index 85a183a9..18cd8142 100644 --- a/AutonomousAI/commands.py +++ b/AutonomousAI/commands.py @@ -60,7 +60,7 @@ def google_search(query, num_results = 3): return json.dumps(search_results, ensure_ascii=False, indent=4) def transcribe_summarise(url): - text = browse.scrape_main_content(url) + text = browse.scrape_text(url) summary = browse.summarize_text(text) return """ "Result" : """ + summary