Feat : web navigation improvement, better prompting

sources/browser.py

@@ -13,6 +13,7 @@ import markdownify
 import logging
 import sys
 import re
+from urllib.parse import urlparse
 
 class Browser:
     def __init__(self, headless=False, anticaptcha_install=False):
@@ -58,7 +59,8 @@ class Browser:
                 os.path.join(os.environ.get("LOCALAPPDATA", ""), "Google\\Chrome\\Application\\chrome.exe")  # User install
             ]
         elif sys.platform.startswith("darwin"):  # macOS
-            paths = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta"]
+            paths = ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                     "/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta"]
         else:  # Linux
             paths = ["/usr/bin/google-chrome", "/usr/bin/chromium-browser", "/usr/bin/chromium"]
 
@@ -79,19 +81,6 @@ class Browser:
             self.logger.error(f"Error navigating to {url}: {str(e)}")
             return False
     
-    def is_sentence(self, text):
-        """Check if the text qualifies as a meaningful sentence or contains important error codes."""
-        text = text.strip()
-        error_codes = ["404", "403", "500", "502", "503"]
-        if any(code in text for code in error_codes):
-            return True
-        words = text.split()
-        word_count = len(words)
-        has_punctuation = text.endswith(('.', '!', '?'))
-        is_long_enough = word_count > 5
-        has_letters = any(word.isalpha() for word in words)
-        return (word_count >= 5 and (has_punctuation or is_long_enough) and has_letters)
-
     def is_sentence(self, text):
         """Check if the text qualifies as a meaningful sentence or contains important error codes."""
         text = text.strip()
@@ -118,10 +107,8 @@ class Browser:
             lines = (line.strip() for line in text.splitlines())
             chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
             text = "\n".join(chunk for chunk in chunks if chunk and self.is_sentence(chunk))
-            
-            markdown_text = markdownify.markdownify(text, heading_style="ATX")
-            
-            return markdown_text
+            #markdown_text = markdownify.markdownify(text, heading_style="ATX")
+            return "[Start of page]\n" + text + "\n[End of page]"
         except Exception as e:
             self.logger.error(f"Error getting text: {str(e)}")
             return None
@@ -142,6 +129,22 @@ class Browser:
             if essential_params:
                 return f"{base_url}?{'&'.join(essential_params)}"
         return base_url
+    
+    def is_link_valid(self, url):
+        """Check if a URL is a valid link (page, not related to icon or metadata)."""
+        if len(url) > 64:
+            return False
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme or not parsed_url.netloc:
+            return False
+        if re.search(r'/\d+$', parsed_url.path):
+            return False
+        image_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp']
+        metadata_extensions = ['.ico', '.xml', '.json', '.rss', '.atom']
+        for ext in image_extensions + metadata_extensions:
+            if url.lower().endswith(ext):
+                return False
+        return True
 
     def get_navigable(self):
         """Get all navigable links on the current page."""
@@ -159,7 +162,7 @@ class Browser:
                     })
             
             self.logger.info(f"Found {len(links)} navigable links")
-            return [self.clean_url(link['url']) for link in links if (link['is_displayed'] == True and len(link['url']) < 64)]
+            return [self.clean_url(link['url']) for link in links if (link['is_displayed'] == True and self.is_link_valid(link['url']))]
         except Exception as e:
             self.logger.error(f"Error getting navigable links: {str(e)}")
             return []
@@ -225,7 +228,7 @@ if __name__ == "__main__":
     browser = Browser(headless=False)
     
     try:
-        browser.go_to("https://www.seoul.co.kr/")
+        browser.go_to("https://github.com/Fosowl/agenticSeek")
         text = browser.get_text()
         print("Page Text in Markdown:")
         print(text)