Feat : prompt improve and start link check to avoid paywall

sources/agents/agent.py

@@ -34,7 +34,7 @@ class Agent():
                        name: str,
                        prompt_path:str,
                        provider,
-                       recover_last_session=False) -> None:
+                       recover_last_session=True) -> None:
         self.agent_name = name
         self.role = None
         self.current_directory = os.getcwd()

sources/agents/browser_agent.py

@@ -19,6 +19,7 @@ class BrowserAgent(Agent):
         self.browser.go_to("https://github.com/")
         self.search_history = []
         self.navigable_links = []
+        self.ai_notes = []
     
     def extract_links(self, search_result: str):
         links = re.findall(r'https?://[^\s]+', search_result)
@@ -64,8 +65,8 @@ class BrowserAgent(Agent):
         If you found a clear answer, please say "REQUEST_EXIT".
         You must choose a link to navigate to, go back or exit.
         Do not explain your choice.
+        You can take note about your finding with TAKE_NOTE("<your note>")
         """
-
     
     def llm_decide(self, prompt):
         animate_thinking("Thinking...", color="status")

sources/tools/webSearch.py

@@ -22,6 +22,37 @@ class webSearch(Tools):
         super().__init__()
         self.tag = "web_search"
         self.api_key = api_key or os.getenv("SERPAPI_KEY")  # Requires a SerpApi key
+        self.paywall_keywords = [
+            "subscribe", "paywall", "login to continue", "access denied", "restricted content"
+        ]
+
+    async def link_valid(self, session, link):
+        """asyncronously check if a link is shit."""
+        if not link.startswith("http"):
+            return "Status: Invalid URL"
+        try:
+            async with session.get(link, timeout=aiohttp.ClientTimeout(total=5)) as response:
+                status = response.status
+                if status == 200:
+                    content = await response.text(encoding='utf-8', errors='ignore')[:1000]
+                    if any(keyword in content.lower() for keyword in self.paywall_keywords):
+                        return "Status: Possible Paywall"
+                    return "Status: Accessible"
+                elif status == 404:
+                    return "Status: 404 Not Found"
+                elif status == 403:
+                    return "Status: 403 Forbidden"
+                else:
+                    return f"Status: {status} {response.reason}"
+        except Exception as e:
+            return f"Error: {str(e)}"
+
+    async def check_all_links(self, links):
+        """Check all links asynchronously using a single session."""
+        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
+        async with aiohttp.ClientSession(headers=headers) as session:
+            tasks = [self.link_valid(session, link) for link in links]
+            return await asyncio.gather(*tasks)
 
     def execute(self, blocks: str, safety: bool = True) -> str:
         if self.api_key is None: