feat : latest docker attempt + fix attempt for #249

.env.example

@@ -1,4 +1,5 @@
 SEARXNG_BASE_URL="http://127.0.0.1:8080"
+TOKENIZERS_PARALLELISM="false"
 OPENAI_API_KEY='xxxxx'
 DEEPSEEK_API_KEY='xxxxx'
 OPENROUTER_API_KEY='xxxxx'

Dockerfile.backend

@@ -60,33 +60,35 @@ COPY sources/ ./sources/
 COPY prompts/ ./prompts/
 COPY crx/ crx/
 COPY llm_router/ llm_router/
-COPY .env .
-COPY config.ini .
+RUN ls
+COPY .env.example .env
 
 # Install Chrome and ChromeDriver from chrome-for-testing
-RUN wget -O chrome-headless-shell-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chrome-headless-shell-linux64.zip && \ 
-    wget -O chromedriver-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chromedriver-linux64.zip && \
-    unzip chrome-headless-shell-linux64.zip && \
-    unzip chromedriver-linux64.zip && \
-    mkdir -p /opt/google && \
-    ls -la && \
-    mv chrome-headless-shell-linux64 /opt/google/chrome && \
-    mv chromedriver-linux64/chromedriver /usr/local/bin/chromedriver && \
-    chmod +x /opt/google/chrome/chrome-headless-shell && \
-    chmod +x /usr/local/bin/chromedriver
+#RUN wget -O chrome-headless-shell-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chrome-headless-shell-linux64.zip && \ 
+#    wget -O chromedriver-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chromedriver-linux64.zip && \
+#    unzip chrome-headless-shell-linux64.zip && \
+#    unzip chromedriver-linux64.zip && \
+#    mkdir -p /opt/google && \
+#    ls -la && \
+#    mv chrome-headless-shell-linux64 /opt/google/chrome && \
+#    mv chromedriver-linux64/chromedriver /usr/local/bin/chromedriver && \
+#    chmod +x /opt/google/chrome/chrome-headless-shell && \
+#    chmod +x /usr/local/bin/chromedriver
 
-RUN ln -s /opt/google/chrome/chrome /usr/local/bin/chrome
+RUN wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - && \
+    echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list && \
+    apt-get update && \
+    apt-get install -y google-chrome-stable && \
+    rm -rf /var/lib/apt/lists/*
 
-# Debug ChromeDriver
-RUN echo "=== ChromeDriver Debug ===" && \
-    /usr/local/bin/chromedriver --version && \
-    echo "Chrome binary:" && \
-    /opt/google/chrome/chrome --version && \
-    echo "Testing ChromeDriver startup:" && \
-    timeout 5 /usr/local/bin/chromedriver --port=9999 || echo "ChromeDriver failed to start"
+# Install matching ChromeDriver
+RUN CHROME_VERSION=$(google-chrome --version | grep -oP '\d+\.\d+\.\d+') && \
+    wget -O chromedriver.zip "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_VERSION%%.*}/chromedriver_linux64.zip" && \
+    unzip chromedriver.zip && \
+    mv chromedriver /usr/local/bin/ && \
+    chmod +x /usr/local/bin/chromedriver && \
+    rm chromedriver.zip
 
-ENV CHROME_BIN=/opt/google/chrome/chrome
-ENV CHROMEDRIVER_PATH=/usr/local/bin/chromedriver
 ENV DISPLAY=:99
 
 # Expose port

sources/agents/browser_agent.py

@@ -41,7 +41,7 @@ class BrowserAgent(Agent):
         self.memory = Memory(self.load_prompt(prompt_path),
                         recover_last_session=False, # session recovery in handled by the interaction class
                         memory_compression=False,
-                        model_provider=provider.get_model_name())
+                        model_provider=provider.get_model_name() if provider else None)
     
     def get_today_date(self) -> str:
         """Get the date"""
@@ -77,14 +77,14 @@ class BrowserAgent(Agent):
     def get_unvisited_links(self) -> List[str]:
         return "\n".join([f"[{i}] {link}" for i, link in enumerate(self.navigable_links) if link not in self.search_history])
 
-    def make_newsearch_prompt(self, user_prompt: str, search_result: dict) -> str:
+    def make_newsearch_prompt(self, prompt: str, search_result: dict) -> str:
         search_choice = self.stringify_search_results(search_result)
         self.logger.info(f"Search results: {search_choice}")
         return f"""
         Based on the search result:
         {search_choice}
         Your goal is to find accurate and complete information to satisfy the user’s request.
-        User request: {user_prompt}
+        User request: {prompt}
         To proceed, choose a relevant link from the search results. Announce your choice by saying: "I will navigate to <link>"
         Do not explain your choice.
         """
@@ -235,13 +235,17 @@ class BrowserAgent(Agent):
         return links
     
     def select_link(self, links: List[str]) -> str | None:
+        """
+        Select the first unvisited link that is not the current page.
+        Preference is given to links not in search_history.
+        """
         for lk in links:
-            if lk == self.current_page:
-                self.logger.info(f"Already visited {lk}. Skipping.")
+            if lk == self.current_page or lk in self.search_history:
+                self.logger.info(f"Skipping already visited or current link: {lk}")
                 continue
             self.logger.info(f"Selected link: {lk}")
             return lk
-        self.logger.warning("No link selected.")
+        self.logger.warning("No suitable link selected.")
         return None
     
     def get_page_text(self, limit_to_model_ctx = False) -> str:
@@ -396,7 +400,10 @@ class BrowserAgent(Agent):
             if (link == None and len(extracted_form) < 3) or Action.GO_BACK.value in answer or link in self.search_history:
                 pretty_print(f"Going back to results. Still {len(unvisited)}", color="status")
                 self.status_message = "Going back to search results..."
-                prompt = self.make_newsearch_prompt(user_prompt, unvisited)
+                request_prompt = user_prompt
+                if link is None:
+                    request_prompt += f"\nYou previously choosen:\n{self.last_answer} but the website is unavailable. Consider other options."
+                prompt = self.make_newsearch_prompt(request_prompt, unvisited)
                 self.search_history.append(link)
                 self.current_page = link
                 continue

sources/browser.py

@@ -136,18 +136,21 @@ def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx
     if headless:
         #chrome_options.add_argument("--headless")
         chrome_options.add_argument("--headless=new")
+        chrome_options.add_argument("--disable-dev-shm-usage")
         chrome_options.add_argument("--disable-gpu")
         chrome_options.add_argument("--disable-webgl")
+        chrome_options.add_argument("--remote-debugging-port=9222")
     user_data_dir = tempfile.mkdtemp()
     user_agent = get_random_user_agent()
     width, height = (1920, 1080)
     chrome_options.add_argument(f"--user-data-dir={user_data_dir}")
     chrome_options.add_argument(f"--accept-lang={lang}-{lang.upper()},{lang};q=0.9")
+    chrome_options.add_argument("--disable-extensions")
+    chrome_options.add_argument("--disable-background-timer-throttling")
     chrome_options.add_argument("--timezone=Europe/Paris")
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument('--disable-dev-shm-usage')
     chrome_options.add_argument('--remote-debugging-port=9222')
-    chrome_options.add_argument('--disable-extensions')
     chrome_options.add_argument('--disable-background-timer-throttling')
     chrome_options.add_argument('--disable-backgrounding-occluded-windows')
     chrome_options.add_argument('--disable-renderer-backgrounding')
@@ -721,8 +724,6 @@ if __name__ == "__main__":
     
     input("press enter to continue")
     print("AntiCaptcha / Form Test")
-    browser.go_to("https://www.google.com/recaptcha/api2/demo")
-    time.sleep(50)
     browser.go_to("https://bot.sannysoft.com")
     time.sleep(5)
     #txt = browser.get_text()

tests/test_browser_agent_parsing.py

@@ -23,7 +23,7 @@ class TestBrowserAgentParsing(unittest.TestCase):
             "https://thriveonai.com/15-ai-startups-in-japan-to-take-note-of",
             "www.google.com",
             "https://test.org/about?page=1",
-            "https://weatherstack.com/documentation",
+            "https://weatherstack.com/documentation"
         ]
         result = self.agent.extract_links(test_text)
         self.assertEqual(result, expected)