From b96e83dbbe58c3af5ac8bbb6bc62d87ea96e5b71 Mon Sep 17 00:00:00 2001 From: martin legrand Date: Thu, 29 May 2025 21:33:48 +0200 Subject: [PATCH] feat : latest docker attempt + fix attempt for #249 --- .env.example | 1 + Dockerfile.backend | 46 +++++++++++++++-------------- sources/agents/browser_agent.py | 21 ++++++++----- sources/browser.py | 7 +++-- tests/test_browser_agent_parsing.py | 2 +- 5 files changed, 44 insertions(+), 33 deletions(-) diff --git a/.env.example b/.env.example index 0e98844..3b38eb2 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ SEARXNG_BASE_URL="http://127.0.0.1:8080" +TOKENIZERS_PARALLELISM="false" OPENAI_API_KEY='xxxxx' DEEPSEEK_API_KEY='xxxxx' OPENROUTER_API_KEY='xxxxx' diff --git a/Dockerfile.backend b/Dockerfile.backend index befe60f..228618e 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -60,33 +60,35 @@ COPY sources/ ./sources/ COPY prompts/ ./prompts/ COPY crx/ crx/ COPY llm_router/ llm_router/ -COPY .env . -COPY config.ini . +RUN ls +COPY .env.example .env # Install Chrome and ChromeDriver from chrome-for-testing -RUN wget -O chrome-headless-shell-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chrome-headless-shell-linux64.zip && \ - wget -O chromedriver-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chromedriver-linux64.zip && \ - unzip chrome-headless-shell-linux64.zip && \ - unzip chromedriver-linux64.zip && \ - mkdir -p /opt/google && \ - ls -la && \ - mv chrome-headless-shell-linux64 /opt/google/chrome && \ - mv chromedriver-linux64/chromedriver /usr/local/bin/chromedriver && \ - chmod +x /opt/google/chrome/chrome-headless-shell && \ - chmod +x /usr/local/bin/chromedriver +#RUN wget -O chrome-headless-shell-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chrome-headless-shell-linux64.zip && \ +# wget -O chromedriver-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chromedriver-linux64.zip && \ +# unzip chrome-headless-shell-linux64.zip && \ +# unzip chromedriver-linux64.zip && \ +# mkdir -p /opt/google && \ +# ls -la && \ +# mv chrome-headless-shell-linux64 /opt/google/chrome && \ +# mv chromedriver-linux64/chromedriver /usr/local/bin/chromedriver && \ +# chmod +x /opt/google/chrome/chrome-headless-shell && \ +# chmod +x /usr/local/bin/chromedriver -RUN ln -s /opt/google/chrome/chrome /usr/local/bin/chrome +RUN wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - && \ + echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list && \ + apt-get update && \ + apt-get install -y google-chrome-stable && \ + rm -rf /var/lib/apt/lists/* -# Debug ChromeDriver -RUN echo "=== ChromeDriver Debug ===" && \ - /usr/local/bin/chromedriver --version && \ - echo "Chrome binary:" && \ - /opt/google/chrome/chrome --version && \ - echo "Testing ChromeDriver startup:" && \ - timeout 5 /usr/local/bin/chromedriver --port=9999 || echo "ChromeDriver failed to start" +# Install matching ChromeDriver +RUN CHROME_VERSION=$(google-chrome --version | grep -oP '\d+\.\d+\.\d+') && \ + wget -O chromedriver.zip "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_VERSION%%.*}/chromedriver_linux64.zip" && \ + unzip chromedriver.zip && \ + mv chromedriver /usr/local/bin/ && \ + chmod +x /usr/local/bin/chromedriver && \ + rm chromedriver.zip -ENV CHROME_BIN=/opt/google/chrome/chrome -ENV CHROMEDRIVER_PATH=/usr/local/bin/chromedriver ENV DISPLAY=:99 # Expose port diff --git a/sources/agents/browser_agent.py b/sources/agents/browser_agent.py index 3817fb3..3f5de5d 100644 --- a/sources/agents/browser_agent.py +++ b/sources/agents/browser_agent.py @@ -41,7 +41,7 @@ class BrowserAgent(Agent): self.memory = Memory(self.load_prompt(prompt_path), recover_last_session=False, # session recovery in handled by the interaction class memory_compression=False, - model_provider=provider.get_model_name()) + model_provider=provider.get_model_name() if provider else None) def get_today_date(self) -> str: """Get the date""" @@ -77,14 +77,14 @@ class BrowserAgent(Agent): def get_unvisited_links(self) -> List[str]: return "\n".join([f"[{i}] {link}" for i, link in enumerate(self.navigable_links) if link not in self.search_history]) - def make_newsearch_prompt(self, user_prompt: str, search_result: dict) -> str: + def make_newsearch_prompt(self, prompt: str, search_result: dict) -> str: search_choice = self.stringify_search_results(search_result) self.logger.info(f"Search results: {search_choice}") return f""" Based on the search result: {search_choice} Your goal is to find accurate and complete information to satisfy the user’s request. - User request: {user_prompt} + User request: {prompt} To proceed, choose a relevant link from the search results. Announce your choice by saying: "I will navigate to " Do not explain your choice. """ @@ -235,13 +235,17 @@ class BrowserAgent(Agent): return links def select_link(self, links: List[str]) -> str | None: + """ + Select the first unvisited link that is not the current page. + Preference is given to links not in search_history. + """ for lk in links: - if lk == self.current_page: - self.logger.info(f"Already visited {lk}. Skipping.") + if lk == self.current_page or lk in self.search_history: + self.logger.info(f"Skipping already visited or current link: {lk}") continue self.logger.info(f"Selected link: {lk}") return lk - self.logger.warning("No link selected.") + self.logger.warning("No suitable link selected.") return None def get_page_text(self, limit_to_model_ctx = False) -> str: @@ -396,7 +400,10 @@ class BrowserAgent(Agent): if (link == None and len(extracted_form) < 3) or Action.GO_BACK.value in answer or link in self.search_history: pretty_print(f"Going back to results. Still {len(unvisited)}", color="status") self.status_message = "Going back to search results..." - prompt = self.make_newsearch_prompt(user_prompt, unvisited) + request_prompt = user_prompt + if link is None: + request_prompt += f"\nYou previously choosen:\n{self.last_answer} but the website is unavailable. Consider other options." + prompt = self.make_newsearch_prompt(request_prompt, unvisited) self.search_history.append(link) self.current_page = link continue diff --git a/sources/browser.py b/sources/browser.py index 496cfdb..ea7e088 100644 --- a/sources/browser.py +++ b/sources/browser.py @@ -136,18 +136,21 @@ def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx if headless: #chrome_options.add_argument("--headless") chrome_options.add_argument("--headless=new") + chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--disable-webgl") + chrome_options.add_argument("--remote-debugging-port=9222") user_data_dir = tempfile.mkdtemp() user_agent = get_random_user_agent() width, height = (1920, 1080) chrome_options.add_argument(f"--user-data-dir={user_data_dir}") chrome_options.add_argument(f"--accept-lang={lang}-{lang.upper()},{lang};q=0.9") + chrome_options.add_argument("--disable-extensions") + chrome_options.add_argument("--disable-background-timer-throttling") chrome_options.add_argument("--timezone=Europe/Paris") chrome_options.add_argument("--no-sandbox") chrome_options.add_argument('--disable-dev-shm-usage') chrome_options.add_argument('--remote-debugging-port=9222') - chrome_options.add_argument('--disable-extensions') chrome_options.add_argument('--disable-background-timer-throttling') chrome_options.add_argument('--disable-backgrounding-occluded-windows') chrome_options.add_argument('--disable-renderer-backgrounding') @@ -721,8 +724,6 @@ if __name__ == "__main__": input("press enter to continue") print("AntiCaptcha / Form Test") - browser.go_to("https://www.google.com/recaptcha/api2/demo") - time.sleep(50) browser.go_to("https://bot.sannysoft.com") time.sleep(5) #txt = browser.get_text() diff --git a/tests/test_browser_agent_parsing.py b/tests/test_browser_agent_parsing.py index aac95bc..40d0eec 100644 --- a/tests/test_browser_agent_parsing.py +++ b/tests/test_browser_agent_parsing.py @@ -23,7 +23,7 @@ class TestBrowserAgentParsing(unittest.TestCase): "https://thriveonai.com/15-ai-startups-in-japan-to-take-note-of", "www.google.com", "https://test.org/about?page=1", - "https://weatherstack.com/documentation", + "https://weatherstack.com/documentation" ] result = self.agent.extract_links(test_text) self.assertEqual(result, expected)