Merge pull request #188 from Fosowl/dev

Better browser fingerprint spoofing + Markdown support for frontend + block color display fix

api.py

@@ -34,7 +34,7 @@ config.read('config.ini')
 
 api.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=["http://localhost", "http://localhost:3000"],
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
@@ -58,7 +58,7 @@ def initialize_system():
     logger.info(f"Provider initialized: {provider.provider_name} ({provider.model})")
 
     browser = Browser(
-        create_driver(headless=config.getboolean('BROWSER', 'headless_browser'), stealth_mode=stealth_mode),
+        create_driver(headless=config.getboolean('BROWSER', 'headless_browser'), stealth_mode=stealth_mode, lang=languages[0]),
         anticaptcha_manual_install=stealth_mode
     )
     logger.info("Browser initialized")
@@ -212,7 +212,6 @@ async def process_query(request: QueryRequest):
         query_resp.success = str(interaction.last_success)
         query_resp.blocks = blocks_json
         
-        # Store the raw dictionary representation
         query_resp_dict = {
             "done": query_resp.done,
             "answer": query_resp.answer,

cli.py

@@ -29,7 +29,7 @@ async def main():
                         is_local=config.getboolean('MAIN', 'is_local'))
 
     browser = Browser(
-        create_driver(headless=config.getboolean('BROWSER', 'headless_browser'), stealth_mode=stealth_mode),
+        create_driver(headless=config.getboolean('BROWSER', 'headless_browser'), stealth_mode=stealth_mode, lang=languages[0]),
         anticaptcha_manual_install=stealth_mode
     )
 

frontend/agentic-seek-front/package.json

@@ -10,6 +10,7 @@
     "axios": "^1.8.4",
     "react": "^19.1.0",
     "react-dom": "^19.1.0",
+    "react-markdown": "^10.1.0",
     "react-scripts": "5.0.1",
     "web-vitals": "^2.1.4"
   },

frontend/agentic-seek-front/src/App.css

@@ -366,6 +366,10 @@ body {
   color: #28a745; /* success */
 }
 
+.block-failure {
+  color: #d21b0b; /* success */
+}
+
 .block pre {
   background-color: #1a202c; /* Darker than darkCard */
   padding: 12px;

frontend/agentic-seek-front/src/App.js

@@ -1,4 +1,5 @@
 import React, { useState, useEffect, useRef } from 'react';
+import ReactMarkdown from 'react-markdown';
 import axios from 'axios';
 import './App.css';
 import { colors } from './colors';
@@ -193,7 +194,7 @@ function App() {
                                         {msg.type === 'agent' && (
                                             <span className="agent-name">{msg.agentName}</span>
                                         )}
-                                        <p>{msg.content}</p>
+                                        <ReactMarkdown>{msg.content}</ReactMarkdown>
                                     </div>
                                 ))
                             )}
@@ -241,9 +242,11 @@ function App() {
                                                 <p className="block-tool">Tool: {block.tool_type}</p>
                                                 <pre>{block.block}</pre>
                                                 <p className="block-feedback">Feedback: {block.feedback}</p>
-                                                <p className="block-success">
-                                                    Success: {block.success ? 'Yes' : 'No'}
-                                                </p>
+                                                {block.success ? (
+                                                    <p className="block-success">Success</p>
+                                                ) : (
+                                                    <p className="block-failure">Failure</p>
+                                                )}
                                             </div>
                                         ))
                                     ) : (

scripts/linux_install.sh

@@ -3,10 +3,22 @@
 echo "Starting installation for Linux..."
 
 set -e
+
+if ! command -v python3.10 &> /dev/null; then
+    echo "Error: Python 3.10 is not installed. Please install Python 3.10 and try again."
+    echo "You can install it using: sudo apt-get install python3.10 python3.10-dev python3.10-venv"
+    exit 1
+fi
+
+# Check if pip3.10 is available
+if ! python3.10 -m pip --version &> /dev/null; then
+    echo "Error: pip for Python 3.10 is not installed. Installing python3.10-pip..."
+    sudo apt-get install -y python3.10-pip || { echo "Failed to install python3.10-pip"; exit 1; }
+fi
+
 # Update package list
 sudo apt-get update || { echo "Failed to update package list"; exit 1; }
 # make sure essential tool are installed
-# Install essential tools
 sudo apt-get install -y \
     python3-dev \
     python3-pip \
@@ -21,15 +33,15 @@ sudo apt-get install -y \
     libnss3 \
     libxss1 || { echo "Failed to install packages"; exit 1; }
 
-# upgrade pip
-pip install --upgrade pip
-# install wheel
-pip install --upgrade pip setuptools wheel
+# Upgrade pip for Python 3.10
+python3.10 -m pip install --upgrade pip || { echo "Failed to upgrade pip"; exit 1; }
+# Install and upgrade setuptools and wheel
+python3.10 -m pip install --upgrade setuptools wheel || { echo "Failed to install setuptools and wheel"; exit 1; }
+# Install Selenium for chromedriver
+python3.10 -m pip install selenium || { echo "Failed to install selenium"; exit 1; }
+# Install Python dependencies from requirements.txt
+python3.10 -m pip install -r requirements.txt --no-cache-dir || { echo "Failed to install requirements.txt"; exit 1; }
 # install docker compose
 sudo apt install -y docker-compose
-# Install Selenium for chromedriver
-pip3 install selenium
-# Install Python dependencies from requirements.txt
-pip3 install -r requirements.txt --no-cache-dir
 
 echo "Installation complete for Linux!"

scripts/macos_install.sh

@@ -4,6 +4,18 @@ echo "Starting installation for macOS..."
 
 set -e
 
+if ! command -v python3.10 &> /dev/null; then
+    echo "Error: Python 3.10 is not installed. Please install Python 3.10 and try again."
+    echo "You can install it using: sudo apt-get install python3.10 python3.10-dev python3.10-venv"
+    exit 1
+fi
+
+# Check if pip3.10 is available
+if ! python3.10 -m pip --version &> /dev/null; then
+    echo "Error: pip for Python 3.10 is not installed. Installing python3.10-pip..."
+    sudo apt-get install -y python3.10-pip || { echo "Failed to install python3.10-pip"; exit 1; }
+fi
+
 # Check if homebrew is installed
 if ! command -v brew &> /dev/null; then
     echo "Homebrew not found. Installing Homebrew..."
@@ -18,13 +30,14 @@ brew install wget
 brew install --cask chromedriver
 # Install portaudio for pyAudio using Homebrew
 brew install portaudio
-# update pip
-python3 -m pip install --upgrade pip
-# upgrade setuptools and wheel
-pip3 install --upgrade setuptools wheel
-# Install Selenium
-pip3 install selenium
+
+# Upgrade pip for Python 3.10
+python3.10 -m pip install --upgrade pip || { echo "Failed to upgrade pip"; exit 1; }
+# Install and upgrade setuptools and wheel
+python3.10 -m pip install --upgrade setuptools wheel || { echo "Failed to install setuptools and wheel"; exit 1; }
+# Install Selenium for chromedriver
+python3.10 -m pip install selenium || { echo "Failed to install selenium"; exit 1; }
 # Install Python dependencies from requirements.txt
-pip3 install -r requirements.txt --no-cache-dir
+python3.10 -m pip install -r requirements.txt --no-cache-dir || { echo "Failed to install requirements.txt"; exit 1; }
 
 echo "Installation complete for macOS!"

sources/browser.py

@@ -45,7 +45,7 @@ def get_chrome_path() -> str:
         paths = ["/usr/bin/google-chrome", "/usr/bin/chromium-browser", "/usr/bin/chromium", "/opt/chrome/chrome", "/usr/local/bin/chrome"]
 
     for path in paths:
-        if os.path.exists(path) and os.access(path, os.X_OK):  # Check if executable
+        if os.path.exists(path) and os.access(path, os.X_OK):
             return path
     print("Looking for Google Chrome in these locations failed:")
     print('\n'.join(paths))
@@ -62,9 +62,9 @@ def get_chrome_path() -> str:
 def get_random_user_agent() -> str:
     """Get a random user agent string with associated vendor."""
     user_agents = [
-        {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36", "vendor": "Google Inc."},
-        {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", "vendor": "Apple Inc."},
-        {"ua": "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0", "vendor": ""},
+        {"ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "vendor": "Google Inc."},
+        {"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "vendor": "Apple Inc."},
+        {"ua": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36", "vendor": "Google Inc."},
     ]
     return random.choice(user_agents)
 
@@ -91,7 +91,7 @@ def bypass_ssl() -> str:
     """
     This is a fallback for stealth mode to bypass SSL verification. Which can fail on some setup.
     """
-    pretty_print("This is a workaround for SSL issues but upsafe we strongly advice you update your certifi SSL certificate.", color="warning")
+    pretty_print("Bypassing SSL verification issues, we strongly advice you update your certifi SSL certificate.", color="warning")
     ssl._create_default_https_context = ssl._create_unverified_context
 
 def create_undetected_chromedriver(service, chrome_options) -> webdriver.Chrome:
@@ -107,11 +107,10 @@ def create_undetected_chromedriver(service, chrome_options) -> webdriver.Chrome:
             pretty_print(f"Failed to create Chrome driver, fallback failed:\n{str(e)}.", color="failure")
             raise e
         raise e
-    # hide webdriver flag
     driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") 
     return driver
 
-def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx") -> webdriver.Chrome:
+def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx", lang="en") -> webdriver.Chrome:
     """Create a Chrome WebDriver with specified options."""
     chrome_options = Options()
     chrome_path = get_chrome_path()
@@ -126,19 +125,21 @@ def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx
         chrome_options.add_argument("--disable-webgl")
     user_data_dir = tempfile.mkdtemp()
     user_agent = get_random_user_agent()
+    width, height = (1920, 1080)
     chrome_options.add_argument(f"--user-data-dir={user_data_dir}")
+    chrome_options.add_argument(f"--accept-lang={lang}-{lang.upper()},{lang};q=0.9")
+    chrome_options.add_argument("--timezone=Europe/Paris")
     chrome_options.add_argument("--no-sandbox")
     chrome_options.add_argument("--disable-dev-shm-usage")
     chrome_options.add_argument("--mute-audio")
     chrome_options.add_argument("--disable-notifications")
     chrome_options.add_argument("--autoplay-policy=user-gesture-required")
+    chrome_options.add_argument("--disable-features=SitePerProcess,IsolateOrigins")
+    chrome_options.add_argument("--enable-features=NetworkService,NetworkServiceInProcess")
     chrome_options.add_argument("--disable-blink-features=AutomationControlled")
     chrome_options.add_argument(f'user-agent={user_agent["ua"]}')
-    resolutions = [(1920, 1080), (1366, 768), (1440, 900)]
-    width, height = random.choice(resolutions)
     chrome_options.add_argument(f'--window-size={width},{height}')
     if not stealth_mode:
-        # crx file can't be installed in stealth mode
         if not os.path.exists(crx_path):
             pretty_print(f"Anti-captcha CRX not found at {crx_path}.", color="failure")
         else:
@@ -154,16 +155,31 @@ def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx
         stealth(driver,
             languages=["en-US", "en"],
             vendor=user_agent["vendor"],
-            platform="Win64" if "Windows" in user_agent["ua"] else "MacIntel" if "Macintosh" in user_agent["ua"] else "Linux x86_64",
+            platform="Win64" if "windows" in user_agent["ua"].lower() else "MacIntel" if "mac" in user_agent["ua"].lower() else "Linux x86_64",
             webgl_vendor="Intel Inc.",
             renderer="Intel Iris OpenGL Engine",
             fix_hairline=True,
         )
         return driver
     security_prefs = {
-        "profile.default_content_setting_values.media_stream": 2,
-        "profile.default_content_setting_values.geolocation": 2,
+        "profile.default_content_setting_values.geolocation": 0,
+        "profile.default_content_setting_values.notifications": 0,
+        "profile.default_content_setting_values.camera": 0,
+        "profile.default_content_setting_values.microphone": 0,
+        "profile.default_content_setting_values.midi_sysex": 0,
+        "profile.default_content_setting_values.clipboard": 0,
+        "profile.default_content_setting_values.media_stream": 0,
+        "profile.default_content_setting_values.background_sync": 0,
+        "profile.default_content_setting_values.sensors": 0,
+        "profile.default_content_setting_values.accessibility_events": 0,
         "safebrowsing.enabled": True,
+        "credentials_enable_service": False,
+        "profile.password_manager_enabled": False,
+        "webkit.webprefs.accelerated_2d_canvas_enabled": True,
+        "webkit.webprefs.force_dark_mode_enabled": False,
+        "webkit.webprefs.accelerated_2d_canvas_msaa_sample_count": 4,
+        "enable_webgl": True,
+        "enable_webgl2_compute_context": True
     }
     chrome_options.add_experimental_option("prefs", security_prefs)
     chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
@@ -184,6 +200,7 @@ class Browser:
         except Exception as e:
             raise Exception(f"Failed to initialize browser: {str(e)}")
         self.setup_tabs()
+        self.patch_browser_fingerprint()
         if anticaptcha_manual_install:
             self.load_anticatpcha_manually()
     
@@ -200,12 +217,34 @@ class Browser:
         pretty_print("You might want to install the AntiCaptcha extension for captchas.", color="warning")
         self.driver.get(self.anticaptcha)
 
+    def human_move(element):
+        actions = ActionChains(driver)
+        x_offset = random.randint(-5,5)
+        for _ in range(random.randint(2,5)):
+            actions.move_by_offset(x_offset, random.randint(-2,2))
+            actions.pause(random.uniform(0.1,0.3))
+        actions.click().perform()
+
+    def human_scroll(self):
+        for _ in range(random.randint(1, 3)):
+            scroll_pixels = random.randint(150, 1200)
+            self.driver.execute_script(f"window.scrollBy(0, {scroll_pixels});")
+            time.sleep(random.uniform(0.5, 2.0))
+            if random.random() < 0.4:
+                self.driver.execute_script(f"window.scrollBy(0, -{random.randint(50, 300)});")
+                time.sleep(random.uniform(0.3, 1.0))
+
+    def patch_browser_fingerprint(self) -> None:
+        script = self.load_js("spoofing.js")
+        self.driver.execute_script(script)
+    
     def go_to(self, url:str) -> bool:
         """Navigate to a specified URL."""
-        time.sleep(random.uniform(0.4, 2.5)) # more human behavior
+        time.sleep(random.uniform(0.4, 2.5))
         try:
             initial_handles = self.driver.window_handles
             self.driver.get(url)
+            time.sleep(random.uniform(0.01, 0.3))
             try:
                 wait = WebDriverWait(self.driver, timeout=10)
                 wait.until(
@@ -217,6 +256,8 @@ class Browser:
             except TimeoutException:
                 self.logger.warning("Timeout while waiting for page to bypass 'checking your browser'")
             self.apply_web_safety()
+            time.sleep(random.uniform(0.01, 0.2))
+            self.human_scroll()
             self.logger.log(f"Navigated to: {url}")
             return True
         except TimeoutException as e:
@@ -622,7 +663,7 @@ class Browser:
         try:
             original_zoom = self.driver.execute_script("return document.body.style.zoom || 1;")
             self.driver.execute_script("document.body.style.zoom='75%'")
-            time.sleep(0.1)  # Allow time for the zoom to take effect
+            time.sleep(0.1)
             path = os.path.join(self.screenshot_folder, filename)
             if not os.path.exists(self.screenshot_folder):
                 os.makedirs(self.screenshot_folder)
@@ -645,11 +686,12 @@ class Browser:
 
 if __name__ == "__main__":
     driver = create_driver(headless=False, stealth_mode=True)
-    browser = Browser(driver, anticaptcha_manual_install=True)
+    browser = Browser(driver, anticaptcha_manual_install=False)
     
     input("press enter to continue")
     print("AntiCaptcha / Form Test")
-    #browser.go_to("https://www.browserscan.net/bot-detection")
+    browser.go_to("https://bot.sannysoft.com")
+    time.sleep(5)
     #txt = browser.get_text()
     #browser.go_to("https://www.google.com/recaptcha/api2/demo")
     browser.go_to("https://home.openweathermap.org/users/sign_up")
@@ -658,3 +700,11 @@ if __name__ == "__main__":
     #inputs_fill = ['[q](checked)', '[q](checked)', '[user[username]](mlg)', '[user[email]](mlg.fcu@gmail.com)', '[user[password]](placeholder_P@ssw0rd123)', '[user[password_confirmation]](placeholder_P@ssw0rd123)']
     #browser.fill_form(inputs_fill)
     input("press enter to exit")
+
+# Test sites for browser fingerprinting and captcha
+# https://nowsecure.nl/
+# https://bot.sannysoft.com
+# https://browserleaks.com/
+# https://bot.incolumitas.com/
+# https://fingerprintjs.github.io/fingerprintjs/
+# https://antoinevastel.com/bots/

sources/web_scripts/inject_safety_script.js

@@ -21,7 +21,5 @@ window.fetch = function() {
     console.log('Blocked fetch request');
     return Promise.reject('Blocked');
 };
-// Block annoying dialogs
-window.alert = function() {};
-window.confirm = function() { return false; };
+
 window.prompt = function() { return null; };

sources/web_scripts/spoofing.js

@@ -0,0 +1,126 @@
+
+// Core automation masking 
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array; 
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise; 
+delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol; 
+
+window.RTCPeerConnection = undefined;
+window.webkitRTCPeerConnection = undefined;
+window.mozRTCPeerConnection = undefined;
+
+window.Notification = class Notification {
+    constructor(title, options = {}) {
+        this.title = title;
+        this.options = options;
+    }
+    static permission = 'granted';
+    static requestPermission = () => Promise.resolve('granted');
+    close() {}
+    onclick = null;
+    onerror = null;
+    onclose = null;
+    onshow = null;
+};
+
+Object.keys(window).forEach((key) => {
+  if (key.includes("webdriver") || key.includes("selenium") || key.includes("driver")) {
+    delete window[key];
+  }
+});
+
+// Randomize plugins
+
+const pluginsList = [
+    {type: 'application/x-google-chrome-pdf', description: 'Portable Document Format', filename: 'internal-pdf-viewer', name: 'Chrome PDF Plugin'},
+    {type: 'application/x-nacl', description: 'Native Client Executable', filename: 'internal-nacl-plugin', name: 'Native Client'},
+    {type: 'application/x-ppapi-widevine-cdm', description: 'Widevine Content Decryption Module', filename: 'widevinecdm', name: 'Widevine CDM'}
+];
+Object.defineProperty(navigator, 'plugins', {
+    get: () => pluginsList.slice(0, Math.floor(Math.random() * pluginsList.length) + 1)
+});
+
+// Font spoofing 
+
+const fontList = ['Arial', 'Helvetica', 'Times New Roman', 'Courier New', 'Verdana'];
+Object.defineProperty(document, 'fonts', {
+    value: {
+        add: function() {},
+        check: function(font) { return fontList.includes(font.split(' ').slice(-1)[0]); },
+        delete: function() {},
+        forEach: function(cb) { fontList.forEach(f => cb(f)); },
+        has: function(font) { return fontList.includes(font.split(' ').slice(-1)[0]); },
+        keys: function() { return fontList; },
+        size: fontList.length
+    }
+});
+
+// Canvas fingerprint spoofing
+
+HTMLCanvasElement.prototype.toDataURL = function() {
+    const ctx = this.getContext('2d');
+    // Add varied noise to avoid consistent fingerprints
+    for (let i = 0; i < 10; i++) {
+        ctx.fillStyle = `rgba(${Math.random() * 5}, ${Math.random() * 5}, ${Math.random() * 5}, 0.005)`;
+        ctx.fillRect(Math.random() * this.width, Math.random() * this.height, 1, 1);
+    }
+    return originalToDataURL.apply(this, arguments);
+};
+
+const [w, h] = [1920, 1080];
+Object.defineProperty(window, 'screen', {
+  value: {
+    width: w,
+    height: h,
+    availWidth: w - 20,
+    availHeight: h - 100,
+    colorDepth: 24,
+    pixelDepth: 24
+  }
+});
+
+
+// ===== WebGL Consistency =====
+const os = navigator.userAgent.includes('Windows') ? 'Windows' : 'Mac';
+const webGLParams = {
+  'Windows': {
+    37445: 'Google Inc. (NVIDIA)', // VENDOR
+    37446: 'ANGLE (NVIDIA, NVIDIA GeForce RTX 3060)', // RENDERER
+    36349: 'NVIDIA Corporation', // UNMASKED_VENDOR_WEBGL
+    37444: 'NVIDIA GeForce RTX 3060', // UNMASKED_RENDERER_WEBGL
+    35661: 'WebGL 2.0' // VERSION
+  },
+  'Mac': {
+    37445: 'Apple Inc.', 
+    37446: 'Apple M1 Pro',
+    36349: 'Apple',
+    37444: 'Apple M1 Pro',
+    35661: 'WebGL 2.0 (Metal)'
+  }
+};
+
+// replace WebGL parameters
+WebGLRenderingContext.prototype.getParameter = function(parameter) {
+    return webGLParams[os][parameter] || getParameter.call(this, parameter);
+  };
+
+// Performance API spoofing
+if ('performance' in window) {
+    Object.defineProperty(performance, 'memory', {
+      value: {
+        jsHeapSizeLimit: 4294705152,
+        totalJSHeapSize: 78365432,
+        usedJSHeapSize: 46543210
+      },
+      configurable: true
+    });
+  }
+
+const originalCreate = window.AudioContext || window.webkitAudioContext;
+window.AudioContext = window.webkitAudioContext = function() {
+  const context = new originalCreate();
+  const analyser = context.createAnalyser();
+  analyser.fake = true; // Mark as spoofed
+  // Spoof common methods
+  analyser.getFloatFrequencyData = () => new Float32Array(1024).fill(Math.random() * -100);
+  return context;
+};