From 54cc2a03ec755f8a9c787bdbb26f3bfb7dcfc731 Mon Sep 17 00:00:00 2001 From: martin legrand Date: Sat, 31 May 2025 17:20:40 +0200 Subject: [PATCH] feat : updating with latest docker backend build thx to #265 --- .env.example | 8 ++- Dockerfile.backend | 143 +++++++++++++++++++++++++---------------- api.py | 2 +- docker-compose.yml | 13 +++- sources/browser.py | 2 +- sources/tools/tools.py | 2 +- 6 files changed, 104 insertions(+), 66 deletions(-) diff --git a/.env.example b/.env.example index 3b38eb2..a476049 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,9 @@ SEARXNG_BASE_URL="http://127.0.0.1:8080" -TOKENIZERS_PARALLELISM="false" +REDIS_BASE_URL="redis://redis:6379/0" +WORK_DIR_HOST="/Users/mlg/Documents/workspace_for_ai" OPENAI_API_KEY='xxxxx' DEEPSEEK_API_KEY='xxxxx' OPENROUTER_API_KEY='xxxxx' -BACKEND_PORT=8000 -WORK_DIR="/tmp/" \ No newline at end of file +TOGETHER_API_KEY='xxxx' +GOOGLE_API_KEY='xxxxx' +ANTHROPIC_API_KEY='xxxxx' \ No newline at end of file diff --git a/Dockerfile.backend b/Dockerfile.backend index 228618e..f5f51d2 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -1,67 +1,100 @@ -FROM --platform=linux/amd64 ubuntu:22.04 -ARG DEBIAN_FRONTEND=noninteractive + +FROM --platform=linux/amd64 python:3.11-slim +ENV DEBIAN_FRONTEND=noninteractive WORKDIR /app # Install essential packages and Chrome dependencies -RUN apt-get update && apt-get install -y \ +RUN apt-get update -y && apt-get install -y \ wget \ + gnupg2 \ + ca-certificates \ unzip \ - curl \ - gnupg \ - python3-dev \ - python3-pip \ - python3-wheel \ - build-essential \ + xvfb \ + libxss1 \ + libappindicator1 \ fonts-liberation \ - libasound-dev \ - libasound2 \ - portaudio19-dev \ + libnss3 \ + libatk1.0-0 \ libatk-bridge2.0-0 \ + libcups2 \ libdrm2 \ libxcomposite1 \ libxdamage1 \ - libfontconfig1 \ libxrandr2 \ - libgbm1 \ - libxss1 \ - libnss3 \ - libnspr4 \ - libxshmfence1 \ - libgconf-2-4 \ - libxfixes3 \ - libxinerama1 \ - libgtk-3-0 \ - libgdk-pixbuf2.0-0 \ - libatspi2.0-0 \ - libdrm2 \ - libxkbcommon0 \ - libepoxy0 \ - libgtk-3-0 \ - libharfbuzz0b \ - libegl1-mesa \ - libgles2-mesa \ - # Virtual display for headless operation - xvfb \ + xdg-utils \ + dbus \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install --upgrade pip setuptools wheel && \ -pip3 install selenium +RUN apt-get update -y && \ + apt-get install -y \ + gcc \ + g++ \ + gfortran \ + libportaudio2 \ + portaudio19-dev \ + ffmpeg \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + gnupg2 \ + wget \ + unzip \ + python3 \ + python3-pip \ + libasound2 \ + libatk-bridge2.0-0 \ + libgtk-4-1 \ + libnss3 \ + xdg-utils \ + wget \ + && rm -rf /var/lib/apt/lists/* + + +RUN apt-get update -y && \ +apt-get install -y \ + alsa-utils \ +&& rm -rf /var/lib/apt/lists/* + +ENV CHROME_TESTING_VERSION=134.0.6998.88 +ENV DISPLAY=:99 + +RUN set -eux; \ + wget -qO /tmp/chrome.zip \ + "https://storage.googleapis.com/chrome-for-testing-public/${CHROME_TESTING_VERSION}/linux64/chrome-linux64.zip"; \ + unzip -q /tmp/chrome.zip -d /opt; \ + rm /tmp/chrome.zip; \ + ln -s /opt/chrome-linux64/chrome /usr/local/bin/google-chrome; \ + ln -s /opt/chrome-linux64/chrome /usr/local/bin/chrome; \ + mkdir -p /opt/chrome; \ + ln -s /opt/chrome-linux64/chrome /opt/chrome/chrome; \ + google-chrome --version + +RUN set -eux; \ + wget -qO /tmp/chromedriver.zip \ + "https://storage.googleapis.com/chrome-for-testing-public/${CHROME_TESTING_VERSION}/linux64/chromedriver-linux64.zip"; \ + unzip -q /tmp/chromedriver.zip -d /tmp; \ + mv /tmp/chromedriver-linux64/chromedriver /usr/local/bin; \ + rm /tmp/chromedriver.zip; \ + chmod +x /usr/local/bin/chromedriver; \ + chromedriver --version + +RUN chmod +x /opt/chrome/chrome + +RUN pip3 install --upgrade pip setuptools wheel -# Install dependencies COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt +RUN mkdir -p /opt/workspace + # Copy application code COPY api.py . -# Chrome bundle approach, commented out opting for direct download -#COPY chrome_bundle/ ./chrome_bundle/ COPY sources/ ./sources/ COPY prompts/ ./prompts/ COPY crx/ crx/ COPY llm_router/ llm_router/ -RUN ls -COPY .env.example .env +COPY config.ini . # Install Chrome and ChromeDriver from chrome-for-testing #RUN wget -O chrome-headless-shell-linux64.zip https://storage.googleapis.com/chrome-for-testing-public/137.0.7151.55/linux64/chrome-headless-shell-linux64.zip && \ @@ -75,24 +108,20 @@ COPY .env.example .env # chmod +x /opt/google/chrome/chrome-headless-shell && \ # chmod +x /usr/local/bin/chromedriver -RUN wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - && \ - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list && \ - apt-get update && \ - apt-get install -y google-chrome-stable && \ - rm -rf /var/lib/apt/lists/* +#RUN wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - && \ +# echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list && \ +# apt-get update && \ +# apt-get install -y google-chrome-stable && \ +# rm -rf /var/lib/apt/lists/* +# +## Install matching ChromeDriver +#RUN CHROME_VERSION=$(google-chrome --version | grep -oP '\d+\.\d+\.\d+') && \ +# wget -O chromedriver.zip "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_VERSION%%.*}/chromedriver_linux64.zip" && \ +# unzip chromedriver.zip && \ +# mv chromedriver /usr/local/bin/ && \ +# chmod +x /usr/local/bin/chromedriver && \ +# rm chromedriver.zip -# Install matching ChromeDriver -RUN CHROME_VERSION=$(google-chrome --version | grep -oP '\d+\.\d+\.\d+') && \ - wget -O chromedriver.zip "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROME_VERSION%%.*}/chromedriver_linux64.zip" && \ - unzip chromedriver.zip && \ - mv chromedriver /usr/local/bin/ && \ - chmod +x /usr/local/bin/chromedriver && \ - rm chromedriver.zip - -ENV DISPLAY=:99 - -# Expose port EXPOSE 8000 - # Run the application CMD ["python3", "api.py"] \ No newline at end of file diff --git a/api.py b/api.py index 3689c04..9e70a4a 100755 --- a/api.py +++ b/api.py @@ -38,7 +38,7 @@ config.read('config.ini') api.add_middleware( CORSMiddleware, - allow_origins=["http://localhost", "http://localhost:3000"], + allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], diff --git a/docker-compose.yml b/docker-compose.yml index 3360e8e..55d98a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,11 +79,18 @@ services: - ${BACKEND_PORT:-8000}:${BACKEND_PORT:-8000} volumes: - ./:/app - - ${WORK_DIR}:${WORK_DIR} + - ${WORK_DIR_HOST:-.}:/opt/workspace command: python3 api.py environment: - - SEARXNG_URL=http://localhost:8080 - - WORK_DIR=${WORK_DIR} + - SEARXNG_URL=${SEARXNG_BASE_URL:-http://searxng:8080} + - REDIS_URL=${REDIS_BASE_URL:-redis://redis:6379/0} + - WORK_DIR_HOST=${WORK_DIR_HOST:-.} + - OPENAI_API_KEY=${OPENAI_API_KEY} + - DEEPSEEK_API_KEY=${DEEPSEEK_API_KEY} + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} + - TOGETHER_API_KEY=${TOGETHER_API_KEY} + - GOOGLE_API_KEY=${GOOGLE_API_KEY} + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} network_mode: "host" volumes: diff --git a/sources/browser.py b/sources/browser.py index ea7e088..4f05f0d 100644 --- a/sources/browser.py +++ b/sources/browser.py @@ -143,7 +143,7 @@ def create_driver(headless=False, stealth_mode=True, crx_path="./crx/nopecha.crx user_data_dir = tempfile.mkdtemp() user_agent = get_random_user_agent() width, height = (1920, 1080) - chrome_options.add_argument(f"--user-data-dir={user_data_dir}") + #chrome_options.add_argument(f"--user-data-dir={user_data_dir}") chrome_options.add_argument(f"--accept-lang={lang}-{lang.upper()},{lang};q=0.9") chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--disable-background-timer-throttling") diff --git a/sources/tools/tools.py b/sources/tools/tools.py index 555f30c..25b9fac 100644 --- a/sources/tools/tools.py +++ b/sources/tools/tools.py @@ -73,7 +73,7 @@ class Tools(): default_path = os.path.dirname(os.getcwd()) if self.config_exists(): self.config.read('./config.ini') - config_path = self.config['MAIN']['work_dir'] + config_path = self.config['MAIN']['work_dir'] or os.getenv('WORK_DIR') dir_path = default_path if not self.check_config_dir_validity() else config_path else: dir_path = default_path