@samitouri / QOSami_Agents_Zero-2 / commits / 33e861c5

The Merge - builds

base image separation

frdel committed May 27, 2025 at 21:55 UTC 33e861c5a66f3cb520cc392e1c63e4acbec9c1c7
17 files changed +210 -78
docker/base/Dockerfile
+6 -3
@@ -18,12 +18,15 @@ ENV TZ=UTC
18 # Copy contents of the project to /
19 COPY ./fs/ /
20
21 -# install python
22 -RUN bash /ins/install_python.sh
23 -
21 # install packages software
22 RUN bash /ins/install_base_packages.sh
23
24 +# install python after packages to ensure version overriding
25 +RUN bash /ins/install_python.sh
26 +
27 +# install searxng
28 +RUN bash /ins/install_searxng.sh
29 +
30 # configure ssh
31 RUN bash /ins/configure_ssh.sh
32
docker/base/build.txt
+3
@@ -13,3 +13,6 @@ docker buildx build -t frdel/agent-zero-base:latest --platform linux/amd64,linux
13
14 # without cache
15 docker buildx build -t frdel/agent-zero-base:latest --platform linux/amd64,linux/arm64 --push --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) --no-cache .
16 +
17 +# plain output
18 +--progress=plain
\ No newline at end of file
docker/base/fs/etc/searxng/limiter.toml new
+33
@@ -0,0 +1,33 @@
1 +[real_ip]
2 +# Number of values to trust for X-Forwarded-For.
3 +x_for = 1
4 +
5 +# The prefix defines the number of leading bits in an address that are compared
6 +# to determine whether or not an address is part of a (client) network.
7 +ipv4_prefix = 32
8 +ipv6_prefix = 48
9 +
10 +[botdetection.ip_limit]
11 +# To get unlimited access in a local network, by default link-local addresses
12 +# (networks) are not monitored by the ip_limit
13 +filter_link_local = false
14 +
15 +# Activate link_token method in the ip_limit method
16 +link_token = false
17 +
18 +[botdetection.ip_lists]
19 +# In the limiter, the ip_lists method has priority over all other methods.
20 +# If an IP is in the pass_ip list, it has unrestricted access and is not
21 +# checked if, for example, the "user agent" suggests a bot (e.g., curl).
22 +block_ip = [
23 + # '93.184.216.34', # Example IPv4 address
24 + # '257.1.1.1', # Invalid IP --> will be ignored, logged in ERROR class
25 +]
26 +pass_ip = [
27 + # '192.168.0.0/16', # IPv4 private network
28 + # 'fe80::/10', # IPv6 link-local; overrides botdetection.ip_limit.filter_link_local
29 +]
30 +
31 +# Activate passlist of (hardcoded) IPs from the SearXNG organization,
32 +# e.g., `check.searx.space`.
33 +pass_searxng_org = true
docker/base/fs/etc/searxng/settings.yml new
+78
@@ -0,0 +1,78 @@
1 +# SearXNG settings
2 +
3 +use_default_settings: true
4 +
5 +general:
6 + debug: false
7 + instance_name: "SearXNG"
8 +
9 +search:
10 + safe_search: 0
11 + # autocomplete: 'duckduckgo'
12 + formats:
13 + - json
14 + # - html
15 +
16 +server:
17 + # Is overwritten by ${SEARXNG_SECRET}
18 + secret_key: "dummy"
19 + port: 55510
20 + limiter: false
21 + image_proxy: false
22 + # public URL of the instance, to ensure correct inbound links. Is overwritten
23 + # by ${SEARXNG_URL}.
24 + # base_url: http://example.com/location
25 +
26 +# redis:
27 +# # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.
28 +# url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
29 +
30 +ui:
31 + static_use_hash: true
32 +
33 +# preferences:
34 +# lock:
35 +# - autocomplete
36 +# - method
37 +
38 +enabled_plugins:
39 + - 'Hash plugin'
40 + - 'Self Informations'
41 + - 'Tracker URL remover'
42 + - 'Ahmia blacklist'
43 + # - 'Hostnames plugin' # see 'hostnames' configuration below
44 + # - 'Open Access DOI rewrite'
45 +
46 +# plugins:
47 +# - only_show_green_results
48 +
49 +# hostnames:
50 +# replace:
51 +# '(.*\.)?youtube\.com$': 'invidious.example.com'
52 +# '(.*\.)?youtu\.be$': 'invidious.example.com'
53 +# remove:
54 +# - '(.*\.)?facebook.com$'
55 +# low_priority:
56 +# - '(.*\.)?google\.com$'
57 +# high_priority:
58 +# - '(.*\.)?wikipedia.org$'
59 +
60 +engines:
61 +
62 +# - name: fdroid
63 +# disabled: false
64 +#
65 +# - name: apk mirror
66 +# disabled: false
67 +#
68 +# - name: mediathekviewweb
69 +# categories: TV
70 +# disabled: false
71 +#
72 +# - name: invidious
73 +# disabled: false
74 +# base_url:
75 +# - https://invidious.snopyta.org
76 +# - https://invidious.tiekoetter.com
77 +# - https://invidio.xamh.de
78 +# - https://inv.riverside.rocks
\ No newline at end of file
docker/base/fs/ins/configure_venv.sh deleted
-9
@@ -1,9 +0,0 @@
1 -#!/bin/bash
2 -
3 -if [ ! -d /opt/venv ]; then
4 - # Create and activate Python virtual environment
5 - python3 -m venv /opt/venv
6 - source /opt/venv/bin/activate
7 -else
8 - source /opt/venv/bin/activate
9 -fi
\ No newline at end of file
docker/base/fs/ins/install_base_packages.sh
+3 -13
@@ -1,19 +1,9 @@
1 #!/bin/bash
2 set -e
3
4 -echo "=====BASE PACKAGES====="
4 +echo "====================BASE PACKAGES START===================="
5
6 -# Hold python3 packages to prevent overrides
7 -apt-mark hold python3
8 -
9 -# Install with --no-install-recommends to minimize unwanted dependencies
6 apt-get install -y --no-install-recommends \
11 - nodejs openssh-server sudo curl wget git ffmpeg supervisor cron
12 -
13 -
14 -echo "=====AFTER UPDATE====="
15 -
7 + nodejs npm openssh-server sudo curl wget git ffmpeg supervisor cron
8
17 -echo "=====PYTHON VERSION: $(python3 --version) ====="
18 -echo "=====PYTHON OTHERS: $(ls /usr/bin/python*) ====="
19 -sleep 10
\ No newline at end of file
9 +echo "====================BASE PACKAGES END===================="
docker/base/fs/ins/install_python.sh
+36 -28
@@ -1,44 +1,52 @@
1 #!/bin/bash
2 set -e
3
4 -# echo "=====PYTHON 3.12 & SID REPO====="
4 +echo "====================PYTHON START===================="
5
6 -# apt clean
6 +echo "====================PYTHON 3.12 & SID REPO===================="
7
8 -# # ★ 1. Add sid repo & pin it for python 3.12
9 -# echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list.d/debian-sid.list
10 -# cat >/etc/apt/preferences.d/python312 <<'EOF'
11 -# Package: *
12 -# Pin: release a=sid
13 -# Pin-Priority: 100
8 +apt clean
9
15 -# Package: python3.12*
16 -# Pin: release a=sid
17 -# Pin-Priority: 990
10 +# ★ 1. Add sid repo & pin it for python 3.12
11 +echo "deb http://deb.debian.org/debian sid main" > /etc/apt/sources.list.d/debian-sid.list
12 +cat >/etc/apt/preferences.d/python312 <<'EOF'
13 +Package: *
14 +Pin: release a=sid
15 +Pin-Priority: 100
16
19 -# # Prevent Python 3.13 from being installed
20 -# Package: python3.13*
21 -# Pin: release *
22 -# Pin-Priority: -1
23 -# EOF
17 +Package: python3.12*
18 +Pin: release a=sid
19 +Pin-Priority: 990
20
25 -# apt-get update && apt-get -y upgrade
21 +# Prevent Python 3.13 from being installed
22 +Package: python3.13*
23 +Pin: release *
24 +Pin-Priority: -1
25 +EOF
26
27 -# apt-get install -y --no-install-recommends \
28 -# python3.12 python3.12-venv python3.12-dev
27 +apt-get update && apt-get -y upgrade
28 +
29 +apt-get install -y --no-install-recommends \
30 + python3.12 python3.12-venv python3.12-dev
31
30 -# # ★ 3. Switch the interpreter
31 -# # update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 0
32 +# ★ 3. Switch the interpreter
33 +# update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 0
34 # update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
35 # update-alternatives --set python3 /usr/bin/python3.12
36
35 -# echo "=====PYTHON VERSION: $(python3 --version) ====="
36 -# echo "=====PYTHON OTHERS: $(ls /usr/bin/python*) ====="
37 -# sleep 10
37 +echo "====================PYTHON VERSION: $(python3 --version) ===================="
38 +echo "====================PYTHON OTHERS: $(ls /usr/bin/python*) "
39
39 -apt-get update && apt-get -y upgrade
40 +echo "====================PYTHON VENV===================="
41 +
42 +# create and activate default venv
43 +python3.12 -m venv /opt/venv
44 +source /opt/venv/bin/activate
45 +
46 +# upgrade pip and install static packages
47 +pip install --upgrade pip ipython requests
48 +# Install some packages in specific variants
49 +pip install torch --index-url https://download.pytorch.org/whl/cpu
50
41 -apt-get install -y --no-install-recommends \
42 - python3 python3-venv python3-dev
51
44 -. /ins/configure_venv.sh
\ No newline at end of file
52 +echo "====================PYTHON END===================="
docker/base/fs/ins/install_searxng.sh renamed
+9 -3
@@ -1,10 +1,13 @@
1 #!/bin/bash
2
3 +echo "====================SEARXNG1 START===================="
4 +
5 # Install necessary packages
6 apt-get install -y \
5 - python3-dev python3-babel python3-venv \
6 - uwsgi uwsgi-plugin-python3 \
7 + python3.12-dev python3.12-venv \
8 git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
9 +# python3.12-babel uwsgi uwsgi-plugin-python3
10 +
11
12 # Add the searxng system user
13 useradd --shell /bin/bash --system \
@@ -19,5 +22,8 @@ usermod -aG sudo searxng
22 mkdir "/usr/local/searxng"
23 chown -R "searxng:searxng" "/usr/local/searxng"
24
25 +echo "====================SEARXNG1 END===================="
26 +
27 # Start a new shell as the searxng user and run the installation script
23 -su - searxng -c "bash /ins/install_searxng2.sh"
\ No newline at end of file
28 +su - searxng -c "bash /ins/install_searxng2.sh"
29 +
docker/base/fs/ins/install_searxng2.sh renamed
+11 -2
@@ -1,11 +1,16 @@
1 #!/bin/bash
2
3 +echo "====================SEARXNG2 START===================="
4 +
5 +
6 # clone SearXNG repo
7 git clone "https://github.com/searxng/searxng" \
8 "/usr/local/searxng/searxng-src"
9
10 +echo "====================SEARXNG2 VENV===================="
11 +
12 # create virtualenv:
8 -python3 -m venv "/usr/local/searxng/searx-pyenv"
13 +python3.12 -m venv "/usr/local/searxng/searx-pyenv"
14
15 # make it default
16 echo ". /usr/local/searxng/searx-pyenv/bin/activate" \
@@ -14,6 +19,8 @@ echo ". /usr/local/searxng/searx-pyenv/bin/activate" \
19 # activate venv
20 source "/usr/local/searxng/searx-pyenv/bin/activate"
21
22 +echo "====================SEARXNG2 INST===================="
23 +
24 # update pip's boilerplate
25 pip install -U pip
26 pip install -U setuptools
@@ -25,4 +32,6 @@ cd "/usr/local/searxng/searxng-src"
32 pip install --use-pep517 --no-build-isolation -e .
33
34 # cleanup cache
28 -pip cache purge
\ No newline at end of file
35 +pip cache purge
36 +
37 +echo "====================SEARXNG2 END===================="
\ No newline at end of file
docker/run/Dockerfile
+4 -4
@@ -1,6 +1,6 @@
1 -# Use the latest slim version of Debian
2 -FROM agent-zero-base:local
3 -# FROM frdel/agent-zero-base:latest
1 +# Use the pre-built base image for A0
2 +# FROM agent-zero-base:local
3 +FROM frdel/agent-zero-base:latest
4
5 # Check if the argument is provided, else throw an error
6 ARG BRANCH
@@ -27,7 +27,7 @@ RUN echo "cache buster $CACHE_DATE" && bash /ins/install_A02.sh $BRANCH
27 RUN bash /ins/post_install.sh $BRANCH
28
29 # Expose ports
30 -EXPOSE 22 80 3000-9999
30 +EXPOSE 22 80 9000-9009
31
32 RUN chmod +x /exe/initialize.sh /exe/run_A0.sh /exe/run_searxng.sh /exe/run_tunnel_api.sh
33
docker/run/build.txt
+4
@@ -19,3 +19,7 @@ docker buildx build --build-arg BRANCH=testing -t frdel/agent-zero-run:testing -
19
20 # main
21 docker buildx build --build-arg BRANCH=main -t frdel/agent-zero-run:vx.x.x -t frdel/agent-zero-run:latest --platform linux/amd64,linux/arm64 --push --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) .
22 +
23 +
24 +# plain output
25 +--progress=plain
\ No newline at end of file
docker/run/fs/exe/initialize.sh
+4 -1
@@ -1,5 +1,7 @@
1 #!/bin/bash
2
3 +echo "Running initialization script..."
4 +
5 # branch from parameter
6 if [ -z "$1" ]; then
7 echo "Error: Branch parameter is empty. Please provide a valid branch name."
@@ -17,4 +19,5 @@ chmod 444 /root/.profile
19 # update package list to save time later
20 apt-get update > /dev/null 2>&1 &
21
20 -exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
22 +# Redirect only stdout to /dev/null to suppress normal messages but keep errors visible
23 +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf > /dev/null
docker/run/fs/ins/install_A0.sh
+6 -5
@@ -1,4 +1,5 @@
1 #!/bin/bash
2 +set -e
3
4 # branch from parameter
5 if [ -z "$1" ]; then
@@ -13,11 +14,11 @@ git clone -b "$BRANCH" "https://github.com/frdel/agent-zero" "/git/agent-zero"
14 # setup python environment
15 . "/ins/setup_venv.sh" "$@"
16
16 -# Ensure the virtual environment and pip setup
17 -pip install --upgrade pip ipython requests
18 -
19 -# Install some packages in specific variants
20 -pip install torch --index-url https://download.pytorch.org/whl/cpu
17 +# moved to base image
18 +# # Ensure the virtual environment and pip setup
19 +# pip install --upgrade pip ipython requests
20 +# # Install some packages in specific variants
21 +# pip install torch --index-url https://download.pytorch.org/whl/cpu
22
23 # Install remaining A0 python packages
24 pip install -r /git/agent-zero/requirements.txt
docker/run/fs/ins/install_A02.sh
+2
@@ -1,6 +1,8 @@
1 #!/bin/bash
2
3 # cachebuster script, this helps speed up docker builds
4 +
5 +# remove repo
6 rm -rf /git/agent-zero
7
8 # run the original install script again
docker/run/fs/ins/install_additional.sh
+2 -2
@@ -3,5 +3,5 @@
3 # install playwright - moved to install A0
4 # bash /ins/install_playwright.sh "$@"
5
6 -# searxng
7 -bash /ins/install_searxng.sh "$@"
\ No newline at end of file
6 +# searxng - moved to base image
7 +# bash /ins/install_searxng.sh "$@"
\ No newline at end of file
docker/run/fs/ins/install_playwright.sh
+2 -2
@@ -3,13 +3,13 @@
3 # activate venv
4 . "/ins/setup_venv.sh" "$@"
5
6 -# install playwright if not installed
6 +# install playwright if not installed (should be from requirements.txt)
7 pip install playwright
8
9 # install chromium with dependencies
10 # for kali-based
11 # if [ "$@" = "hacking" ]; then
12 - apt-get install -y fonts-unifont libnss3 libnspr4
12 + apt-get install -y fonts-unifont libnss3 libnspr4 libatk1.0-0 libatspi2.0-0 libxcomposite1 libxdamage1
13 playwright install chromium-headless-shell
14 # else
15 # # for debian based
docker/run/fs/ins/setup_venv.sh
+7 -6
@@ -1,9 +1,10 @@
1 #!/bin/bash
2
3 -if [ ! -d /opt/venv ]; then
4 - # Create and activate Python virtual environment
5 - python3 -m venv /opt/venv
3 +# this has to be ready from base image
4 +# if [ ! -d /opt/venv ]; then
5 +# # Create and activate Python virtual environment
6 +# python3.12 -m venv /opt/venv
7 +# source /opt/venv/bin/activate
8 +# else
9 source /opt/venv/bin/activate
7 -else
8 - source /opt/venv/bin/activate
9 -fi
\ No newline at end of file
10 +# fi
\ No newline at end of file