Bundler updater
- /bundle dir - autobundler scripts for macos and win
frdel committed
Oct 11, 2024 at 14:02 UTC
823957673ecb23744422616b77ca329f0140e187
4 files changed
+218
-87
.gitignore
+3
-4
@@ -9,9 +9,8 @@
9
# Ignore all contents of the virtual environment directory
10
.venv/
11
12
-# Ignore bundled files
13
-build/
14
-dist/
12
+# ignore all folders under /bundle
13
+/bundle/*/
14
15
# Ignore all contents of the directory "work_dir"
16
work_dir/*
@@ -59,4 +58,4 @@ instruments/**/*.*
58
59
# Explicitly allow the default folder and its contents
60
!instruments/default/
62
-!instruments/default/**
\ No newline at end of file
61
+!instruments/default/**
bundle/bundle.py
renamed
+36
-83
@@ -8,83 +8,56 @@ import pathspec
8
import importlib
9
import importlib.metadata as metadata
10
11
-
11
def get_package_data_folder(package_name):
12
"""Return the package path if it contains data files."""
13
try:
15
- # Get the module and its file path
14
package = importlib.import_module(package_name)
15
package_path = os.path.dirname(package.__file__) # type: ignore
18
- # only if package path does not end with "site-packages"
16
if not package_path.endswith("site-packages"):
20
- # Check if the package contains any relevant data files
17
has_data = any(
18
file.endswith((".json", ".txt", ".csv", ".yml", ".yaml"))
19
for root, dirs, files in os.walk(package_path)
20
for file in files
21
)
26
-
22
if has_data:
23
return package_path
29
-
24
except ImportError:
31
- print(
32
- f"Warning: Unable to import {package_name}. Skipping data folder discovery for this package."
33
- )
34
-
25
+ print(f"Warning: Unable to import {package_name}. Skipping data folder discovery for this package.")
26
return None
27
37
-
28
def get_add_data_args():
29
"""Return an array of --add-data arguments for PyInstaller, one per package."""
30
add_data_args = []
41
-
42
- # Use importlib.metadata to get the installed packages
31
installed_packages = [dist.metadata["Name"] for dist in metadata.distributions()]
44
-
45
- # Discover the data folder for each package and add it as a --add-data argument
32
for package in installed_packages:
33
package_data_folder = get_package_data_folder(package)
34
if package_data_folder:
49
- # Add the whole package directory
50
- add_data_args.append(
51
- f"--add-data={package_data_folder}{os.pathsep}{package}"
52
- )
53
-
35
+ add_data_args.append(f"--add-data={package_data_folder}{os.pathsep}{package}")
36
return add_data_args
37
56
-
38
def get_site_packages_path():
39
"""Get the path to the site-packages directory of the current environment."""
40
if hasattr(site, "getsitepackages"):
41
paths = site.getsitepackages()
42
else:
43
paths = [site.getusersitepackages()]
63
-
44
if paths:
45
return paths[0]
46
else:
47
raise RuntimeError("Couldn't determine the site-packages path.")
48
69
-
49
def parse_gitignore(gitignore_path):
50
"""Parse .gitignore file and return a PathSpec object."""
51
if not os.path.exists(gitignore_path):
52
return pathspec.PathSpec.from_lines("gitwildmatch", [])
74
-
53
with open(gitignore_path, "r") as f:
54
return pathspec.PathSpec.from_lines("gitwildmatch", f)
55
78
-
56
def copy_project_files(src_dir, dst_dir, spec):
57
"""Copy project files respecting .gitignore rules using pathspec."""
58
src_path = Path(src_dir)
82
-
83
- # Walk through all the directories and files in the source directory
59
for root, dirs, files in os.walk(src_dir):
60
rel_root = Path(root).relative_to(src_path)
86
-
87
- # Filter out directories and files based on the .gitignore rules
61
for file in files:
62
rel_path = rel_root / file
63
if not spec.match_file(str(rel_path)):
@@ -93,39 +66,48 @@ def copy_project_files(src_dir, dst_dir, spec):
66
dst_file.parent.mkdir(parents=True, exist_ok=True)
67
shutil.copy2(src_file, dst_file)
68
96
-
97
-def cleanup_directories(bundle_name, keep_dist=False):
69
+def cleanup_directories(bundle_name, build_dir, dist_dir, keep_dist=False):
70
"""Remove build directory and .spec file. Optionally keep dist."""
99
- if not keep_dist and os.path.exists("dist"):
100
- shutil.rmtree("dist")
101
-
102
- if os.path.exists("build"):
103
- shutil.rmtree("build")
104
-
71
+ if not keep_dist and os.path.exists(dist_dir):
72
+ shutil.rmtree(dist_dir)
73
+ if os.path.exists(build_dir):
74
+ shutil.rmtree(build_dir)
75
spec_file = f"{bundle_name}.spec"
76
if os.path.exists(spec_file):
77
os.remove(spec_file)
78
109
-
110
-def build_executable(script_name, exe_name=None):
79
+def build_executable(script_path, exe_name=None):
80
"""Run PyInstaller with the correct site-packages path, clean, and additional data."""
81
try:
82
+ # Resolve the absolute path to the script, relative to the current file location (__file__)
83
+ bundling_script_dir = Path(__file__).parent.resolve()
84
+ script_path = (bundling_script_dir / script_path).resolve()
85
+ script_name = script_path.name # run_bundle.py
86
+ project_dir = script_path.parent # Folder containing run_bundle.py
87
+
88
+ # Define build and dist paths under the /bundle directory (bundling_script_dir)
89
+ build_dir = bundling_script_dir / "build"
90
+ dist_dir = bundling_script_dir / "dist"
91
+
92
# Initial cleanup
114
- cleanup_directories(exe_name, keep_dist=False)
93
+ cleanup_directories(exe_name, build_dir, dist_dir, keep_dist=False)
94
95
site_packages_path = get_site_packages_path()
96
print(f"Using site-packages path: {site_packages_path}")
97
+ print(f"Bundling project from: {project_dir}")
98
+ print(f"Build directory: {build_dir}")
99
+ print(f"Dist directory: {dist_dir}")
100
119
- # Parse .gitignore
120
- gitignore_path = os.path.join(os.getcwd(), ".gitignore")
101
+ # Parse .gitignore in the project directory
102
+ gitignore_path = project_dir / ".gitignore"
103
spec = parse_gitignore(gitignore_path)
104
123
- # Create a temporary directory for project files
124
- temp_project_dir = os.path.join("build", "temp_project")
105
+ # Create a temporary directory for project files inside build
106
+ temp_project_dir = build_dir / "temp_project"
107
os.makedirs(temp_project_dir, exist_ok=True)
108
109
# Copy project files respecting .gitignore
128
- copy_project_files(os.getcwd(), temp_project_dir, spec)
110
+ copy_project_files(project_dir, temp_project_dir, spec)
111
112
# Construct the PyInstaller command
113
pyinstaller_command = [
@@ -134,7 +116,8 @@ def build_executable(script_name, exe_name=None):
116
"--noconfirm",
117
"--onedir",
118
f"--paths={site_packages_path}",
137
- "--workpath=build", # Specify the build directory
119
+ f"--workpath={build_dir}", # Specify the build directory under /bundle
120
+ f"--distpath={dist_dir}", # Specify the dist directory under /bundle
121
]
122
123
# Add data arguments
@@ -146,61 +129,31 @@ def build_executable(script_name, exe_name=None):
129
else:
130
exe_name = os.path.splitext(script_name)[0]
131
149
- # Add the script path
132
+ # Add the script path (in the temp_project directory)
133
pyinstaller_command.append(os.path.join(temp_project_dir, script_name))
134
135
# Run the PyInstaller command
136
print("Running PyInstaller...")
137
subprocess.run(pyinstaller_command, check=True)
138
156
- # Post-processing: Manually create the desired structure
157
- dist_dir = os.path.join("dist", exe_name)
158
- project_files_dir = os.path.join(dist_dir, exe_name + "-files")
159
-
160
- # Move the executable to the root of dist/
161
- # shutil.move(os.path.join(temp_exe_dir, exe_name), os.path.join(dist_dir, exe_name))
162
-
163
- # Move _internal and other PyInstaller files to dist/
164
- # for item in os.listdir(temp_exe_dir):
165
- # if item != exe_name:
166
- # shutil.move(os.path.join(temp_exe_dir, item), os.path.join(dist_dir, item))
167
-
168
- # Remove the now-empty temporary executable directory
169
- # os.rmdir(temp_exe_dir)
170
-
171
- # Create project files directory
139
+ # Post-processing: Create a folder for project files inside dist/
140
+ project_files_dir = dist_dir / exe_name / f"{exe_name}-files"
141
os.makedirs(project_files_dir, exist_ok=True)
142
174
- # Copy project files
143
+ # Copy project files to the dist folder
144
copy_project_files(temp_project_dir, project_files_dir, spec)
145
146
print(f"PyInstaller finished successfully.")
178
- print(f"Executable created at: '{os.path.join(dist_dir, exe_name)}'")
147
+ print(f"Executable created at: '{dist_dir}/{exe_name}'")
148
print(f"Project files copied to: '{project_files_dir}'")
180
- print(f"Bundled contents are in: '{dist_dir}'")
181
-
182
- # Remove all __pycache__ directories in the _internal subdirectory
183
- internal_dir = os.path.join(dist_dir, "_internal")
184
- if os.path.exists(internal_dir):
185
- print(f"Cleaning up __pycache__ folders in {internal_dir}...")
186
- remove_pycache_folders(internal_dir)
149
150
# Final cleanup (keeping dist folder)
189
- cleanup_directories(exe_name, keep_dist=True)
151
+ cleanup_directories(exe_name, build_dir, dist_dir, keep_dist=True)
152
153
except subprocess.CalledProcessError as e:
154
print(f"Error during PyInstaller execution: {e}")
155
except Exception as e:
156
print(f"Error: {e}")
157
196
-
197
-def remove_pycache_folders(root_dir):
198
- for root, dirs, files in os.walk(root_dir):
199
- for dir_name in dirs:
200
- if dir_name == "__pycache__":
201
- pycache_path = os.path.join(root, dir_name)
202
- shutil.rmtree(pycache_path)
203
-
204
-
158
if __name__ == "__main__":
206
- build_executable("run_bundle.py", "agent-zero")
159
+ build_executable("../run_bundle.py", "agent-zero")
bundle/macos_bundle.sh
new
+70
@@ -0,0 +1,70 @@
1
+#!/bin/bash
2
+
3
+set -e
4
+
5
+# 1. Remove conda environment if it exists
6
+echo "Removing conda environment 'az-bundle' if it exists..."
7
+conda env remove -n az-bundle -y || echo "Conda environment 'az-bundle' does not exist."
8
+
9
+# 2. Create new environment with Python 3.12 and activate it
10
+echo "Creating new conda environment 'az-bundle' with Python 3.12..."
11
+conda create -n az-bundle python=3.12 -y
12
+if [ $? -ne 0 ]; then
13
+ echo "Error creating conda environment."
14
+ exit 1
15
+fi
16
+
17
+echo "Activating conda environment 'az-bundle'..."
18
+source $(conda info --base)/etc/profile.d/conda.sh
19
+conda activate az-bundle
20
+if [ $? -ne 0 ]; then
21
+ echo "Error activating conda environment."
22
+ exit 1
23
+fi
24
+
25
+# 3. Purge folder ./agent-zero (retry mechanism in case of failure)
26
+if [ -d "agent-zero" ]; then
27
+ echo "Deleting agent-zero folder..."
28
+ rm -rf agent-zero
29
+ if [ -d "agent-zero" ]; then
30
+ echo "Error: Unable to delete agent-zero folder, retrying..."
31
+ sleep 3
32
+ rm -rf agent-zero
33
+ fi
34
+ if [ -d "agent-zero" ]; then
35
+ echo "Error: Failed to purge agent-zero folder after retry."
36
+ exit 1
37
+ fi
38
+fi
39
+
40
+# 4. Clone the repository (development branch)
41
+echo "Cloning the repository (development branch)..."
42
+git clone --branch development https://github.com/frdel/agent-zero agent-zero
43
+if [ $? -ne 0 ]; then
44
+ echo "Error cloning the repository."
45
+ exit 1
46
+fi
47
+
48
+# 5. Change directory to agent-zero
49
+cd agent-zero || { echo "Error changing directory"; exit 1; }
50
+
51
+# 6. Install requirements
52
+echo "Installing requirements from requirements.txt..."
53
+pip install -r requirements.txt
54
+if [ $? -ne 0 ]; then
55
+ echo "Error installing requirements."
56
+ exit 1
57
+fi
58
+
59
+# 7. Install specific version of pefile
60
+# skip
61
+
62
+# 8. Run bundle.py
63
+echo "Running bundle.py..."
64
+python ./bundle/bundle.py
65
+if [ $? -ne 0 ]; then
66
+ echo "Error running bundle.py."
67
+ exit 1
68
+fi
69
+
70
+echo "Script completed."
bundle/windows_bundle.bat
new
+109
@@ -0,0 +1,109 @@
1
+@echo off
2
+setlocal enabledelayedexpansion
3
+
4
+:: Check if conda is recognized
5
+where conda >nul 2>nul
6
+if %errorlevel% neq 0 (
7
+ echo Conda not found in PATH. Checking known location...
8
+
9
+ set "CONDA_PATH=C:\Users\%USERNAME%\miniconda3"
10
+ if exist "!CONDA_PATH!\Scripts\conda.exe" (
11
+ echo Found Conda at !CONDA_PATH!
12
+ set "PATH=!CONDA_PATH!;!CONDA_PATH!\Scripts;!CONDA_PATH!\Library\bin;%PATH%"
13
+ echo Added Conda to PATH
14
+ ) else (
15
+ echo Conda installation not found at !CONDA_PATH!
16
+ echo Please install Conda or add it to PATH manually.
17
+ pause
18
+ exit /b 1
19
+ )
20
+)
21
+
22
+:: Verify conda is now accessible
23
+where conda >nul 2>nul
24
+if %errorlevel% neq 0 (
25
+ echo Failed to add Conda to PATH. Please add it manually.
26
+ pause
27
+ exit /b 1
28
+)
29
+
30
+:: Initialize conda shell (if not done before)
31
+call conda init bash >nul 2>nul
32
+if %errorlevel% neq 0 (
33
+ echo Error running 'conda init'. Please check your conda installation.
34
+ pause
35
+ exit /b 1
36
+)
37
+
38
+:: 1. Remove conda environment if it exists
39
+conda env remove -n az-bundle -y 2>nul
40
+if %errorlevel% neq 0 (
41
+ echo Error removing conda environment
42
+ pause
43
+)
44
+
45
+:: 2. Create new environment with Python 3.12 and activate it
46
+conda create -n az-bundle python=3.12 -y
47
+if %errorlevel% neq 0 (
48
+ echo Error creating conda environment
49
+ pause
50
+) else (
51
+ call conda.bat activate az-bundle
52
+ if %errorlevel% neq 0 (
53
+ echo Error activating conda environment
54
+ pause
55
+ )
56
+)
57
+
58
+:: 3. Purge folder ./agent-zero (retry mechanism in case of failure)
59
+if exist agent-zero (
60
+ echo Deleting agent-zero folder...
61
+ rmdir /s /q agent-zero
62
+ if exist agent-zero (
63
+ echo Error: Unable to delete agent-zero folder, retrying...
64
+ timeout /t 3 /nobreak >nul
65
+ rmdir /s /q agent-zero
66
+ )
67
+ if exist agent-zero (
68
+ echo Error: Failed to purge agent-zero folder after retry.
69
+ pause
70
+ )
71
+)
72
+
73
+:: 4. Clone the repository (development branch)
74
+git clone --branch development https://github.com/frdel/agent-zero agent-zero
75
+if %ERRORLEVEL% neq 0 (
76
+ echo Error cloning the repository
77
+ pause
78
+)
79
+
80
+:: 5. Change directory to agent-zero
81
+cd agent-zero
82
+if %errorlevel% neq 0 (
83
+ echo Error changing directory
84
+ pause
85
+)
86
+
87
+:: 6. Install requirements
88
+pip install -r requirements.txt
89
+if %errorlevel% neq 0 (
90
+ echo Error installing requirements
91
+ pause
92
+)
93
+
94
+:: 7. Install specific version of pefile
95
+pip install pefile==2023.2.7
96
+if %errorlevel% neq 0 (
97
+ echo Error installing pefile
98
+ pause
99
+)
100
+
101
+:: 8. Run bundle.py
102
+python bundle.py
103
+if %errorlevel% neq 0 (
104
+ echo Error running bundle.py
105
+ pause
106
+)
107
+
108
+echo Script completed
109
+pause