| 1 | #!/usr/bin/python3 |
| 2 | """ |
| 3 | build helper script for edk2, see |
| 4 | https://gitlab.com/kraxel/edk2-build-config |
| 5 | |
| 6 | """ |
| 7 | import os |
| 8 | import sys |
| 9 | import time |
| 10 | import shutil |
| 11 | import argparse |
| 12 | import subprocess |
| 13 | import configparser |
| 14 | |
| 15 | rebase_prefix = "" |
| 16 | version_override = None |
| 17 | release_date = None |
| 18 | |
| 19 | # pylint: disable=unused-variable |
| 20 | def check_rebase(): |
| 21 | """ detect 'git rebase -x edk2-build.py master' testbuilds """ |
| 22 | global rebase_prefix |
| 23 | global version_override |
| 24 | gitdir = '.git' |
| 25 | |
| 26 | if os.path.isfile(gitdir): |
| 27 | with open(gitdir, 'r', encoding = 'utf-8') as f: |
| 28 | (unused, gitdir) = f.read().split() |
| 29 | |
| 30 | if not os.path.exists(f'{gitdir}/rebase-merge/msgnum'): |
| 31 | return |
| 32 | with open(f'{gitdir}/rebase-merge/msgnum', 'r', encoding = 'utf-8') as f: |
| 33 | msgnum = int(f.read()) |
| 34 | with open(f'{gitdir}/rebase-merge/end', 'r', encoding = 'utf-8') as f: |
| 35 | end = int(f.read()) |
| 36 | with open(f'{gitdir}/rebase-merge/head-name', 'r', encoding = 'utf-8') as f: |
| 37 | head = f.read().strip().split('/') |
| 38 | |
| 39 | rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] ' |
| 40 | if msgnum != end and not version_override: |
| 41 | # fixed version speeds up builds |
| 42 | version_override = "test-build-patch-series" |
| 43 | |
| 44 | def get_coredir(cfg): |
| 45 | if cfg.has_option('global', 'core'): |
| 46 | return os.path.abspath(cfg['global']['core']) |
| 47 | return os.getcwd() |
| 48 | |
| 49 | def get_toolchain(cfg, build): |
| 50 | if cfg.has_option(build, 'tool'): |
| 51 | return cfg[build]['tool'] |
| 52 | if cfg.has_option('global', 'tool'): |
| 53 | return cfg['global']['tool'] |
| 54 | return 'GCC' |
| 55 | |
| 56 | def get_hostarch(): |
| 57 | mach = os.uname().machine |
| 58 | if mach == 'x86_64': |
| 59 | return 'X64' |
| 60 | if mach == 'aarch64': |
| 61 | return 'AARCH64' |
| 62 | if mach == 'riscv64': |
| 63 | return 'RISCV64' |
| 64 | return 'UNKNOWN' |
| 65 | |
| 66 | def get_version(cfg, silent = False): |
| 67 | coredir = get_coredir(cfg) |
| 68 | if version_override: |
| 69 | version = version_override |
| 70 | if not silent: |
| 71 | print('') |
| 72 | print(f'### version [override]: {version}') |
| 73 | return version |
| 74 | if os.environ.get('RPM_PACKAGE_NAME'): |
| 75 | version = os.environ.get('RPM_PACKAGE_NAME') |
| 76 | version += '-' + os.environ.get('RPM_PACKAGE_VERSION') |
| 77 | version += '-' + os.environ.get('RPM_PACKAGE_RELEASE') |
| 78 | if not silent: |
| 79 | print('') |
| 80 | print(f'### version [rpmbuild]: {version}') |
| 81 | return version |
| 82 | if os.path.exists(coredir + '/.git'): |
| 83 | cmdline = [ 'git', 'describe', '--tags', '--abbrev=8', |
| 84 | '--match=edk2-stable*' ] |
| 85 | result = subprocess.run(cmdline, cwd = coredir, |
| 86 | stdout = subprocess.PIPE, |
| 87 | check = True) |
| 88 | version = result.stdout.decode().strip() |
| 89 | if not silent: |
| 90 | print('') |
| 91 | print(f'### version [git]: {version}') |
| 92 | return version |
| 93 | return None |
| 94 | |
| 95 | def pcd_string(name, value): |
| 96 | return f'{name}=L{value}\\0' |
| 97 | |
| 98 | def pcd_version(cfg, silent = False): |
| 99 | version = get_version(cfg, silent) |
| 100 | if version is None: |
| 101 | return [] |
| 102 | return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ] |
| 103 | |
| 104 | def pcd_release_date(): |
| 105 | if release_date is None: |
| 106 | return [] |
| 107 | return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ] |
| 108 | |
| 109 | def build_message(line, line2 = None, silent = False): |
| 110 | if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]: |
| 111 | # setxterm title |
| 112 | start = '\x1b]2;' |
| 113 | end = '\x07' |
| 114 | print(f'{start}{rebase_prefix}{line}{end}', end = '') |
| 115 | |
| 116 | if silent: |
| 117 | print(f'### {rebase_prefix}{line}', flush = True) |
| 118 | else: |
| 119 | print('') |
| 120 | print('###') |
| 121 | print(f'### {rebase_prefix}{line}') |
| 122 | if line2: |
| 123 | print(f'### {line2}') |
| 124 | print('###', flush = True) |
| 125 | |
| 126 | def build_run(cmdline, name, section, silent = False, nologs = False): |
| 127 | if silent: |
| 128 | logfile = f'{section}.log' |
| 129 | if nologs: |
| 130 | print(f'### building in silent mode [no log] ...', flush = True) |
| 131 | else: |
| 132 | print(f'### building in silent mode [{logfile}] ...', flush = True) |
| 133 | start = time.time() |
| 134 | result = subprocess.run(cmdline, check = False, |
| 135 | stdout = subprocess.PIPE, |
| 136 | stderr = subprocess.STDOUT) |
| 137 | if not nologs: |
| 138 | with open(logfile, 'wb') as f: |
| 139 | f.write(result.stdout) |
| 140 | |
| 141 | if result.returncode: |
| 142 | print('### BUILD FAILURE') |
| 143 | print('### cmdline') |
| 144 | print(cmdline) |
| 145 | print('### output') |
| 146 | print(result.stdout.decode()) |
| 147 | print(f'### exit code: {result.returncode}') |
| 148 | else: |
| 149 | secs = int(time.time() - start) |
| 150 | print(f'### OK ({int(secs)}sec)') |
| 151 | else: |
| 152 | print(cmdline, flush = True) |
| 153 | result = subprocess.run(cmdline, check = False) |
| 154 | if result.returncode: |
| 155 | print(f'ERROR: {cmdline[0]} exited with {result.returncode}' |
| 156 | f' while building {name}') |
| 157 | sys.exit(result.returncode) |
| 158 | |
| 159 | def build_copy(plat, tgt, toolchain, dstdir, copy): |
| 160 | srcdir = f'Build/{plat}/{tgt}_{toolchain}' |
| 161 | names = copy.split() |
| 162 | srcfile = names[0] |
| 163 | if len(names) > 1: |
| 164 | dstfile = names[1] |
| 165 | else: |
| 166 | dstfile = os.path.basename(srcfile) |
| 167 | print(f'# copy: {srcdir} / {srcfile} => {dstdir} / {dstfile}') |
| 168 | |
| 169 | src = srcdir + '/' + srcfile |
| 170 | dst = dstdir + '/' + dstfile |
| 171 | os.makedirs(os.path.dirname(dst), exist_ok = True) |
| 172 | shutil.copy(src, dst) |
| 173 | |
| 174 | def pad_file(dstdir, pad): |
| 175 | args = pad.split() |
| 176 | if len(args) < 2: |
| 177 | raise RuntimeError(f'missing arg for pad ({args})') |
| 178 | name = args[0] |
| 179 | size = args[1] |
| 180 | cmdline = [ |
| 181 | 'truncate', |
| 182 | '--size', size, |
| 183 | dstdir + '/' + name, |
| 184 | ] |
| 185 | print(f'# padding: {dstdir} / {name} => {size}') |
| 186 | subprocess.run(cmdline, check = True) |
| 187 | |
| 188 | # pylint: disable=too-many-branches |
| 189 | def build_one(cfg, build, jobs = None, silent = False, nologs = False): |
| 190 | b = cfg[build] |
| 191 | |
| 192 | cmdline = [ 'build' ] |
| 193 | cmdline += [ '-t', get_toolchain(cfg, build) ] |
| 194 | cmdline += [ '-p', b['conf'] ] |
| 195 | |
| 196 | if (b['conf'].startswith('OvmfPkg/') or |
| 197 | b['conf'].startswith('ArmVirtPkg/')): |
| 198 | cmdline += pcd_version(cfg, silent) |
| 199 | cmdline += pcd_release_date() |
| 200 | |
| 201 | if jobs: |
| 202 | cmdline += [ '-n', jobs ] |
| 203 | for arch in b['arch'].split(): |
| 204 | if arch == 'HOST': |
| 205 | cmdline += [ '-a', get_hostarch() ] |
| 206 | else: |
| 207 | cmdline += [ '-a', arch ] |
| 208 | if 'opts' in b: |
| 209 | for name in b['opts'].split(): |
| 210 | section = 'opts.' + name |
| 211 | for opt in cfg[section]: |
| 212 | cmdline += [ '-D', opt + '=' + cfg[section][opt] ] |
| 213 | if 'pcds' in b: |
| 214 | for name in b['pcds'].split(): |
| 215 | section = 'pcds.' + name |
| 216 | for pcd in cfg[section]: |
| 217 | cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ] |
| 218 | if 'tgts' in b: |
| 219 | tgts = b['tgts'].split() |
| 220 | else: |
| 221 | tgts = [ 'DEBUG' ] |
| 222 | for tgt in tgts: |
| 223 | desc = None |
| 224 | if 'desc' in b: |
| 225 | desc = b['desc'] |
| 226 | build_message(f'building: {b["conf"]} ({b["arch"]}, {tgt})', |
| 227 | f'description: {desc}', |
| 228 | silent = silent) |
| 229 | build_run(cmdline + [ '-b', tgt ], |
| 230 | b['conf'], |
| 231 | build + '.' + tgt, |
| 232 | silent, |
| 233 | nologs) |
| 234 | |
| 235 | if 'plat' in b: |
| 236 | # copy files |
| 237 | for cpy in b: |
| 238 | if not cpy.startswith('cpy'): |
| 239 | continue |
| 240 | build_copy(b['plat'], tgt, |
| 241 | get_toolchain(cfg, build), |
| 242 | b['dest'], b[cpy]) |
| 243 | # pad builds |
| 244 | for pad in b: |
| 245 | if not pad.startswith('pad'): |
| 246 | continue |
| 247 | pad_file(b['dest'], b[pad]) |
| 248 | |
| 249 | def build_basetools(silent = False, nologs = False): |
| 250 | build_message('building: BaseTools', silent = silent) |
| 251 | basedir = os.environ['EDK_TOOLS_PATH'] + '/Source/C' |
| 252 | cmdline = [ 'make', '-C', basedir ] |
| 253 | build_run(cmdline, 'BaseTools', 'build.basetools', silent, nologs) |
| 254 | |
| 255 | def binary_exists(name): |
| 256 | for pdir in os.environ['PATH'].split(':'): |
| 257 | if os.path.exists(pdir + '/' + name): |
| 258 | return True |
| 259 | return False |
| 260 | |
| 261 | def prepare_env(cfg, silent = False): |
| 262 | """ mimic Conf/BuildEnv.sh """ |
| 263 | workspace = os.getcwd() |
| 264 | packages = [ workspace, ] |
| 265 | path = os.environ['PATH'].split(':') |
| 266 | dirs = [ |
| 267 | 'BaseTools/Bin/Linux-x86_64', |
| 268 | 'BaseTools/BinWrappers/PosixLike' |
| 269 | ] |
| 270 | |
| 271 | if cfg.has_option('global', 'pkgs'): |
| 272 | for pkgdir in cfg['global']['pkgs'].split(): |
| 273 | packages.append(os.path.abspath(pkgdir)) |
| 274 | coredir = get_coredir(cfg) |
| 275 | if coredir != workspace: |
| 276 | packages.append(coredir) |
| 277 | |
| 278 | # add basetools to path |
| 279 | for pdir in dirs: |
| 280 | p = coredir + '/' + pdir |
| 281 | if not os.path.exists(p): |
| 282 | continue |
| 283 | if p in path: |
| 284 | continue |
| 285 | path.insert(0, p) |
| 286 | |
| 287 | # run edksetup if needed |
| 288 | toolsdef = coredir + '/Conf/tools_def.txt' |
| 289 | if not os.path.exists(toolsdef): |
| 290 | os.makedirs(os.path.dirname(toolsdef), exist_ok = True) |
| 291 | build_message('running BaseTools/BuildEnv', silent = silent) |
| 292 | cmdline = [ 'bash', 'BaseTools/BuildEnv' ] |
| 293 | subprocess.run(cmdline, cwd = coredir, check = True) |
| 294 | |
| 295 | # set variables |
| 296 | os.environ['PATH'] = ':'.join(path) |
| 297 | os.environ['PACKAGES_PATH'] = ':'.join(packages) |
| 298 | os.environ['WORKSPACE'] = workspace |
| 299 | os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools' |
| 300 | os.environ['CONF_PATH'] = coredir + '/Conf' |
| 301 | os.environ['PYTHON_COMMAND'] = '/usr/bin/python3' |
| 302 | os.environ['PYTHONHASHSEED'] = '1' |
| 303 | |
| 304 | # for cross builds |
| 305 | if binary_exists('arm-linux-gnueabi-gcc'): |
| 306 | # ubuntu |
| 307 | os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnueabi-' |
| 308 | os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnueabi-' |
| 309 | elif binary_exists('arm-linux-gnu-gcc'): |
| 310 | # fedora |
| 311 | os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-' |
| 312 | os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnu-' |
| 313 | if binary_exists('loongarch64-linux-gnu-gcc'): |
| 314 | os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-' |
| 315 | os.environ['GCC_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-' |
| 316 | |
| 317 | hostarch = os.uname().machine |
| 318 | if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64': |
| 319 | os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-' |
| 320 | os.environ['GCC_AARCH64_PREFIX'] = 'aarch64-linux-gnu-' |
| 321 | if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64': |
| 322 | os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-' |
| 323 | os.environ['GCC_RISCV64_PREFIX'] = 'riscv64-linux-gnu-' |
| 324 | if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64': |
| 325 | os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-' |
| 326 | os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-' |
| 327 | os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-' |
| 328 | os.environ['GCC_IA32_PREFIX'] = 'x86_64-linux-gnu-' |
| 329 | os.environ['GCC_X64_PREFIX'] = 'x86_64-linux-gnu-' |
| 330 | os.environ['GCC_BIN'] = 'x86_64-linux-gnu-' |
| 331 | |
| 332 | def build_list(cfg): |
| 333 | for build in cfg.sections(): |
| 334 | if not build.startswith('build.'): |
| 335 | continue |
| 336 | name = build.lstrip('build.') |
| 337 | desc = 'no description' |
| 338 | if 'desc' in cfg[build]: |
| 339 | desc = cfg[build]['desc'] |
| 340 | print(f'# {name:20s} - {desc}') |
| 341 | |
| 342 | def main(): |
| 343 | parser = argparse.ArgumentParser(prog = 'edk2-build', |
| 344 | description = 'edk2 build helper script') |
| 345 | parser.add_argument('-c', '--config', dest = 'configfile', |
| 346 | type = str, default = '.edk2.builds', metavar = 'FILE', |
| 347 | help = 'read configuration from FILE (default: .edk2.builds)') |
| 348 | parser.add_argument('-C', '--directory', dest = 'directory', type = str, |
| 349 | help = 'change to DIR before building', metavar = 'DIR') |
| 350 | parser.add_argument('-j', '--jobs', dest = 'jobs', type = str, |
| 351 | help = 'allow up to JOBS parallel build jobs', |
| 352 | metavar = 'JOBS') |
| 353 | parser.add_argument('-m', '--match', dest = 'match', |
| 354 | type = str, action = 'append', |
| 355 | help = 'only run builds matching INCLUDE (substring)', |
| 356 | metavar = 'INCLUDE') |
| 357 | parser.add_argument('-x', '--exclude', dest = 'exclude', |
| 358 | type = str, action = 'append', |
| 359 | help = 'skip builds matching EXCLUDE (substring)', |
| 360 | metavar = 'EXCLUDE') |
| 361 | parser.add_argument('-l', '--list', dest = 'list', |
| 362 | action = 'store_true', default = False, |
| 363 | help = 'list build configs available') |
| 364 | parser.add_argument('--silent', dest = 'silent', |
| 365 | action = 'store_true', default = False, |
| 366 | help = 'write build output to logfiles, ' |
| 367 | 'write to console only on errors') |
| 368 | parser.add_argument('--no-logs', dest = 'nologs', |
| 369 | action = 'store_true', default = False, |
| 370 | help = 'do not write build log files (with --silent)') |
| 371 | parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR', |
| 372 | help = 'location of the core edk2 repository ' |
| 373 | '(i.e. where BuildTools are located)') |
| 374 | parser.add_argument('--pkg', '--package', dest = 'pkgs', |
| 375 | type = str, action = 'append', metavar = 'DIR', |
| 376 | help = 'location(s) of additional packages ' |
| 377 | '(can be specified multiple times)') |
| 378 | parser.add_argument('-t', '--toolchain', dest = 'toolchain', |
| 379 | type = str, metavar = 'NAME', |
| 380 | help = 'tool chain to be used to build edk2') |
| 381 | parser.add_argument('--version-override', dest = 'version_override', |
| 382 | type = str, metavar = 'VERSION', |
| 383 | help = 'set firmware build version') |
| 384 | parser.add_argument('--release-date', dest = 'release_date', |
| 385 | type = str, metavar = 'DATE', |
| 386 | help = 'set firmware build release date (in MM/DD/YYYY format)') |
| 387 | options = parser.parse_args() |
| 388 | |
| 389 | if options.directory: |
| 390 | os.chdir(options.directory) |
| 391 | |
| 392 | if not os.path.exists(options.configfile): |
| 393 | print(f'config file "{options.configfile}" not found') |
| 394 | return 1 |
| 395 | |
| 396 | cfg = configparser.ConfigParser() |
| 397 | cfg.optionxform = str |
| 398 | cfg.read(options.configfile) |
| 399 | |
| 400 | if options.list: |
| 401 | build_list(cfg) |
| 402 | return 0 |
| 403 | |
| 404 | if not cfg.has_section('global'): |
| 405 | cfg.add_section('global') |
| 406 | if options.core: |
| 407 | cfg.set('global', 'core', options.core) |
| 408 | if options.pkgs: |
| 409 | cfg.set('global', 'pkgs', ' '.join(options.pkgs)) |
| 410 | if options.toolchain: |
| 411 | cfg.set('global', 'tool', options.toolchain) |
| 412 | |
| 413 | global version_override |
| 414 | global release_date |
| 415 | check_rebase() |
| 416 | if options.version_override: |
| 417 | version_override = options.version_override |
| 418 | if options.release_date: |
| 419 | release_date = options.release_date |
| 420 | |
| 421 | prepare_env(cfg, options.silent) |
| 422 | build_basetools(options.silent, options.nologs) |
| 423 | for build in cfg.sections(): |
| 424 | if not build.startswith('build.'): |
| 425 | continue |
| 426 | if options.match: |
| 427 | matching = False |
| 428 | for item in options.match: |
| 429 | if item in build: |
| 430 | matching = True |
| 431 | if not matching: |
| 432 | print(f'# skipping "{build}" (not matching "{"|".join(options.match)}")') |
| 433 | continue |
| 434 | if options.exclude: |
| 435 | exclude = False |
| 436 | for item in options.exclude: |
| 437 | if item in build: |
| 438 | print(f'# skipping "{build}" (matching "{item}")') |
| 439 | exclude = True |
| 440 | if exclude: |
| 441 | continue |
| 442 | build_one(cfg, build, options.jobs, options.silent, options.nologs) |
| 443 | |
| 444 | return 0 |
| 445 | |
| 446 | if __name__ == '__main__': |
| 447 | sys.exit(main()) |