X-Git-Url: https://git.carlh.net/gitweb/?a=blobdiff_plain;f=cdist;h=0e1df9f7c9abf6d3942daa4aad6ac7516ed75b1b;hb=714a5cf84cf08d53fb76ec42c5bec0296ac245a3;hp=a36f42b744e2672a3777ccdf9fbbad4091af2ebf;hpb=58cf328973c03988fd7873af6818e9a9d7a397b3;p=cdist.git diff --git a/cdist b/cdist index a36f42b..0e1df9f 100755 --- a/cdist +++ b/cdist @@ -1,6 +1,6 @@ #!/usr/bin/python3 -# Copyright (C) 2012-2020 Carl Hetherington +# Copyright (C) 2012-2022 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,8 +26,10 @@ import glob import inspect import multiprocessing import os +from pathlib import Path import platform import re +import signal import shlex import shutil import subprocess @@ -52,25 +54,25 @@ class Trees: def __init__(self): self.trees = [] - def get(self, name, specifier, target, required_by=None): + def get(self, name, commit_ish, target, required_by=None): for t in self.trees: - if t.name == name and t.specifier == specifier and t.target == target: + if t.name == name and t.commit_ish == commit_ish and t.target == target: return t - elif t.name == name and t.specifier != specifier: - a = specifier if specifier is not None else "[Any]" + elif t.name == name and t.commit_ish != commit_ish: + a = commit_ish if commit_ish is not None else "[Any]" if required_by is not None: a += ' by %s' % required_by - b = t.specifier if t.specifier is not None else "[Any]" + b = t.commit_ish if t.commit_ish is not None else "[Any]" if t.required_by is not None: b += ' by %s' % t.required_by raise Error('conflicting versions of %s required (%s versus %s)' % (name, a, b)) - nt = Tree(name, specifier, target, required_by) + nt = Tree(name, commit_ish, target, required_by) self.trees.append(nt) return nt - def add_built(self, name, specifier, target): - self.trees.append(Tree(name, specifier, target, None, built=True)) + def add_built(self, name, commit_ish, target): + self.trees.append(Tree(name, commit_ish, target, None, built=True)) class Globals: @@ -102,7 +104,7 @@ class BoolOption(object): def offer(self, key, value): if key == self.key: - self.value = (value == 'yes' or value == '1' or value == 'true') + self.value = value in ['yes', '1', 'true'] class Config: def __init__(self): @@ -112,6 +114,9 @@ class Config: Option('osx_environment_prefix'), Option('osx_sdk_prefix'), Option('osx_sdk'), + Option('osx_intel_deployment'), + Option('osx_arm_deployment'), + Option('osx_old_deployment'), Option('osx_keychain_file'), Option('osx_keychain_password'), Option('apple_id'), @@ -440,19 +445,19 @@ class Target(object): def _copy_packages(self, tree, packages, output_dir): for p in packages: - copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) + copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, p)))) def package(self, project, checkout, output_dir, options, notarize): - tree = self.build(project, checkout, options) + tree = self.build(project, checkout, options, for_package=True) tree.add_defaults(options) p = self._cscript_package(tree, options) self._copy_packages(tree, p, output_dir) - def build(self, project, checkout, options): + def build(self, project, checkout, options, for_package=False): tree = globals.trees.get(project, checkout, self) if self.build_dependencies: tree.build_dependencies(options) - tree.build(options) + tree.build(options, for_package=for_package) return tree def test(self, project, checkout, target, test, options): @@ -539,12 +544,17 @@ class DockerTarget(Target): if self.privileged: opts += '--privileged=true ' if self.ccache: - opts += "-e CCACHE_DIR=/ccache/%s-%d --mount source=ccache,target=/ccache" % (self.image, os.getuid()) + opts += "-e CCACHE_DIR=/ccache/%s-%d --mount source=ccache,target=/ccache " % (self.image, os.getuid()) + opts += "--rm " tag = self.image if config.has('docker_hub_repository'): tag = '%s:%s' % (config.get('docker_hub_repository'), tag) + def signal_handler(signum, frame): + raise Error('Killed') + signal.signal(signal.SIGTERM, signal_handler) + self.container = command_and_read('%s run %s %s -itd %s /bin/bash' % (config.docker(), self._user_tag(), opts, tag))[0].strip() def command(self, cmd): @@ -746,13 +756,22 @@ def notarize_dmg(dmg, bundle_id): request_uuid = string_after(p, "RequestUUID") if request_uuid is None: + print("Looking for upload ID") + message = string_after(p, "message") + print("Looking in %s" % message) + if message: + m = re.match('.*The upload ID is ([0-9a-f\-]*)', message) + if m: + request_uuid = m.groups()[0] + if request_uuid is None: + print("Response: %s" % p) raise Error('No RequestUUID found in response from Apple') for i in range(0, 30): - print('Checking up on %s' % request_uuid) + print('%s: checking up on %s' % (datetime.datetime.now(), request_uuid)) p = subprocess.run(['xcrun', 'altool', '--notarization-info', request_uuid, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'], capture_output=True) status = string_after(p, 'Status') - print('Got %s' % status) + print('%s: got status %s' % (datetime.datetime.now(), status)) if status == 'invalid': raise Error("Notarization failed") elif status == 'success': @@ -782,9 +801,16 @@ class OSXTarget(Target): def unlock_keychain(self): self.command('security unlock-keychain -p %s %s' % (self.osx_keychain_password, self.osx_keychain_file)) + def _copy_packages(self, tree, packages, output_dir): + for p in packages: + dest = os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, p))) + copyfile(p, dest) + if os.path.exists(p + ".id"): + copyfile(p + ".id", dest + ".id") + def _cscript_package_and_notarize(self, tree, options, notarize): """ - Call package() in the cscript and notarize the .dmgs that are returned, if notarize = True + Call package() in the cscript and notarize the .dmgs that are returned, if notarize == True """ p = self._cscript_package(tree, options) for x in p: @@ -792,30 +818,39 @@ class OSXTarget(Target): raise Error('macOS packages must be returned from cscript as tuples of (dmg-filename, bundle-id)') if notarize: notarize_dmg(x[0], x[1]) + else: + with open(x[0] + '.id', 'w') as f: + print(x[1], file=f) return [x[0] for x in p] class OSXSingleTarget(OSXTarget): - def __init__(self, arch, sdk, directory=None): + def __init__(self, arch, sdk, deployment, directory=None, can_notarize=True): super(OSXSingleTarget, self).__init__(directory) self.arch = arch self.sdk = sdk + self.deployment = deployment + self.can_notarize = can_notarize + self.sub_targets = [self] flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, sdk, arch) - host_enviro = '%s/x86_64/10.9' % config.get('osx_environment_prefix') - target_enviro = '%s/%s/%s' % (config.get('osx_environment_prefix'), arch, sdk) + if arch == 'x86_64': + host_enviro = '%s/x86_64/%s' % (config.get('osx_environment_prefix'), deployment) + else: + host_enviro = '%s/x86_64/10.10' % config.get('osx_environment_prefix') + target_enviro = '%s/%s/%s' % (config.get('osx_environment_prefix'), arch, deployment) self.bin = '%s/bin' % target_enviro # Environment variables self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags)) self.set('CPPFLAGS', '') - self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, target_enviro, flags)) - self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags)) + self.set('CXXFLAGS', '"-I%s/include -I%s/include -stdlib=libc++ %s"' % (self.directory, target_enviro, flags)) + self.set('LDFLAGS', '"-L%s/lib -L%s/lib -stdlib=libc++ %s"' % (self.directory, target_enviro, flags)) self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, target_enviro, flags)) self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, target_enviro)) self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % host_enviro) - self.set('MACOSX_DEPLOYMENT_TARGET', sdk) + self.set('MACOSX_DEPLOYMENT_TARGET', self.deployment) self.set('CCACHE_BASEDIR', self.directory) @Target.ccache.setter @@ -826,35 +861,34 @@ class OSXSingleTarget(OSXTarget): self.set('CXX', '"ccache g++"') def package(self, project, checkout, output_dir, options, notarize): - tree = self.build(project, checkout, options) + tree = self.build(project, checkout, options, for_package=True) tree.add_defaults(options) self.unlock_keychain() - p = self._cscript_package_and_notarize(tree, options, notarize) + p = self._cscript_package_and_notarize(tree, options, self.can_notarize and notarize) self._copy_packages(tree, p, output_dir) class OSXUniversalTarget(OSXTarget): - def __init__(self, archs, directory=None): + def __init__(self, directory=None): super(OSXUniversalTarget, self).__init__(directory) - self.archs = archs self.sdk = config.get('osx_sdk') - for a in self.archs: - if a.find('arm') != -1: - self.sdk = '11.0' + self.sub_targets = [] + for arch, deployment in (('x86_64', config.get('osx_intel_deployment')), ('arm64', config.get('osx_arm_deployment'))): + target = OSXSingleTarget(arch, self.sdk, deployment, os.path.join(self.directory, arch, deployment)) + target.ccache = self.ccache + self.sub_targets.append(target) def package(self, project, checkout, output_dir, options, notarize): - for a in self.archs: - target = OSXSingleTarget(a, self.sdk, os.path.join(self.directory, a)) - target.ccache = self.ccache + for target in self.sub_targets: tree = globals.trees.get(project, checkout, target) tree.build_dependencies(options) - tree.build(options) + tree.build(options, for_package=True) self.unlock_keychain() tree = globals.trees.get(project, checkout, self) with TreeDirectory(tree): - for p in self._cscript_package_and_notarize(tree, options, notarize): - copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) + p = self._cscript_package_and_notarize(tree, options, notarize) + self._copy_packages(tree, p, output_dir) class SourceTarget(Target): """Build a source .tar.bz2""" @@ -874,7 +908,7 @@ class SourceTarget(Target): name = read_wscript_variable(os.getcwd(), 'APPNAME') command('./waf dist') p = os.path.abspath('%s-%s.tar.bz2' % (name, tree.version)) - copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p)))) + copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.commit, p)))) # @param s Target string: # windows-{32,64} @@ -883,11 +917,10 @@ class SourceTarget(Target): # or centos-version-{32,64} # or fedora-version-{32,64} # or mageia-version-{32,64} -# or osx-{intel,arm-intel,arm64} +# or osx # or source # or flatpak # or appimage -# @param debug True to build with debugging symbols (where possible) def target_factory(args): s = args.target target = None @@ -914,14 +947,12 @@ def target_factory(args): target = LinuxTarget(p[0], None, int(p[1]), args.work) elif s == 'raspbian': target = LinuxTarget(s, None, None, args.work) + elif s == 'osx': + target = OSXUniversalTarget(args.work) elif s == 'osx-intel': - # Intel 64-bit built for config's os_sdk - target = OSXSingleTarget('x86_64', config.get('osx_sdk'), args.work) - elif s == 'osx-arm-intel': - # Universal arm64 and Intel 64-bit built for SDK 11.0 - target = OSXUniversalTarget(('arm64', 'x86_64'), args.work) - elif s == 'osx-arm64': - target = OSXSingleTarget('arm64', '11.0', args.work) + target = OSXSingleTarget('x86_64', config.get('osx_sdk'), config.get('osx_intel_deployment'), args.work) + elif s == 'osx-old': + target = OSXSingleTarget('x86_64', config.get('osx_sdk'), config.get('osx_old_deployment'), args.work, False) elif s == 'source': target = SourceTarget() elif s == 'flatpak': @@ -956,20 +987,20 @@ class Tree(object): possibly built. This class is never exposed to cscripts. Attributes: name -- name of git repository (without the .git) - specifier -- git tag or revision to use + commit_ish -- git tag or revision to use target -- target object that we are using version -- version from the wscript (if one is present) - git_commit -- git revision that is actually being used + commit -- git revision that is actually being used built -- true if the tree has been built yet in this run required_by -- name of the tree that requires this one """ - def __init__(self, name, specifier, target, required_by, built=False): + def __init__(self, name, commit_ish, target, required_by, built=False): self.name = name - self.specifier = specifier + self.commit_ish = commit_ish self.target = target self.version = None - self.git_commit = None + self.commit = None self.built = built self.required_by = required_by @@ -989,12 +1020,9 @@ class Tree(object): command('git clone %s %s %s/%s.git %s/src/%s' % (flags, ref, config.get('git_prefix'), self.name, target.directory, self.name)) os.chdir('%s/src/%s' % (target.directory, self.name)) - spec = self.specifier - if spec is None: - spec = 'master' - - command('git checkout %s %s %s' % (flags, spec, redirect)) - self.git_commit = command_and_read('git rev-parse --short=7 HEAD')[0].strip() + if self.commit_ish is not None: + command('git checkout %s %s %s' % (flags, self.commit_ish, redirect)) + self.commit = command_and_read('git rev-parse --short=7 HEAD')[0].strip() self.cscript = {} exec(open('%s/cscript' % proj).read(), self.cscript) @@ -1022,7 +1050,7 @@ class Tree(object): self.version = Version(v) except: try: - tag = command_and_read('git -C %s describe --tags' % proj)[0][1:] + tag = command_and_read('git -C %s describe --match v* --tags' % proj)[0][1:] self.version = Version.from_git_tag(tag) except: # We'll leave version as None if we can't read it; maybe this is a bad idea @@ -1087,11 +1115,11 @@ class Tree(object): for i in self.dependencies(options): i[0].build(i[1]) - def build(self, options): + def build(self, options, for_package=False): if self.built: return - log_verbose("Building %s %s %s with %s" % (self.name, self.specifier, self.version, options)) + log_verbose("Building %s %s %s with %s" % (self.name, self.commit_ish, self.version, options)) variables = copy.copy(self.target.variables) @@ -1099,7 +1127,10 @@ class Tree(object): self.add_defaults(options) if not globals.dry_run: - if len(inspect.getfullargspec(self.cscript['build']).args) == 2: + num_args = len(inspect.getfullargspec(self.cscript['build']).args) + if num_args == 3: + self.call('build', options, for_package) + elif num_args == 2: self.call('build', options) else: self.call('build') @@ -1173,6 +1204,8 @@ def main(): parser_checkout = subparsers.add_parser("checkout", help="check out the project") parser_revision = subparsers.add_parser("revision", help="print the head git revision number") parser_dependencies = subparsers.add_parser("dependencies", help="print details of the project's dependencies as a .dot file") + parser_notarize = subparsers.add_parser("notarize", help="notarize .dmgs in a directory using *.dmg.id files") + parser_notarize.add_argument('--dmgs', help='directory containing *.dmg and *.dmg.id') global args args = parser.parse_args() @@ -1203,7 +1236,7 @@ def main(): if not os.path.exists(args.work): os.makedirs(args.work) - if args.project is None and args.command != 'shell': + if args.project is None and not args.command in ['shell', 'notarize']: raise Error('you must specify -p or --project') globals.quiet = args.quiet @@ -1215,9 +1248,11 @@ def main(): raise Error('you must specify -t or --target') target = target_factory(args) - target.build(args.project, args.checkout, get_command_line_options(args)) - if not args.keep: - target.cleanup() + try: + target.build(args.project, args.checkout, get_command_line_options(args)) + finally: + if not args.keep: + target.cleanup() elif args.command == 'package': if args.target is None: @@ -1237,13 +1272,9 @@ def main(): makedirs(output_dir) target.package(args.project, args.checkout, output_dir, get_command_line_options(args), not args.no_notarize) - except Error as e: + finally: if target is not None and not args.keep: target.cleanup() - raise - - if target is not None and not args.keep: - target.cleanup() elif args.command == 'release': if args.minor is False and args.micro is False: @@ -1278,6 +1309,7 @@ def main(): elif args.command == 'manual': target = SourceTarget() tree = globals.trees.get(args.project, args.checkout, target) + tree.checkout_dependencies() outs = tree.call('make_manual') for o in outs: @@ -1383,6 +1415,18 @@ def main(): print("%s -> %s;" % (d[2].name.replace("-", "-"), d[0].name.replace("-", "_"))) print("}") + elif args.command == 'notarize': + if args.dmgs is None: + raise Error('you must specify ---dmgs') + + for dmg in Path(args.dmgs).glob('*.dmg'): + id = None + try: + with open(str(dmg) + '.id') as f: + id = f.readline().strip() + except OSError: + raise Error('could not find ID file for %s' % dmg) + notarize_dmg(dmg, id) try: main()