-#!/usr/bin/python
+#!/usr/bin/python3
# Copyright (C) 2012-2020 Carl Hetherington <cth@carlh.net>
#
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
from __future__ import print_function
-import os
-import sys
-import shutil
-import glob
-import tempfile
+
import argparse
-import datetime
-import subprocess
-import re
import copy
-import inspect
+import datetime
import getpass
-import shlex
+import glob
+import inspect
import multiprocessing
+import os
+import re
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
TEMPORARY_DIRECTORY = '/var/tmp'
def __init__(self):
self.options = [ Option('mxe_prefix'),
Option('git_prefix'),
+ Option('git_reference'),
Option('osx_environment_prefix'),
Option('osx_sdk_prefix'),
Option('osx_sdk'),
+ Option('osx_keychain_file'),
+ Option('osx_keychain_password'),
Option('apple_id'),
Option('apple_password'),
BoolOption('docker_sudo'),
command('scp -r %s %s' % (scp_escape(a), scp_escape(b)))
def copyfile(a, b):
- log_normal('copy %s -> %s' % (scp_escape(a), scp_escape(b)))
+ log_normal('copy %s -> %s with cwd %s' % (scp_escape(a), scp_escape(b), os.getcwd()))
if b.startswith('s3://'):
command('s3cmd -P put "%s" "%s"' % (a, b))
else:
(out, err) = p.communicate()
if p.returncode != 0:
raise Error('command %s failed (%s)' % (c, err))
- return out.splitlines()
+ return str(out, 'utf-8').splitlines()
def read_wscript_variable(directory, variable):
f = open('%s/wscript' % directory, 'r')
self.set('CCACHE_BASEDIR', os.path.realpath(self.directory))
self.set('CCACHE_NOHASHDIR', '')
else:
- self.directory = directory
+ self.directory = os.path.realpath(directory)
self.rmdir = False
def setup(self):
pass
- def package(self, project, checkout, output_dir, options):
- tree = self.build(project, checkout, options)
- tree.add_defaults(options)
- if len(inspect.getargspec(tree.cscript['package']).args) == 3:
+ def _build_packages(self, tree, options):
+ if len(inspect.getfullargspec(tree.cscript['package']).args) == 3:
packages = tree.call('package', tree.version, options)
else:
log_normal("Deprecated cscript package() method with no options parameter")
packages = tree.call('package', tree.version)
- if isinstance(packages, (str, unicode)):
- copyfile(packages, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, packages))))
- else:
- for p in packages:
- copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p))))
+ return packages if isinstance(packages, list) else [packages]
+
+ def _copy_packages(self, tree, packages, output_dir):
+ for p in packages:
+ copyfile(p, os.path.join(output_dir, os.path.basename(devel_to_git(tree.git_commit, p))))
+
+ def package(self, project, checkout, output_dir, options, no_notarize):
+ tree = self.build(project, checkout, options)
+ tree.add_defaults(options)
+ p = self._build_packages(tree, options)
+ self._copy_packages(tree, p, output_dir)
def build(self, project, checkout, options):
tree = globals.trees.get(project, checkout, self)
if self.build_dependencies:
tree.build_dependencies(options)
tree.build(options)
- return tree.call('test', test)
+
+ tree.add_defaults(options)
+ if len(inspect.getfullargspec(tree.cscript['test']).args) == 3:
+ return tree.call('test', options, test)
+ else:
+ log_normal('Deprecated cscript test() method with no options parameter')
+ return tree.call('test', test)
def set(self, a, b):
self.variables[a] = b
return ''
return '-u %s' % getpass.getuser()
+ def _mount_option(self, d):
+ return '-v %s:%s ' % (os.path.realpath(d), os.path.realpath(d))
+
def setup(self):
- opts = '-v %s:%s ' % (self.directory, self.directory)
+ opts = self._mount_option(self.directory)
for m in self.mounts:
- opts += '-v %s:%s ' % (m, m)
+ opts += self._mount_option(m)
+ if config.has('git_reference'):
+ opts += self._mount_option(config.get('git_reference'))
if self.privileged:
opts += '--privileged=true '
if self.ccache:
- opts += "-e CCACHE_DIR=/ccache --volumes-from ccache-%s" % self.image
+ opts += "-e CCACHE_DIR=/ccache/%s-%d --mount source=ccache,target=/ccache" % (self.image, os.getuid())
tag = self.image
if config.has('docker_hub_repository'):
self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.environment_prefix)
self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory))
self.set('PATH', '%s/bin:%s:%s' % (self.environment_prefix, self.tool_path, os.environ['PATH']))
- self.set('CC', '%s-gcc' % self.name)
- self.set('CXX', '%s-g++' % self.name)
self.set('LD', '%s-ld' % self.name)
self.set('RANLIB', '%s-ranlib' % self.name)
self.set('WINRC', '%s-windres' % self.name)
if environment_version is not None:
self.image += '_%s' % environment_version
+ def setup(self):
+ super().setup()
+ if self.ccache:
+ self.set('CC', '"ccache %s-gcc"' % self.name)
+ self.set('CXX', '"ccache %s-g++"' % self.name)
+ else:
+ self.set('CC', '%s-gcc' % self.name)
+ self.set('CXX', '%s-g++' % self.name)
+
@property
def library_prefix(self):
log_normal('Deprecated property library_prefix: use environment_prefix')
class AppImageTarget(LinuxTarget):
def __init__(self, work):
- super(AppImageTarget, self).__init__('ubuntu', '16.04', 64, work)
+ super(AppImageTarget, self).__init__('ubuntu', '18.04', 64, work)
self.detail = 'appimage'
self.privileged = True
+def notarize(dmg, bundle_id):
+ p = subprocess.run(
+ ['xcrun', 'altool', '--notarize-app', '-t', 'osx', '-f', dmg, '--primary-bundle-id', bundle_id, '-u', config.get('apple_id'), '-p', config.get('apple_password'), '--output-format', 'xml'],
+ capture_output=True
+ )
+
+ def string_after(process, key):
+ lines = p.stdout.decode('utf-8').splitlines()
+ request_uuid = None
+ for i in range(0, len(lines)):
+ if lines[i].find(key) != -1:
+ return lines[i+1].strip().replace('<string>', '').replace('</string>', '')
+
+ raise Error("Missing expected response %s from Apple" % key)
+
+ request_uuid = string_after(p, "RequestUUID")
+
+ for i in range(0, 30):
+ print('Checking up on %s' % request_uuid)
+ p = subprocess.run(['xcrun', 'altool', '--notarization-info', request_uuid, '-u', apple_id, '-p', apple_password, '--output-format', 'xml'], capture_output=True)
+ status = string_after(p, 'Status')
+ print('Got %s' % status)
+ if status == 'invalid':
+ raise Error("Notarization failed")
+ elif status == 'success':
+ subprocess.run(['xcrun', 'stapler', 'staple', dmg])
+ return
+ time.sleep(30)
+
+ raise Error("Notarization timed out")
+
+
class OSXTarget(Target):
def __init__(self, directory=None):
super(OSXTarget, self).__init__('osx', directory)
self.environment_prefix = config.get('osx_environment_prefix')
self.apple_id = config.get('apple_id')
self.apple_password = config.get('apple_password')
+ self.osx_keychain_file = config.get('osx_keychain_file')
+ self.osx_keychain_password = config.get('osx_keychain_password')
def command(self, c):
command('%s %s' % (self.variables_string(False), c))
+ def build(self, *a, **k):
+ self.command('security unlock-keychain -p %s %s' % (self.osx_keychain_password, self.osx_keychain_file))
+ return super().build(*a, **k)
+
class OSXSingleTarget(OSXTarget):
def __init__(self, bits, directory=None):
self.set('CC', '"ccache gcc"')
self.set('CXX', '"ccache g++"')
+ def package(self, project, checkout, output_dir, options, no_notarize):
+ tree = self.build(project, checkout, options)
+ tree.add_defaults(options)
+ p = self._build_packages(tree, options)
+ for x in p:
+ if not isinstance(x, tuple):
+ raise Error('macOS packages must be returned from cscript as tuples of (dmg-filename, bundle-id)')
+ if not no_notarize:
+ notarize(x[0], x[1])
+ self._copy_packages(tree, [x[0] for x in p], output_dir)
+
class OSXUniversalTarget(OSXTarget):
def __init__(self, directory=None):
super(OSXUniversalTarget, self).__init__(directory)
self.bits = None
- def package(self, project, checkout, output_dir, options):
+ def package(self, project, checkout, output_dir, options, no_notarize):
for b in [32, 64]:
target = OSXSingleTarget(b, os.path.join(self.directory, '%d' % b))
tree = globals.trees.get(project, checkout, self)
with TreeDirectory(tree):
- if len(inspect.getargspec(tree.cscript['package']).args) == 3:
+ if len(inspect.getfullargspec(tree.cscript['package']).args) == 3:
packages = tree.call('package', tree.version, options)
else:
log_normal("Deprecated cscript package() method with no options parameter")
if globals.quiet:
flags = '-q'
redirect = '>/dev/null'
- command('git clone %s %s/%s.git %s/src/%s' % (flags, config.get('git_prefix'), self.name, target.directory, self.name))
+ if config.has('git_reference'):
+ ref = '--reference-if-able %s/%s.git' % (config.get('git_reference'), self.name)
+ else:
+ ref = ''
+ command('git clone %s %s %s/%s.git %s/src/%s' % (flags, ref, config.get('git_prefix'), self.name, target.directory, self.name))
os.chdir('%s/src/%s' % (target.directory, self.name))
spec = self.specifier
exec(open('%s/cscript' % proj).read(), self.cscript)
# cscript can include submodules = False to stop submodules being fetched
- if not 'submodules' in self.cscript or self.cscript['submodules'] == True:
- command('git submodule init --quiet')
- command('git submodule update --quiet')
+ if (not 'submodules' in self.cscript or self.cscript['submodules'] == True) and os.path.exists('.gitmodules'):
+ command('git submodule --quiet init')
+ paths = command_and_read('git config --file .gitmodules --get-regexp path')
+ urls = command_and_read('git config --file .gitmodules --get-regexp url')
+ for path, url in zip(paths, urls):
+ ref = ''
+ if config.has('git_reference'):
+ url = url.split(' ')[1]
+ ref_path = os.path.join(config.get('git_reference'), os.path.basename(url))
+ if os.path.exists(ref_path):
+ ref = '--reference %s' % ref_path
+ path = path.split(' ')[1]
+ command('git submodule --quiet update %s %s' % (ref, path))
if os.path.exists('%s/wscript' % proj):
v = read_wscript_variable(proj, "VERSION");
try:
self.version = Version(v)
except:
- tag = subprocess.Popen(shlex.split('git -C %s describe --tags' % proj), stdout=subprocess.PIPE).communicate()[0][1:]
+ tag = command_and_read('git -C %s describe --tags' % proj)[0][1:]
self.version = Version.from_git_tag(tag)
os.chdir(cwd)
def dependencies(self, options):
"""
yield details of the dependencies of this tree. Each dependency is returned
- as a tuple of (tree, options). The 'options' parameter are the options that
+ as a tuple of (tree, options, parent_tree). The 'options' parameter are the options that
we want to force for 'self'.
"""
if not 'dependencies' in self.cscript:
return
- if len(inspect.getargspec(self.cscript['dependencies']).args) == 2:
+ if len(inspect.getfullargspec(self.cscript['dependencies']).args) == 2:
self_options = copy.copy(options)
self.add_defaults(self_options)
deps = self.call('dependencies', self_options)
dep_options = d[2] if len(d) > 2 else {}
for i in dep.dependencies(dep_options):
yield i
- yield (dep, dep_options)
+ yield (dep, dep_options, self)
def checkout_dependencies(self, options={}):
for i in self.dependencies(options):
self.add_defaults(options)
if not globals.dry_run:
- if len(inspect.getargspec(self.cscript['build']).args) == 2:
+ if len(inspect.getfullargspec(self.cscript['build']).args) == 2:
self.call('build', options)
else:
self.call('build')
commands = {
"build": "build project",
- "package": "package and build project",
+ "package": "build and package project",
"release": "release a project using its next version number (changing wscript and tagging)",
"pot": "build the project's .pot files",
"manual": "build the project's manual",
"test": "run the project's unit tests",
"shell": "build the project then start a shell",
"checkout": "check out the project",
- "revision": "print the head git revision number"
+ "revision": "print the head git revision number",
+ "dependencies" : "print details of the project's dependencies as a .dot file"
}
one_of = "Command is one of:\n"
parser.add_argument('--option', help='set an option for the build (use --option key:value)', action='append')
parser.add_argument('--ccache', help='use ccache', action='store_true')
parser.add_argument('--verbose', help='be verbose', action='store_true')
+ parser.add_argument('--no-notarize', help='don\'t notarize .dmg packages', action='store_true')
global args
args = parser.parse_args()
output_dir = args.output
makedirs(output_dir)
- target.package(args.project, args.checkout, output_dir, get_command_line_options(args))
+ target.package(args.project, args.checkout, output_dir, get_command_line_options(args), args.no_notarize)
except Error as e:
if target is not None and not args.keep:
target.cleanup()
shutil.copytree('.', args.output)
target.cleanup()
+ elif globals.command == 'dependencies':
+ if args.target is None:
+ raise Error('you must specify -t or --target')
+ if args.checkout is None:
+ raise Error('you must specify -c or --checkout')
+
+ target = target_factory(args)
+ tree = globals.trees.get(args.project, args.checkout, target)
+ print("strict digraph {")
+ for d in list(tree.dependencies({})):
+ print("%s -> %s;" % (d[2].name.replace("-", "-"), d[0].name.replace("-", "_")))
+ print("}")
+
else:
raise Error('invalid command %s' % globals.command)