X-Git-Url: https://git.carlh.net/gitweb/?a=blobdiff_plain;f=cdist;h=3aa09a3c62fd2e55e4e9a276b55719ab879221f7;hb=960ce7582d6624c3858593a7511166302f8c923e;hp=85b3274ab2ebb37fddbe2f69dadf67fcc40142b7;hpb=27fd0790a9efdc6fc3dbc7aff09dea23c020cbda;p=cdist.git diff --git a/cdist b/cdist index 85b3274..3aa09a3 100755 --- a/cdist +++ b/cdist @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Carl Hetherington +# Copyright (C) 2012-2015 Carl Hetherington # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,6 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +from __future__ import print_function import os import sys import shutil @@ -28,22 +29,52 @@ import re import copy import inspect +TEMPORARY_DIRECTORY = '/var/tmp' + class Error(Exception): def __init__(self, value): self.value = value def __str__(self): - return '\x1b[31m%s\x1b[0m' % repr(self.value) + return self.value def __repr__(self): return str(self) +class Trees: + """ + Store for Tree objects which re-uses already-created objects + and checks for requests for different versions of the same thing. + """ + + def __init__(self): + self.trees = [] + + def get(self, name, specifier, target): + for t in self.trees: + if t.name == name and t.specifier == specifier and t.target == target: + return t + elif t.name == name and t.specifier != specifier: + raise Error('conflicting versions of %s requested (%s and %s)' % (name, specifier, t.specifier)) + + nt = Tree(name, specifier, target) + self.trees.append(nt) + return nt + +class Globals: + quiet = False + command = None + trees = Trees() + +globals = Globals() + + # # Configuration # class Option(object): - def __init__(self, key): + def __init__(self, key, default=None): self.key = key - self.value = None + self.value = default def offer(self, key, value): if key == self.key: @@ -61,18 +92,18 @@ class BoolOption(object): class Config: def __init__(self): self.options = [ Option('linux_chroot_prefix'), - BoolOption('chroot_host_mounted'), Option('windows_environment_prefix'), Option('mingw_prefix'), Option('git_prefix'), Option('osx_build_host'), Option('osx_environment_prefix'), Option('osx_sdk_prefix'), - Option('osx_sdk') ] + Option('osx_sdk'), + Option('parallel', 4) ] try: f = open('%s/.config/cdist' % os.path.expanduser('~'), 'r') - while 1: + while True: l = f.readline() if l == '': break @@ -90,27 +121,46 @@ class Config: def get(self, k): for o in self.options: if o.key == k: + if o.value is None: + raise Error('Required setting %s not found' % k) return o.value - raise Error('Required setting %s not found' % k) + def set(self, k, v): + for o in self.options: + o.offer(k, v) config = Config() # # Utility bits -# +# def log(m): - if not args.quiet: - print '\x1b[33m* %s\x1b[0m' % m + if not globals.quiet: + print('\x1b[33m* %s\x1b[0m' % m) + +def scp_escape(n): + s = n.split(':') + assert(len(s) == 1 or len(s) == 2) + if len(s) == 2: + return '%s:"\'%s\'"' % (s[0], s[1]) + else: + return '\"%s\"' % s[0] def copytree(a, b): - log('copy %s -> %s' % (a, b)) - shutil.copytree(a, b) + log('copy %s -> %s' % (scp_escape(b), scp_escape(b))) + command('scp -r %s %s' % (scp_escape(a), scp_escape(b))) def copyfile(a, b): - log('copy %s -> %s' % (a, b)) - shutil.copyfile(a, b) + log('copy %s -> %s' % (scp_escape(a), scp_escape(b))) + command('scp %s %s' % (scp_escape(a), scp_escape(b))) + +def makedirs(d): + if d.find(':') == -1: + os.makedirs(d) + else: + s = d.split(':') + command('ssh %s -- mkdir -p %s' % (s[0], s[1])) def rmdir(a): log('remove %s' % a) @@ -120,10 +170,10 @@ def rmtree(a): log('remove %s' % a) shutil.rmtree(a, ignore_errors=True) -def command(c, can_fail=False): +def command(c): log(c) r = os.system(c) - if (r >> 8) and not can_fail: + if (r >> 8): raise Error('command %s failed' % c) def command_and_read(c): @@ -134,11 +184,11 @@ def command_and_read(c): def read_wscript_variable(directory, variable): f = open('%s/wscript' % directory, 'r') - while 1: + while True: l = f.readline() if l == '': break - + s = l.split() if len(s) == 3 and s[0] == variable: f.close() @@ -147,9 +197,60 @@ def read_wscript_variable(directory, variable): f.close() return None -def remove_prefix(string, prefix): - assert(string.startswith(prefix)) - return string[len(prefix):] +def set_version_in_wscript(version): + f = open('wscript', 'rw') + o = open('wscript.tmp', 'w') + while True: + l = f.readline() + if l == '': + break + + s = l.split() + if len(s) == 3 and s[0] == "VERSION": + print("Writing %s" % version) + print("VERSION = '%s'" % version, file=o) + else: + print(l, file=o, end="") + f.close() + o.close() + + os.rename('wscript.tmp', 'wscript') + +def append_version_to_changelog(version): + try: + f = open('ChangeLog', 'r') + except: + log('Could not open ChangeLog') + return + + c = f.read() + f.close() + + f = open('ChangeLog', 'w') + now = datetime.datetime.now() + f.write('%d-%02d-%02d Carl Hetherington \n\n\t* Version %s released.\n\n' % (now.year, now.month, now.day, version)) + f.write(c) + +def append_version_to_debian_changelog(version): + if not os.path.exists('debian'): + log('Could not find debian directory') + return + + command('dch -b -v %s-1 "New upstream release."' % version) + +def devel_to_git(git_commit, filename): + if git_commit is not None: + filename = filename.replace('devel', '-%s' % git_commit) + return filename + +class TreeDirectory: + def __init__(self, tree): + self.tree = tree + def __enter__(self): + self.cwd = os.getcwd() + os.chdir('%s/src/%s' % (self.tree.target.directory, self.tree.name)) + def __exit__(self, type, value, traceback): + os.chdir(self.cwd) # # Version @@ -163,7 +264,7 @@ class Version: s = s[1:] if s.endswith("'"): s = s[0:-1] - + if s.endswith('devel'): s = s[0:-5] self.devel = True @@ -204,14 +305,18 @@ class Version: # class Target(object): - # @param directory directory to work in; if None we will use a temporary directory - # Temporary directories will be removed after use; specified directories will not - def __init__(self, platform, parallel, directory=None): + """ + platform -- platform string (e.g. 'windows', 'linux', 'osx') + directory -- directory to work in; if None we will use a temporary directory + Temporary directories will be removed after use; specified directories will not. + """ + def __init__(self, platform, directory=None): self.platform = platform - self.parallel = parallel + self.parallel = int(config.get('parallel')) + # self.directory is the working directory if directory is None: - self.directory = tempfile.mkdtemp('', 'tmp', self.temporary_directory) + self.directory = tempfile.mkdtemp('', 'tmp', TEMPORARY_DIRECTORY) self.rmdir = True else: self.directory = directory @@ -221,52 +326,19 @@ class Target(object): self.variables = {} self.debug = False - def build_dependencies(self, project): - cwd = os.getcwd() - if 'dependencies' in project.cscript: - print project.cscript['dependencies'](self) - for d in project.cscript['dependencies'](self): - log('Building dependency %s %s of %s' % (d[0], d[1], project.name)) - dep = Project(d[0], '.', d[1]) - dep.checkout(self) - self.build_dependencies(dep) - - # Make the options to pass in from the option_defaults of the thing - # we are building and any options specified by the parent. - options = {} - if 'option_defaults' in dep.cscript: - options = dep.cscript['option_defaults']() - if len(d) > 2: - for k, v in d[2].iteritems(): - options[k] = v - - self.build(dep, options) + def package(self, project, checkout): + tree = globals.trees.get(project, checkout, self) + tree.build_dependencies() + tree.build(tree) + return tree.call('package', tree.version), tree.git_commit - os.chdir(cwd) - - def build(self, project, options=None): - variables = copy.copy(self.variables) - print 'Target %s builds %s with %s' % (self.platform, project.name, self.variables) - if len(inspect.getargspec(project.cscript['build']).args) == 2: - project.cscript['build'](self, options) - else: - project.cscript['build'](self) - self.variables = variables - - def package(self, project): - project.checkout(self) - self.build_dependencies(project) - self.build(project) - return project.cscript['package'](self, project.version) - - def test(self, project): - project.checkout(self) - self.build_dependencies(project) - self.build(project) - project.cscript['test'](self) + def test(self, tree, test): + """test is the test case to run, or None""" + tree.build_dependencies() + tree.build() + return tree.call('test', test) def set(self, a, b): - print "Target set %s=%s" % (a, b) self.variables[a] = b def unset(self, a): @@ -276,14 +348,18 @@ class Target(object): return self.variables[a] def append_with_space(self, k, v): - if not k in self.variables: - self.variables[k] = v + if (not k in self.variables) or len(self.variables[k]) == 0: + self.variables[k] = '"%s"' % v else: - self.variables[k] = '%s %s' % (self.variables[k], v) + e = self.variables[k] + if e[0] == '"' and e[-1] == '"': + self.variables[k] = '"%s %s"' % (e[1:-1], v) + else: + self.variables[k] = '"%s %s"' % (e, v) def variables_string(self, escaped_quotes=False): e = '' - for k, v in self.variables.iteritems(): + for k, v in self.variables.items(): if escaped_quotes: v = v.replace('"', '\\"') e += '%s=%s ' % (k, v) @@ -293,96 +369,88 @@ class Target(object): if self.rmdir: rmtree(self.directory) -# +# # Windows # class WindowsTarget(Target): - def __init__(self, bits, directory=None): - super(WindowsTarget, self).__init__('windows', 2, directory) + def __init__(self, version, bits, directory=None): + super(WindowsTarget, self).__init__('windows', directory) + self.version = version self.bits = bits self.windows_prefix = '%s/%d' % (config.get('windows_environment_prefix'), self.bits) if not os.path.exists(self.windows_prefix): raise Error('windows prefix %s does not exist' % self.windows_prefix) - + if self.bits == 32: self.mingw_name = 'i686' else: self.mingw_name = 'x86_64' - mingw_path = '%s/%d/bin' % (config.get('mingw_prefix'), self.bits) + self.mingw_path = '%s/%d/bin' % (config.get('mingw_prefix'), self.bits) self.mingw_prefixes = ['/%s/%d' % (config.get('mingw_prefix'), self.bits), '%s/%d/%s-w64-mingw32' % (config.get('mingw_prefix'), bits, self.mingw_name)] self.set('PKG_CONFIG_LIBDIR', '%s/lib/pkgconfig' % self.windows_prefix) - self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.work_dir_cscript(), self.work_dir_cscript())) - self.set('PATH', '%s/bin:%s:%s' % (self.windows_prefix, mingw_path, os.environ['PATH'])) + self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/bin/pkgconfig' % (self.directory, self.directory)) + self.set('PATH', '%s/bin:%s:%s' % (self.windows_prefix, self.mingw_path, os.environ['PATH'])) self.set('CC', '%s-w64-mingw32-gcc' % self.mingw_name) self.set('CXX', '%s-w64-mingw32-g++' % self.mingw_name) self.set('LD', '%s-w64-mingw32-ld' % self.mingw_name) self.set('RANLIB', '%s-w64-mingw32-ranlib' % self.mingw_name) self.set('WINRC', '%s-w64-mingw32-windres' % self.mingw_name) - cxx = '-I%s/include -I%s/include' % (self.windows_prefix, self.work_dir_cscript()) - link = '-L%s/lib -L%s/lib' % (self.windows_prefix, self.work_dir_cscript()) + cxx = '-I%s/include -I%s/include' % (self.windows_prefix, self.directory) + link = '-L%s/lib -L%s/lib' % (self.windows_prefix, self.directory) for p in self.mingw_prefixes: cxx += ' -I%s/include' % p link += ' -L%s/lib' % p self.set('CXXFLAGS', '"%s"' % cxx) + self.set('CPPFLAGS', '') self.set('LINKFLAGS', '"%s"' % link) - - def work_dir_cdist(self): - return '%s/%d' % (self.directory, self.bits) - - def work_dir_cscript(self): - return '%s/%d' % (self.directory, self.bits) + self.set('LDFLAGS', '"%s"' % link) def command(self, c): log('host -> %s' % c) command('%s %s' % (self.variables_string(), c)) -# -# Linux -# - class LinuxTarget(Target): + """Parent for Linux targets""" def __init__(self, distro, version, bits, directory=None): - self.temporary_directory = '/tmp' - super(LinuxTarget, self).__init__('linux', 2, directory) + super(LinuxTarget, self).__init__('linux', directory) self.distro = distro self.version = version self.bits = bits + + self.set('CXXFLAGS', '-I%s/include' % self.directory) + self.set('CPPFLAGS', '') + self.set('LINKFLAGS', '-L%s/lib' % self.directory) + self.set('PKG_CONFIG_PATH', + '%s/lib/pkgconfig:%s/lib64/pkgconfig:/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig' % (self.directory, self.directory)) + self.set('PATH', '/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin') + +class ChrootTarget(LinuxTarget): + """Build in a chroot""" + def __init__(self, distro, version, bits, directory=None): + super(ChrootTarget, self).__init__(distro, version, bits, directory) # e.g. ubuntu-14.04-64 - self.chroot = '%s-%s-%d' % (self.distro, self.version, self.bits) + if self.version is not None and self.bits is not None: + self.chroot = '%s-%s-%d' % (self.distro, self.version, self.bits) + else: + self.chroot = self.distro # e.g. /home/carl/Environments/ubuntu-14.04-64 self.chroot_prefix = '%s/%s' % (config.get('linux_chroot_prefix'), self.chroot) - # e.g. /home/carl/Test/frobozz - if config.get('chroot_host_mounted'): - self.dir_in_chroot = self.directory - else: - self.dir_in_chroot = remove_prefix(self.directory, self.chroot_prefix) - self.set('CXXFLAGS', '-I%s/include' % self.work_dir_cscript()) - self.set('LINKFLAGS', '-L%s/lib' % self.work_dir_cscript()) - self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:/usr/local/lib/pkgconfig' % self.work_dir_cscript()) - self.set('PATH', '%s:/usr/local/bin' % (os.environ['PATH'])) + def command(self, c): + command('%s schroot -c %s -p -- %s' % (self.variables_string(), self.chroot, c)) - def work_dir_cdist(self): - if config.get('chroot_host_mounted'): - return self.work_dir_cscript() - else: - return '%s%s' % (self.chroot_prefix, self.dir_in_chroot) - def work_dir_cscript(self): - return self.dir_in_chroot +class HostTarget(LinuxTarget): + """Build directly on the host""" + def __init__(self, distro, version, bits, directory=None): + super(HostTarget, self).__init__(distro, version, bits, directory) def command(self, c): - if config.get('chroot_host_mounted'): - command('%s schroot -c %s -p -- %s' % (self.variables_string(), self.chroot, c)) - else: - # Work out the cwd for the chrooted command - cwd = remove_prefix(os.getcwd(), self.chroot_prefix) - log('schroot [%s] -> %s' % (cwd, c)) - command('%s schroot -c %s -d %s -p -- %s' % (self.variables_string(), self.chroot, cwd, c)) + command('%s %s' % (self.variables_string(), c)) # # OS X @@ -390,7 +458,9 @@ class LinuxTarget(Target): class OSXTarget(Target): def __init__(self, directory=None): - super(OSXTarget, self).__init__('osx', 4, directory) + super(OSXTarget, self).__init__('osx', directory) + self.sdk = config.get('osx_sdk') + self.sdk_prefix = config.get('osx_sdk_prefix') def command(self, c): command('%s %s' % (self.variables_string(False), c)) @@ -406,61 +476,43 @@ class OSXSingleTarget(OSXTarget): else: arch = 'x86_64' - flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (config.get('osx_sdk_prefix'), config.get('osx_sdk'), arch) + flags = '-isysroot %s/MacOSX%s.sdk -arch %s' % (self.sdk_prefix, self.sdk, arch) enviro = '%s/%d' % (config.get('osx_environment_prefix'), bits) # Environment variables - self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.work_dir_cscript(), enviro, flags)) - self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.work_dir_cscript(), enviro)) + self.set('CFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags)) + self.set('CPPFLAGS', '') + self.set('CXXFLAGS', '"-I%s/include -I%s/include %s"' % (self.directory, enviro, flags)) + self.set('LDFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags)) + self.set('LINKFLAGS', '"-L%s/lib -L%s/lib %s"' % (self.directory, enviro, flags)) + self.set('PKG_CONFIG_PATH', '%s/lib/pkgconfig:%s/lib/pkgconfig:/usr/lib/pkgconfig' % (self.directory, enviro)) self.set('PATH', '$PATH:/usr/bin:/sbin:/usr/local/bin:%s/bin' % enviro) self.set('MACOSX_DEPLOYMENT_TARGET', config.get('osx_sdk')) - def work_dir_cdist(self): - return self.work_dir_cscript() - - def work_dir_cscript(self): - return '%s/%d' % (self.directory, self.bits) - - def package(self, project): + def package(self, project, checkout): raise Error('cannot package non-universal OS X versions') class OSXUniversalTarget(OSXTarget): def __init__(self, directory=None): super(OSXUniversalTarget, self).__init__(directory) - self.parts = [] - self.parts.append(OSXSingleTarget(32, directory)) - self.parts.append(OSXSingleTarget(64, directory)) - - def work_dir_cscript(self): - return self.dir_in_host - def package(self, project): - for p in self.parts: - project.checkout(p) - p.build_dependencies(project) - p.build(project) + def package(self, project, checkout): - return project.cscript['package'](self, project.version) - + for b in [32, 64]: + target = OSXSingleTarget(b, os.path.join(self.directory, '%d' % b)) + tree = globals.trees.get(project, checkout, target) + tree.build_dependencies() + tree.build() -# -# Source -# + tree = globals.trees.get(project, checkout, self) + with TreeDirectory(tree): + return tree.call('package', tree.version), tree.git_commit class SourceTarget(Target): + """Build a source .tar.bz2""" def __init__(self): - super(SourceTarget, self).__init__('source', 2) - self.directory = tempfile.mkdtemp() - - def work_dir_cdist(self): - return self.directory - - def work_dir_cscript(self): - return self.directory + super(SourceTarget, self).__init__('source') def command(self, c): log('host -> %s' % c) @@ -469,11 +521,12 @@ class SourceTarget(Target): def cleanup(self): rmtree(self.directory) - def package(self, project): - project.checkout(self) - name = read_wscript_variable(os.getcwd(), 'APPNAME') - command('./waf dist') - return os.path.abspath('%s-%s.tar.bz2' % (name, project.version)) + def package(self, project, checkout): + tree = globals.trees.get(project, checkout, self) + with TreeDirectory(tree): + name = read_wscript_variable(os.getcwd(), 'APPNAME') + command('./waf dist') + return os.path.abspath('%s-%s.tar.bz2' % (name, tree.version)), tree.git_commit # @param s Target string: @@ -482,327 +535,445 @@ class SourceTarget(Target): # or debian-version-{32,64} # or centos-version-{32,64} # or osx-{32,64} -# or source +# or source # @param debug True to build with debugging symbols (where possible) def target_factory(s, debug, work): target = None if s.startswith('windows-'): - target = WindowsTarget(int(s.split('-')[1]), work) + x = s.split('-') + if len(x) == 2: + target = WindowsTarget(None, int(x[1]), work) + elif len(x) == 3: + target = WindowsTarget(x[1], int(x[2]), work) + else: + raise Error("Bad Windows target name `%s'") elif s.startswith('ubuntu-') or s.startswith('debian-') or s.startswith('centos-'): p = s.split('-') if len(p) != 3: - print >>sys.stderr,"Bad Linux target name `%s'; must be something like ubuntu-12.04-32 (i.e. distro-version-bits)" % s - sys.exit(1) - target = LinuxTarget(p[0], p[1], int(p[2]), work) + raise Error("Bad Linux target name `%s'; must be something like ubuntu-12.04-32 (i.e. distro-version-bits)" % s) + target = ChrootTarget(p[0], p[1], int(p[2]), work) + elif s.startswith('arch-'): + p = s.split('-') + if len(p) != 2: + raise Error("Bad Arch target name `%s'; must be arch-32 or arch-64") + target = ChrootTarget(p[0], None, p[1], work) + elif s == 'raspbian': + target = ChrootTarget(s, None, None, work) + elif s == 'host': + if command_and_read('uname -m').read().strip() == 'x86_64': + bits = 64 + else: + bits = 32 + try: + f = open('/etc/fedora-release', 'r') + l = f.readline().strip().split() + target = HostTarget("fedora", l[2], bits, work) + except Exception as e: + if os.path.exists('/etc/arch-release'): + target = HostTarget("arch", None, bits, work) + else: + raise Error("could not identify distribution for `host' target (%s)" % e) elif s.startswith('osx-'): target = OSXSingleTarget(int(s.split('-')[1]), work) elif s == 'osx': - if args.command == 'build': + if globals.command == 'build': target = OSXSingleTarget(64, work) else: target = OSXUniversalTarget(work) elif s == 'source': target = SourceTarget() - if target is not None: - target.debug = debug + if target is None: + raise Error("Bad target `%s'" % s) + target.debug = debug return target # -# Project +# Tree # - -class Project(object): - def __init__(self, name, directory, specifier=None): + +class Tree(object): + """Description of a tree, which is a checkout of a project, + possibly built. This class is never exposed to cscripts. + Attributes: + name -- name of git repository (without the .git) + specifier -- git tag or revision to use + target --- target object that we are using + version --- version from the wscript (if one is present) + git_commit -- git revision that is actually being used + built --- true if the tree has been built yet in this run + """ + + def __init__(self, name, specifier, target): self.name = name - self.directory = directory - self.version = None self.specifier = specifier - if self.specifier is None: - self.specifier = 'master' + self.target = target + self.version = None + self.git_commit = None + self.built = False + + cwd = os.getcwd() - def checkout(self, target): flags = '' redirect = '' - if args.quiet: + if globals.quiet: flags = '-q' redirect = '>/dev/null' - command('git clone %s %s/%s.git %s/src/%s' % (flags, config.get('git_prefix'), self.name, target.work_dir_cdist(), self.name)) - os.chdir('%s/src/%s' % (target.work_dir_cdist(), self.name)) - command('git checkout %s %s %s' % (flags, self.specifier, redirect)) + command('git clone %s %s/%s.git %s/src/%s' % (flags, config.get('git_prefix'), self.name, target.directory, self.name)) + os.chdir('%s/src/%s' % (target.directory, self.name)) + + spec = self.specifier + if spec is None: + spec = 'master' + + command('git checkout %s %s %s' % (flags, spec, redirect)) + self.git_commit = command_and_read('git rev-parse --short=7 HEAD').readline().strip() command('git submodule init --quiet') command('git submodule update --quiet') - os.chdir(self.directory) - proj = '%s/src/%s/%s' % (target.work_dir_cdist(), self.name, self.directory) + proj = '%s/src/%s' % (target.directory, self.name) + + self.cscript = {} + exec(open('%s/cscript' % proj).read(), self.cscript) - self.read_cscript('%s/cscript' % proj) - if os.path.exists('%s/wscript' % proj): v = read_wscript_variable(proj, "VERSION"); if v is not None: self.version = Version(v) - def read_cscript(self, s): - self.cscript = {} - execfile(s, self.cscript) + os.chdir(cwd) -def set_version_in_wscript(version): - f = open('wscript', 'rw') - o = open('wscript.tmp', 'w') - while 1: - l = f.readline() - if l == '': - break + def call(self, function, *args): + with TreeDirectory(self): + return self.cscript[function](self.target, *args) - s = l.split() - if len(s) == 3 and s[0] == "VERSION": - print "Writing %s" % version - print >>o,"VERSION = '%s'" % version - else: - print >>o,l, - f.close() - o.close() + def build_dependencies(self): + if 'dependencies' in self.cscript: + for d in self.cscript['dependencies'](self.target): + log('Building dependency %s %s of %s' % (d[0], d[1], self.name)) + dep = globals.trees.get(d[0], d[1], self.target) + dep.build_dependencies() - os.rename('wscript.tmp', 'wscript') + # Make the options to pass in from the option_defaults of the thing + # we are building and any options specified by the parent. + options = {} + if 'option_defaults' in dep.cscript: + options = dep.cscript['option_defaults']() + if len(d) > 2: + for k, v in d[2].items(): + options[k] = v -def append_version_to_changelog(version): - try: - f = open('ChangeLog', 'r') - except: - log('Could not open ChangeLog') - return + dep.build(options) - c = f.read() - f.close() + def build(self, options=None): + if self.built: + return - f = open('ChangeLog', 'w') - now = datetime.datetime.now() - f.write('%d-%02d-%02d Carl Hetherington \n\n\t* Version %s released.\n\n' % (now.year, now.month, now.day, version)) - f.write(c) + variables = copy.copy(self.target.variables) -def append_version_to_debian_changelog(version): - if not os.path.exists('debian'): - log('Could not find debian directory') - return + if len(inspect.getargspec(self.cscript['build']).args) == 2: + self.call('build', options) + else: + self.call('build') - command('dch -b -v %s-1 "New upstream release."' % version) + self.target.variables = variables + self.built = True # # Command-line parser # -parser = argparse.ArgumentParser() -parser.add_argument('command') -parser.add_argument('-p', '--project', help='project name') -parser.add_argument('-d', '--directory', help='directory within project repo', default='.') -parser.add_argument('--minor', help='minor version number bump', action='store_true') -parser.add_argument('--micro', help='micro version number bump', action='store_true') -parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') -parser.add_argument('-o', '--output', help='output directory', default='.') -parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') -parser.add_argument('-t', '--target', help='target') -parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') -parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') -parser.add_argument('-w', '--work', help='override default work directory') -args = parser.parse_args() - -args.output = os.path.abspath(args.output) -if args.work is not None: - args.work = os.path.abspath(args.work) - -if args.project is None and args.command != 'shell': - raise Error('you must specify -p or --project') - -project = Project(args.project, args.directory, args.checkout) - -if args.command == 'build': - if args.target is None: - raise Error('you must specify -t or --target') - - target = target_factory(args.target, args.debug, args.work) - project.checkout(target) - target.build_dependencies(project) - target.build(project) - if not args.keep: - target.cleanup() +def main(): + + commands = { + "build": "build project", + "package": "package and build project", + "release": "release a project using its next version number (changing wscript and tagging)", + "pot": "build the project's .pot files", + "changelog": "generate a simple HTML changelog", + "manual": "build the project's manual", + "doxygen": "build the project's Doxygen documentation", + "latest": "print out the latest version", + "test": "run the project's unit tests", + "shell": "build the project then start a shell in its chroot", + "checkout": "check out the project", + "revision": "print the head git revision number" + } + + one_of = "Command is one of:\n" + summary = "" + for k, v in commands.items(): + one_of += "\t%s\t%s\n" % (k, v) + summary += k + " " + + parser = argparse.ArgumentParser() + parser.add_argument('command', help=summary) + parser.add_argument('-p', '--project', help='project name') + parser.add_argument('--minor', help='minor version number bump', action='store_true') + parser.add_argument('--micro', help='micro version number bump', action='store_true') + parser.add_argument('--major', help='major version to return with latest', type=int) + parser.add_argument('-c', '--checkout', help='string to pass to git for checkout') + parser.add_argument('-o', '--output', help='output directory', default='.') + parser.add_argument('-q', '--quiet', help='be quiet', action='store_true') + parser.add_argument('-t', '--target', help='target') + parser.add_argument('-k', '--keep', help='keep working tree', action='store_true') + parser.add_argument('--debug', help='build with debugging symbols where possible', action='store_true') + parser.add_argument('-w', '--work', help='override default work directory') + parser.add_argument('-g', '--git-prefix', help='override configured git prefix') + parser.add_argument('--test', help='name of test to run (with `test''), defaults to all') + args = parser.parse_args() + + # Override configured stuff + if args.git_prefix is not None: + config.set('git_prefix', args.git_prefix) + + if args.output.find(':') == -1: + # This isn't of the form host:path so make it absolute + args.output = os.path.abspath(args.output) + '/' + else: + if args.output[-1] != ':' and args.output[-1] != '/': + args.output += '/' -elif args.command == 'package': - if args.target is None: - raise Error('you must specify -t or --target') - - target = target_factory(args.target, args.debug, args.work) + # Now, args.output is 'host:', 'host:path/' or 'path/' - packages = target.package(project) - if hasattr(packages, 'strip') or (not hasattr(packages, '__getitem__') and not hasattr(packages, '__iter__')): - packages = [packages] + if args.work is not None: + args.work = os.path.abspath(args.work) - if target.platform == 'linux': - out = '%s/%s-%s-%d' % (args.output, target.distro, target.version, target.bits) - try: - os.makedirs(out) - except: - pass - for p in packages: - copyfile(p, '%s/%s' % (out, os.path.basename(p))) - else: - for p in packages: - copyfile(p, '%s/%s' % (args.output, os.path.basename(p))) + if args.project is None and args.command != 'shell': + raise Error('you must specify -p or --project') - if not args.keep: - target.cleanup() + globals.quiet = args.quiet + globals.command = args.command -elif args.command == 'release': - if args.minor is False and args.micro is False: - raise Error('you must specify --minor or --micro') + if not globals.command in commands: + e = 'command must be one of:\n' + one_of + raise Error('command must be one of:\n%s' % one_of) - target = SourceTarget() - project.checkout(target) + if globals.command == 'build': + if args.target is None: + raise Error('you must specify -t or --target') - version = project.version - version.to_release() - if args.minor: - version.bump_minor() - else: - version.bump_micro() - - set_version_in_wscript(version) - append_version_to_changelog(version) - append_version_to_debian_changelog(version) - - command('git commit -a -m "Bump version"') - command('git tag -m "v%s" v%s' % (version, version)) - - version.to_devel() - set_version_in_wscript(version) - command('git commit -a -m "Bump version"') - command('git push') - command('git push --tags') - - target.cleanup() - -elif args.command == 'pot': - target = SourceTarget() - project.checkout(target) - - pots = project.cscript['make_pot'](target) - for p in pots: - copyfile(p, '%s/%s' % (args.output, os.path.basename(p))) - - target.cleanup() - -elif args.command == 'changelog': - target = SourceTarget() - project.checkout(target) - - text = open('ChangeLog', 'r') - html = open('%s/changelog.html' % args.output, 'w') - versions = 8 - - last = None - changes = [] - - while 1: - l = text.readline() - if l == '': - break - - if len(l) > 0 and l[0] == "\t": - s = l.split() - if len(s) == 4 and s[1] == "Version" and s[3] == "released.": - v = Version(s[2]) - if v.micro == 0: - if last is not None and len(changes) > 0: - print >>html,"

Changes between version %s and %s

" % (s[2], last) - print >>html,"
    " - for c in changes: - print >>html,"
  • %s" % c - print >>html,"
" - last = s[2] - changes = [] - versions -= 1 - if versions < 0: - break - else: - c = l.strip() - if len(c) > 0: - if c[0] == '*': - changes.append(c[2:]) - else: - changes[-1] += " " + c - - target.cleanup() - -elif args.command == 'manual': - target = SourceTarget() - project.checkout(target) - - outs = project.cscript['make_manual'](target) - for o in outs: - if os.path.isfile(o): - copyfile(o, '%s/%s' % (args.output, os.path.basename(o))) + target = target_factory(args.target, args.debug, args.work) + tree = globals.trees.get(args.project, args.checkout, target) + tree.build_dependencies() + tree.build() + if not args.keep: + target.cleanup() + + elif globals.command == 'package': + if args.target is None: + raise Error('you must specify -t or --target') + + target = target_factory(args.target, args.debug, args.work) + packages, git_commit = target.package(args.project, args.checkout) + if hasattr(packages, 'strip') or (not hasattr(packages, '__getitem__') and not hasattr(packages, '__iter__')): + packages = [packages] + + if target.platform == 'linux': + out = '%s%s-%s-%d' % (args.output, target.distro, target.version, target.bits) + try: + makedirs(out) + except: + pass + for p in packages: + copyfile(p, '%s/%s' % (out, os.path.basename(devel_to_git(git_commit, p)))) + else: + try: + makedirs(args.output) + except: + pass + for p in packages: + copyfile(p, '%s%s' % (args.output, os.path.basename(devel_to_git(git_commit, p)))) + + if not args.keep: + target.cleanup() + + elif globals.command == 'release': + if args.minor is False and args.micro is False: + raise Error('you must specify --minor or --micro') + + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + version = tree.version + version.to_release() + if args.minor: + version.bump_minor() else: - copytree(o, '%s/%s' % (args.output, os.path.basename(o))) + version.bump_micro() - target.cleanup() + set_version_in_wscript(version) + append_version_to_changelog(version) + append_version_to_debian_changelog(version) -elif args.command == 'doxygen': - target = SourceTarget() - project.checkout(target) + command('git commit -a -m "Bump version"') + command('git tag -m "v%s" v%s' % (version, version)) - dirs = project.cscript['make_doxygen'](target) - if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): - dirs = [dirs] + version.to_devel() + set_version_in_wscript(version) + command('git commit -a -m "Bump version"') + command('git push') + command('git push --tags') - for d in dirs: - copytree(d, '%s/%s' % (args.output, 'doc')) + target.cleanup() - target.cleanup() + elif globals.command == 'pot': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) -elif args.command == 'latest': - target = SourceTarget() - project.checkout(target) + pots = tree.call('make_pot') + for p in pots: + copyfile(p, '%s%s' % (args.output, os.path.basename(p))) - f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') - t = f.readline() - m = re.compile(".*\((.*)\).*").match(t) - latest = None - if m: - tags = m.group(1).split(', ') - for t in tags: - s = t.split() - if len(s) > 1: - t = s[1] - if len(t) > 0 and t[0] == 'v': - latest = t[1:] + target.cleanup() - print latest - target.cleanup() + elif globals.command == 'changelog': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + with TreeDirectory(tree): + text = open('ChangeLog', 'r') + + html = tempfile.NamedTemporaryFile() + versions = 8 + + last = None + changes = [] + + while True: + l = text.readline() + if l == '': + break + + if len(l) > 0 and l[0] == "\t": + s = l.split() + if len(s) == 4 and s[1] == "Version" and s[3] == "released.": + v = Version(s[2]) + if v.micro == 0: + if last is not None and len(changes) > 0: + print("

Changes between version %s and %s

" % (s[2], last), file=html) + print("
    ", file=html) + for c in changes: + print("
  • %s" % c, file=html) + print("
", file=html) + last = s[2] + changes = [] + versions -= 1 + if versions < 0: + break + else: + c = l.strip() + if len(c) > 0: + if c[0] == '*': + changes.append(c[2:]) + else: + changes[-1] += " " + c + + copyfile(html.file, '%schangelog.html' % args.output) + html.close() + target.cleanup() -elif args.command == 'test': - if args.target is None: - raise Error('you must specify -t or --target') + elif globals.command == 'manual': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + outs = tree.call('make_manual') + for o in outs: + if os.path.isfile(o): + copyfile(o, '%s%s' % (args.output, os.path.basename(o))) + else: + copytree(o, '%s%s' % (args.output, os.path.basename(o))) + + target.cleanup() + + elif globals.command == 'doxygen': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + dirs = tree.call('make_doxygen') + if hasattr(dirs, 'strip') or (not hasattr(dirs, '__getitem__') and not hasattr(dirs, '__iter__')): + dirs = [dirs] + + for d in dirs: + copytree(d, args.output) + + target.cleanup() + + elif globals.command == 'latest': + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + + with TreeDirectory(tree): + f = command_and_read('git log --tags --simplify-by-decoration --pretty="%d"') + latest = None + while latest is None: + t = f.readline() + m = re.compile(".*\((.*)\).*").match(t) + if m: + tags = m.group(1).split(', ') + for t in tags: + s = t.split() + if len(s) > 1: + t = s[1] + if len(t) > 0 and t[0] == 'v': + v = Version(t[1:]) + if args.major is None or v.major == args.major: + latest = v + + print(latest) + target.cleanup() + + elif globals.command == 'test': + if args.target is None: + raise Error('you must specify -t or --target') + + target = None + try: + target = target_factory(args.target, args.debug, args.work) + tree = globals.trees.get(args.project, args.checkout, target) + with TreeDirectory(tree): + target.test(tree, args.test) + except Error as e: + if target is not None: + target.cleanup() + raise - target = None - try: - target = target_factory(args.target, args.debug, args.work) - target.test(project) - except Error as e: if target is not None: target.cleanup() - raise - - if target is not None: + + elif globals.command == 'shell': + if args.target is None: + raise Error('you must specify -t or --target') + + target = target_factory(args.target, args.debug, args.work) + target.command('bash') + + elif globals.command == 'revision': + + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + with TreeDirectory(tree): + print(command_and_read('git rev-parse HEAD').readline().strip()[:7]) target.cleanup() -elif args.command == 'shell': - if args.target is None: - raise Error('you must specify -t or --target') + elif globals.command == 'checkout': + + if args.output is None: + raise Error('you must specify -o or --output') + + target = SourceTarget() + tree = globals.trees.get(args.project, args.checkout, target) + with TreeDirectory(tree): + shutil.copytree('.', args.output) + target.cleanup() - target = target_factory(args.target, args.debug, args.work) - target.command('bash') + else: + raise Error('invalid command %s' % globals.command) -else: - raise Error('invalid command %s' % args.command) +try: + main() +except Error as e: + print('cdist: %s' % str(e), file=sys.stderr) + sys.exit(1)