diff options
Diffstat (limited to 'scripts/bb-determine-layers')
| -rwxr-xr-x | scripts/bb-determine-layers | 550 |
1 files changed, 0 insertions, 550 deletions
diff --git a/scripts/bb-determine-layers b/scripts/bb-determine-layers deleted file mode 100755 index 83583474..00000000 --- a/scripts/bb-determine-layers +++ /dev/null @@ -1,550 +0,0 @@ -#!/usr/bin/env python3 - -# --------------------------------------------------------------------------------------------------------------------- -# SPDX-License-Identifier: MIT -# --------------------------------------------------------------------------------------------------------------------- - -import argparse -import collections -import collections.abc -import contextlib -import glob -import os -import signal -import sys - - -glob_default = './*/:./*/*/:{scriptdir}/../../*/:{scriptdir}/../../*/*/' -layers_default = 'core' -override_default = '' - - -class Terminate(BaseException): - """Exception raised if we receive a SIGTERM""" - pass - - -def sigterm_exception(signum, stackframe): - """Raise Terminate exception if we receive a SIGTERM""" - raise Terminate() - - -def setup_command_import(command, relpath='../lib'): - """Set up sys.path based on the location of a binary in the PATH """ - PATH = os.environ['PATH'].split(':') - cmd_paths = [os.path.join(path, relpath) - for path in PATH if os.path.exists(os.path.join(path, command))] - if not cmd_paths: - raise ImportError("Unable to locate bb, please ensure PATH is set correctly.") - - sys.path[0:0] = cmd_paths - - -def find(dir, dirfilter=None, **walkoptions): - """ Given a directory, recurse into that directory, - returning all files as absolute paths. """ - - for root, dirs, files in os.walk(dir, **walkoptions): - if dirfilter is not None: - for d in dirs: - if not dirfilter(d): - dirs.remove(d) - - for file in files: - yield os.path.join(root, file) - - -class StatusDisplay(object): - """Show the user what we're doing, and whether we succeed""" - - def __init__(self, message, output=None): - self.message = message - if output is None: - output = sys.stderr - self.output = output - self.finished = False - - def __enter__(self): - self.output.write('{}..'.format(self.message)) - self.output.flush() - return self - - def __exit__(self, exc_type, exc_value, traceback): - if self.finished: - return - - if not exc_type: - self.set_status('done') - elif isinstance(exc_value, KeyboardInterrupt): - self.set_status('interrupted') - elif isinstance(exc_value, Terminate): - self.set_status('terminated') - else: - self.set_status('failed') - - def set_status(self, finishmessage): - if self.finished: - raise Exception("Status is already finished") - self.output.write('.' + finishmessage + '\n') - self.finished = True - - -status = StatusDisplay - - -class DictArgument(object): - def __init__(self, listsep=None, valuesep='=', keytype=None, valuetype=None): - self.listsep = listsep - self.valuesep = valuesep - self.keytype = keytype - self.valuetype = valuetype - - def __call__(self, string): - d = {} - for o in string.split(self.listsep): - try: - k, v = o.rsplit(self.valuesep, 1) - except ValueError: - raise argparse.ArgumentTypeError('invalid entry `{0}`'.format(o)) - - if self.keytype is not None: - try: - k = self.keytype(k) - except ValueError: - raise argparse.ArgumentTypeError('invalid key `{0}` for type `{1}`'.format(k, self.keytype.__name__)) - - if self.valuetype is not None: - try: - v = self.valuetype(v) - except ValueError: - raise argparse.ArgumentTypeError('invalid value `{0}` for type `{1}`'.format(v, self.valuetype.__name__)) - - d[k] = v - return d - - def __repr__(self): - return 'DictArgument(listsep={0}, valuesep={1}, keytype={2}, valuetype={3})'.format(repr(self.listsep), repr(self.valuesep), repr(self.keytype), repr(self.valuetype)) - - -def process_arguments(cmdline_args): - parser = argparse.ArgumentParser(description='Determine the bitbake layers to use for a given machine, with sanity checks') - parser.add_argument('-l', '--layers', default=layers_default, - help='base layers to include. space separated. default: `{0}`'.format(layers_default)) - parser.add_argument('-o', '--optional-layers', default='', - help='optional layers to include. space separated. These layers are included if they and their dependencies are available, but are silently left out otherwise.') - parser.add_argument('-e', '--extra-layers', default='', - help='extra layers to include. space separated. These layers are expected to exist, but are only included if their dependencies are already included in the configuration.') - parser.add_argument('-x', '--excluded-layers', default='', - help='explicit layers to excluded. space separated.') - parser.add_argument('-O', '--override-layer-priorities', default=override_default, - type=DictArgument(valuetype=int), - help='override the priority of one or more layers. space separated list of layername=priority values. default: `{0}`'.format(override_default)) - parser.add_argument('-g', '--globs', default=glob_default, - help='wildcard patterns to locate layers. colon separated. default: `{0}`'.format(glob_default)) - parser.add_argument('-p', '--paths', - help='paths to search recursively to find layers. colon separated. equivalent to the "find" command', default='') - parser.add_argument('-m', '--machine', help='add layer(s) providing the specified MACHINE') - parser.add_argument('-d', '--distro', help='add layer(s) providing the specified DISTRO') - parser.add_argument('-s', '--sdkmachine', help='add layer(s) providing the specified SDKMACHINE') - parser.add_argument('-v', '--verbose', help='increase verbosity', action='store_true') - - scriptdir = os.path.dirname(__file__) - args = parser.parse_args(cmdline_args) - - args.layers = args.layers.split() - args.optional_layers = args.optional_layers.split() - args.extra_layers = args.extra_layers.split() - args.excluded_layers = args.excluded_layers.split() - - new_globs = [] - for pattern in args.globs.split(':'): - pattern = os.path.abspath(pattern.format(scriptdir=scriptdir)) - new_globs.append(pattern) - args.globs = new_globs - - new_paths = [] - if args.paths: - for path in args.paths.split(':'): - path = os.path.abspath(path.format(scriptdir=scriptdir)) - new_paths.append(path) - args.paths = new_paths - - return args - - -class LayerError(Exception): - pass - - -class Layer(object): - def __init__(self, path, confpath, name, version, priority, pattern, depends, recommends): - self.path = os.path.realpath(path) - self.confpath = os.path.realpath(confpath) - self.name = name - self.version = version - self.priority = priority - self.depends = depends - self.recommends = recommends - self.missingdeps = set() - self.pattern = pattern - - def __repr__(self): - return '{0.__class__.__name__}({0.path!r}, {0.confpath!r},' \ - '{0.name!r}, {0.version!r}, {0.priority!r}, {0.pattern!r}, {0.depends!r}, {0.recommends!r})'.format(self) - - def __hash__(self): - return hash(repr(self)) - - @staticmethod - def from_layerpath(layer_path, data=None): - layers = [] - - if data is None: - data = bb.data.init() - bb.parse.init_parser(data) - - lconf = os.path.join(layer_path, 'conf', 'layer.conf') - ldata = data.createCopy() - ldata.setVar('LAYERDIR', layer_path) - try: - ldata = bb.parse.handle(lconf, ldata, include=True) - except BaseException as exc: - raise LayerError(exc) - ldata.expandVarref('LAYERDIR') - - bbfile_collections = (ldata.getVar('BBFILE_COLLECTIONS', True) or '').split() - if not bbfile_collections: - name = os.path.basename(layer_path) - l = Layer(layer_path, lconf, name, 0, 0, None, {}) - layers.append(l) - - for name in bbfile_collections: - pattern = ldata.getVar('BBFILE_PATTERN_%s' % name, True) - priority = ldata.getVar('BBFILE_PRIORITY_%s' % name, True) or 0 - depends = ldata.getVar('LAYERDEPENDS_%s' % name, True) or '' - version = ldata.getVar('LAYERVERSION_%s' % name, True) - depends = bb.utils.explode_dep_versions2(depends) - recommends = ldata.getVar('LAYERRECOMMENDS_%s' % name, True) or '' - recommends = bb.utils.explode_dep_versions2(recommends) - - l = Layer(layer_path, lconf, name, version, int(priority), pattern, depends, recommends) - layers.append(l) - - if layers: - return layers - - -class Layers(collections.abc.MutableSet): - def __init__(self, iterable=None): - """A collection of layers. Maintains lookup mappings by path and name""" - self.layers = set() - self.duplicates = collections.defaultdict(set) - self.by_path = {} - self.by_name = {} - - if iterable is not None: - self |= iterable - - def __contains__(self, layer): - return layer in self.layers - - def __iter__(self): - return iter(self.layers) - - def __len__(self): - return len(self.layers) - - def __repr__(self): - return '{0.__class__.__name__}({0.layers!r})'.format(self) - - def __hash__(self): - return hash(repr(self)) - - def add(self, layer): - if layer in self: - return - - if layer.path in self.by_path: - return - - if layer.name in self.by_name: - self.duplicates[layer.name].add(self.by_name[layer.name]) - self.duplicates[layer.name].add(layer) - return - - self.layers.add(layer) - self.by_path[layer.path] = layer - self.by_name[layer.name] = layer - - def add_from_path(self, layerpath, data=None): - layerpath = os.path.realpath(layerpath) - if layerpath in self.by_path: - return - - self |= Layer.from_layerpath(layerpath, data) - - def discard(self, layer): - self.layers.discard(layer) - del self.by_path[layer.path] - del self.by_name[layer.name] - - def clear(self): - self.layers.clear() - self.by_path.clear() - self.by_name.clear() - - def priority_sorted(self): - return sorted(self.layers, key=lambda l: (l.priority, l.name), reverse=True) - - def which(self, path, all=True): - layers = [] - for layer in self.priority_sorted(): - if os.path.exists(os.path.join(layer.path, path)): - layers.append(layer) - return layers - - def get_by_name_recursive(self, name, include_recs=True, indirect_recs=False): - """Get a layer by name and its dependencies, recursively. - - Returns a Layers() collection, and a list of missing dependencies.""" - - missing = set() - found_layers = Layers() - - if not include_recs: - indirect_recs = False - - if name in self.duplicates: - raise DuplicateLayers(name, self.duplicates[name]) - - try: - layer = self.by_name[name] - except KeyError: - missing.add(name) - else: - found_layers.add(layer) - for dep, oplist in layer.depends.items(): - try: - dep_found, dep_missing = self.get_by_name_recursive(dep, indirect_recs, indirect_recs) - except KeyError: - missing.add(dep) - continue - - for _dep in dep_found: - for opstr in oplist: - op, depver = opstr.split() - if _dep.version: - try: - res = bb.utils.vercmp_string_op(_dep.version, depver, op) - except bb.utils.VersionStringException as vse: - missing.add(dep) - else: - if not res: - missing.add(dep) - - if dep not in missing: - found_layers |= dep_found - missing |= dep_missing - - if include_recs: - for rec, oplist in layer.recommends.items(): - found = True - - try: - rec_found, _ = self.get_by_name_recursive(rec, indirect_recs, indirect_recs) - except KeyError: - continue - - for _rec in rec_found: - for opstr in oplist: - op, recver = opstr.split() - if _rec.version: - try: - res = bb.utils.vercmp_string_op(_rec.version, recver, op) - except bb.utils.VersionStringException as vse: - found = False - else: - if not res: - found = False - - if found and rec not in missing: - found_layers |= rec_found - - return found_layers, missing - - -class DuplicateLayers(Exception): - def __init__(self, name, layers): - self.name = name - self.layers = layers - - def __str__(self): - return 'Duplicate layers found for %s:\n%s' % (self.name, ''.join(' %s\n' % l.path for l in self.layers)) - - -class LayerStatus(StatusDisplay): - """Show status of work whose results are a set of layers.""" - - def __init__(self, message, verbose=False, output=None): - super(LayerStatus, self).__init__(message, output) - self.verbose = verbose - - def set_layer_status(self, layers): - layer_names = set(l.name for l in layers) - if self.verbose: - self.set_status('') - for f in sorted(l.path for l in layers): - self.output.write(' %s\n' % f) - else: - self.set_status(','.join(sorted(layer_names))) - - -def get_layerpaths_bitbake(globs=None, paths=None): - bitbake_path = None - layer_paths = set() - - for pattern in globs: - for glob_path in glob.glob(pattern): - layerconf_path = os.path.join(glob_path, 'conf', 'layer.conf') - if os.path.exists(layerconf_path): - layer_paths.add(os.path.realpath(glob_path)) - elif not bitbake_path and os.path.exists(os.path.join(glob_path, 'bin', 'bitbake')): - bitbake_path = os.path.join(glob_path, 'bin') - if not os.path.exists(os.path.join(bitbake_path, '..', 'lib', 'bb')): - bitbake_path = None - - for path in paths: - for subpath in find(path, dirfilter=lambda d: d != 'lib'): - if subpath.endswith('/conf/layer.conf'): - layer_paths.add(os.path.realpath(os.path.dirname(os.path.dirname(subpath)))) - elif not bitbake_path and subpath.endswith('/bin/bitbake'): - bitbake_path = os.path.dirname(subpath) - if not os.path.exists(os.path.join(bitbake_path, '..', 'lib', 'bb')): - bitbake_path = None - - return layer_paths, bitbake_path - - - -def get_layers_for(all_layers, value, varname, cfgpath, verbose=False): - with LayerStatus("Determining layers to include for {0} '{1}'".format(varname, value), verbose=verbose) as s: - layers = all_layers.which('conf/{0}/{1}.conf'.format(cfgpath, value)) - if not layers: - sys.exit("No layer found for {0} `{1}`".format(varname, value)) - - found_layers = Layers() - for name in set(l.name for l in layers): - layers, missing_layers = all_layers.get_by_name_recursive(name) - found_layers |= layers - if name in missing_layers: - sys.exit('Layer `{0}` was not found'.format(name)) - elif missing_layers: - sys.exit("Layer `{0}` was found, but its dependencies are missing: `{1}`".format(name, ', '.join(missing_layers))) - s.set_layer_status(found_layers) - return found_layers - - -def determine_layers(cmdline_opts): - args = process_arguments(cmdline_opts) - args.layers.append('core') - - with status('Locating layers and bitbake'): - layer_paths, bitbake_path = get_layerpaths_bitbake(args.globs, args.paths) - if bitbake_path is not None: - os.environ['PATH'] += ':' + bitbake_path - - setup_command_import('bitbake') - try: - import bb - except (ImportError, RuntimeError) as exc: - sys.exit("Unable to import 'bb' python package: %s" % exc) - - import bb.parse - import bb.data - - data = bb.data.init() - bb.parse.init_parser(data) - - all_layers = Layers() - with status('Parsing layer configuration files'): - for layer_path in layer_paths: - try: - all_layers.add_from_path(layer_path, data) - except LayerError as exc: - sys.stderr.write('Warning: error parsing layer at {}\n'.format(layer_path)) - - for layer in list(all_layers): - if layer.name in args.excluded_layers: - all_layers.discard(layer) - else: - priority_override = args.override_layer_priorities.get(layer.name) - if priority_override is not None: - layer.priority = priority_override - - configured_layers = Layers() - with LayerStatus('Checking for base layers', verbose=args.verbose) as s: - found = set() - for name in args.layers: - if name in all_layers.by_name: - layers, missing_layers = all_layers.get_by_name_recursive(name) - if missing_layers: - sys.exit("Layer `{0}` was found, but its dependencies are missing: `{1}`".format(name, ', '.join(missing_layers))) - else: - found.update(layers) - - s.set_layer_status(found) - configured_layers |= found - - if args.distro and args.distro != 'nodistro': - configured_layers |= get_layers_for(all_layers, args.distro, 'DISTRO', 'distro', args.verbose) - - if args.machine: - configured_layers |= get_layers_for(all_layers, args.machine, 'MACHINE', 'machine', args.verbose) - - if args.sdkmachine: - configured_layers |= get_layers_for(all_layers, args.sdkmachine, 'SDKMACHINE', 'machine-sdk', args.verbose) - - if args.optional_layers: - with LayerStatus('Checking for optional layers', verbose=args.verbose) as s: - found_optional = set() - for name in args.optional_layers: - if name in all_layers.by_name: - layers, missing_layers = all_layers.get_by_name_recursive(name) - if missing_layers: - sys.stderr.write("Warning: optional layer `{0}` was found, but its dependencies are missing: `{1}`".format(name, ', '.join(missing_layers))) - else: - found_optional.update(layers) - - found_optional.difference_update(configured_layers) - s.set_layer_status(found_optional) - configured_layers |= found_optional - - if args.extra_layers: - with LayerStatus('Checking for extra layers', verbose=args.verbose) as s: - found_extra = set() - for name in args.extra_layers: - if name in all_layers.by_name: - layers, missing_layers = all_layers.get_by_name_recursive(name) - if not missing_layers: - found_extra.update(layers) - - found_extra.difference_update(configured_layers) - s.set_layer_status(found_extra) - configured_layers |= found_extra - - for layer in configured_layers.priority_sorted(): - print(layer.path) - - -if __name__ == '__main__': - signal.signal(signal.SIGTERM, sigterm_exception) - try: - sys.exit(determine_layers(sys.argv[1:]) or 0) - except DuplicateLayers as exc: - sys.exit(str(exc)) - except KeyboardInterrupt: - signal.signal(signal.SIGINT, signal.SIG_DFL) - os.kill(os.getpid(), signal.SIGINT) - except Terminate: - signal.signal(signal.SIGTERM, signal.SIG_DFL) - os.kill(os.getpid(), signal.SIGTERM) |