diff options
Diffstat (limited to 'bitbake/lib/bb/cookerdata.py')
-rw-r--r-- | bitbake/lib/bb/cookerdata.py | 264 |
1 files changed, 187 insertions, 77 deletions
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py index 144e803b4f..0649e40995 100644 --- a/bitbake/lib/bb/cookerdata.py +++ b/bitbake/lib/bb/cookerdata.py @@ -13,6 +13,7 @@ import logging import os import re import sys +import hashlib from functools import wraps import bb from bb import data @@ -22,8 +23,8 @@ logger = logging.getLogger("BitBake") parselog = logging.getLogger("BitBake.Parsing") class ConfigParameters(object): - def __init__(self, argv=sys.argv): - self.options, targets = self.parseCommandLine(argv) + def __init__(self, argv=None): + self.options, targets = self.parseCommandLine(argv or sys.argv) self.environment = self.parseEnvironment() self.options.pkgs_to_build = targets or [] @@ -56,12 +57,19 @@ class ConfigParameters(object): def updateToServer(self, server, environment): options = {} - for o in ["abort", "force", "invalidate_stamp", - "verbose", "debug", "dry_run", "dump_signatures", - "debug_domains", "extra_assume_provided", "profile", - "prefile", "postfile", "server_timeout"]: + for o in ["halt", "force", "invalidate_stamp", + "dry_run", "dump_signatures", + "extra_assume_provided", "profile", + "prefile", "postfile", "server_timeout", + "nosetscene", "setsceneonly", "skipsetscene", + "runall", "runonly", "writeeventlog"]: options[o] = getattr(self.options, o) + options['build_verbose_shell'] = self.options.verbose + options['build_verbose_stdout'] = self.options.verbose + options['default_loglevel'] = bb.msg.loggerDefaultLogLevel + options['debug_domains'] = bb.msg.loggerDefaultDomains + ret, error = server.runCommand(["updateConfig", options, environment, sys.argv]) if error: raise Exception("Unable to update the server configuration with local parameters: %s" % error) @@ -78,7 +86,7 @@ class ConfigParameters(object): action['msg'] = "Only one target can be used with the --environment option." elif self.options.buildfile and len(self.options.pkgs_to_build) > 0: action['msg'] = "No target should be used with the --environment and --buildfile options." - elif len(self.options.pkgs_to_build) > 0: + elif self.options.pkgs_to_build: action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build] else: action['action'] = ["showEnvironment", self.options.buildfile] @@ -110,13 +118,13 @@ class CookerConfiguration(object): """ def __init__(self): - self.debug_domains = [] + self.debug_domains = bb.msg.loggerDefaultDomains + self.default_loglevel = bb.msg.loggerDefaultLogLevel self.extra_assume_provided = [] self.prefile = [] self.postfile = [] - self.debug = 0 self.cmd = None - self.abort = True + self.halt = True self.force = False self.profile = False self.nosetscene = False @@ -124,34 +132,21 @@ class CookerConfiguration(object): self.skipsetscene = False self.invalidate_stamp = False self.dump_signatures = [] + self.build_verbose_shell = False + self.build_verbose_stdout = False self.dry_run = False self.tracking = False - self.xmlrpcinterface = [] - self.server_timeout = None self.writeeventlog = False - self.server_only = False self.limited_deps = False self.runall = [] self.runonly = [] self.env = {} - def setConfigParameters(self, parameters): - for key in self.__dict__.keys(): - if key in parameters.options.__dict__: - setattr(self, key, parameters.options.__dict__[key]) - self.env = parameters.environment.copy() - - def setServerRegIdleCallback(self, srcb): - self.server_register_idlecallback = srcb - def __getstate__(self): state = {} for key in self.__dict__.keys(): - if key == "server_register_idlecallback": - state[key] = None - else: - state[key] = getattr(self, key) + state[key] = getattr(self, key) return state def __setstate__(self,state): @@ -165,12 +160,7 @@ def catch_parse_error(func): def wrapped(fn, *args): try: return func(fn, *args) - except IOError as exc: - import traceback - parselog.critical(traceback.format_exc()) - parselog.critical("Unable to parse %s: %s" % (fn, exc)) - sys.exit(1) - except bb.data_smart.ExpansionError as exc: + except Exception as exc: import traceback bbdir = os.path.dirname(__file__) + os.sep @@ -181,15 +171,12 @@ def catch_parse_error(func): if not fn.startswith(bbdir): break parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb)) - sys.exit(1) - except bb.parse.ParseError as exc: - parselog.critical(str(exc)) - sys.exit(1) + raise bb.BBHandledException() return wrapped @catch_parse_error def parse_config_file(fn, data, include=True): - return bb.parse.handle(fn, data, include) + return bb.parse.handle(fn, data, include, baseconfig=True) @catch_parse_error def _inherit(bbclass, data): @@ -214,8 +201,8 @@ def findConfigFile(configfile, data): return None # -# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working -# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH. +# We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working +# up to /. If that fails, bitbake would fall back to cwd. # def findTopdir(): @@ -228,11 +215,8 @@ def findTopdir(): layerconf = findConfigFile("bblayers.conf", d) if layerconf: return os.path.dirname(os.path.dirname(layerconf)) - if bbpath: - bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf") - if bitbakeconf: - return os.path.dirname(os.path.dirname(bitbakeconf)) - return None + + return os.path.abspath(os.getcwd()) class CookerDataBuilder(object): @@ -255,10 +239,14 @@ class CookerDataBuilder(object): self.savedenv = bb.data.init() for k in cookercfg.env: self.savedenv.setVar(k, cookercfg.env[k]) + if k in bb.data_smart.bitbake_renamed_vars: + bb.error('Shell environment variable %s has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k])) + bb.fatal("Exiting to allow enviroment variables to be corrected") filtered_keys = bb.utils.approved_variables() bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys) self.basedata.setVar("BB_ORIGENV", self.savedenv) + self.basedata.setVar("__bbclasstype", "global") if worker: self.basedata.setVar("BB_WORKERCONTEXT", "1") @@ -266,14 +254,15 @@ class CookerDataBuilder(object): self.data = self.basedata self.mcdata = {} - def parseBaseConfiguration(self): + def parseBaseConfiguration(self, worker=False): + mcdata = {} + data_hash = hashlib.sha256() try: - bb.parse.init_parser(self.basedata) self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) - if self.data.getVar("BB_WORKERCONTEXT", False) is None: + if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker: bb.fetch.fetcher_init(self.data) - bb.codeparser.parser_cache_init(self.data) + bb.parse.init_parser(self.data) bb.event.fire(bb.event.ConfigParsed(), self.data) @@ -290,37 +279,63 @@ class CookerDataBuilder(object): bb.event.fire(bb.event.ConfigParsed(), self.data) bb.parse.init_parser(self.data) - self.data_hash = self.data.get_hash() - self.mcdata[''] = self.data + data_hash.update(self.data.get_hash().encode('utf-8')) + mcdata[''] = self.data multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split() for config in multiconfig: - mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) - bb.event.fire(bb.event.ConfigParsed(), mcdata) - self.mcdata[config] = mcdata + if config[0].isdigit(): + bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config) + parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config) + bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata) + mcdata[config] = parsed_mcdata + data_hash.update(parsed_mcdata.get_hash().encode('utf-8')) if multiconfig: - bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data) + bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data) - except (SyntaxError, bb.BBHandledException): - raise bb.BBHandledException + self.data_hash = data_hash.hexdigest() except bb.data_smart.ExpansionError as e: logger.error(str(e)) - raise bb.BBHandledException - except Exception: - logger.exception("Error parsing configuration files") - raise bb.BBHandledException + raise bb.BBHandledException() + + bb.codeparser.update_module_dependencies(self.data) + + # Handle obsolete variable names + d = self.data + renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {} + renamedvars.update(bb.data_smart.bitbake_renamed_vars) + issues = False + for v in renamedvars: + if d.getVar(v) != None or d.hasOverrides(v): + issues = True + loginfo = {} + history = d.varhistory.get_variable_refs(v) + for h in history: + for line in history[h]: + loginfo = {'file' : h, 'line' : line} + bb.data.data_smart._print_rename_error(v, loginfo, renamedvars) + if not history: + bb.data.data_smart._print_rename_error(v, loginfo, renamedvars) + if issues: + raise bb.BBHandledException() + + for mc in mcdata: + mcdata[mc].renameVar("__depends", "__base_depends") + mcdata[mc].setVar("__bbclasstype", "recipe") # Create a copy so we can reset at a later date when UIs disconnect - self.origdata = self.data - self.data = bb.data.createCopy(self.origdata) - self.mcdata[''] = self.data + self.mcorigdata = mcdata + for mc in mcdata: + self.mcdata[mc] = bb.data.createCopy(mcdata[mc]) + self.data = self.mcdata[''] def reset(self): # We may not have run parseBaseConfiguration() yet - if not hasattr(self, 'origdata'): + if not hasattr(self, 'mcorigdata'): return - self.data = bb.data.createCopy(self.origdata) - self.mcdata[''] = self.data + for mc in self.mcorigdata: + self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc]) + self.data = self.mcdata[''] def _findLayerConf(self, data): return findConfigFile("bblayers.conf", data) @@ -335,15 +350,23 @@ class CookerDataBuilder(object): layerconf = self._findLayerConf(data) if layerconf: - parselog.debug(2, "Found bblayers.conf (%s)", layerconf) + parselog.debug2("Found bblayers.conf (%s)", layerconf) # By definition bblayers.conf is in conf/ of TOPDIR. # We may have been called with cwd somewhere else so reset TOPDIR data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf))) data = parse_config_file(layerconf, data) + if not data.getVar("BB_CACHEDIR"): + data.setVar("BB_CACHEDIR", "${TOPDIR}/cache") + + bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR")) + layers = (data.getVar('BBLAYERS') or "").split() broken_layers = [] + if not layers: + bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition") + data = bb.data.createCopy(data) approved = bb.utils.approved_variables() @@ -357,10 +380,12 @@ class CookerDataBuilder(object): for layer in broken_layers: parselog.critical(" %s", layer) parselog.critical("Please check BBLAYERS in %s" % (layerconf)) - sys.exit(1) + raise bb.BBHandledException() + layerseries = None + compat_entries = {} for layer in layers: - parselog.debug(2, "Adding layer %s", layer) + parselog.debug2("Adding layer %s", layer) if 'HOME' in approved and '~' in layer: layer = os.path.expanduser(layer) if layer.endswith('/'): @@ -371,8 +396,27 @@ class CookerDataBuilder(object): data.expandVarref('LAYERDIR') data.expandVarref('LAYERDIR_RE') + # Sadly we can't have nice things. + # Some layers think they're going to be 'clever' and copy the values from + # another layer, e.g. using ${LAYERSERIES_COMPAT_core}. The whole point of + # this mechanism is to make it clear which releases a layer supports and + # show when a layer master branch is bitrotting and is unmaintained. + # We therefore avoid people doing this here. + collections = (data.getVar('BBFILE_COLLECTIONS') or "").split() + for c in collections: + compat_entry = data.getVar("LAYERSERIES_COMPAT_%s" % c) + if compat_entry: + compat_entries[c] = set(compat_entry.split()) + data.delVar("LAYERSERIES_COMPAT_%s" % c) + if not layerseries: + layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split()) + if layerseries: + data.delVar("LAYERSERIES_CORENAMES") + data.delVar('LAYERDIR_RE') data.delVar('LAYERDIR') + for c in compat_entries: + data.setVar("LAYERSERIES_COMPAT_%s" % c, " ".join(sorted(compat_entries[c]))) bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split() collections = (data.getVar('BBFILE_COLLECTIONS') or "").split() @@ -383,31 +427,46 @@ class CookerDataBuilder(object): invalid.append(entry) continue l, f = parts - if l in collections: + invert = l[0] == "!" + if invert: + l = l[1:] + if (l in collections and not invert) or (l not in collections and invert): data.appendVar("BBFILES", " " + f) if invalid: - bb.fatal("BBFILES_DYNAMIC entries must be of the form <collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) + bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n %s" % "\n ".join(invalid)) - layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split()) collections_tmp = collections[:] for c in collections: collections_tmp.remove(c) if c in collections_tmp: bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c) - compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split()) + + compat = set() + if c in compat_entries: + compat = compat_entries[c] + if compat and not layerseries: + bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c) if compat and not (compat & layerseries): bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)" % (c, " ".join(layerseries), " ".join(compat))) elif not compat and not data.getVar("BB_WORKERCONTEXT"): bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c)) + data.setVar("LAYERSERIES_CORENAMES", " ".join(sorted(layerseries))) + if not data.getVar("BBPATH"): msg = "The BBPATH variable is not set" if not layerconf: msg += (" and bitbake did not find a conf/bblayers.conf file in" " the expected location.\nMaybe you accidentally" " invoked bitbake from the wrong directory?") - raise SystemExit(msg) + bb.fatal(msg) + + if not data.getVar("TOPDIR"): + data.setVar("TOPDIR", os.path.abspath(os.getcwd())) + if not data.getVar("BB_CACHEDIR"): + data.setVar("BB_CACHEDIR", "${TOPDIR}/cache") + bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR")) data = parse_config_file(os.path.join("conf", "bitbake.conf"), data) @@ -420,17 +479,68 @@ class CookerDataBuilder(object): for bbclass in bbclasses: data = _inherit(bbclass, data) - # Nomally we only register event handlers at the end of parsing .bb files + # Normally we only register event handlers at the end of parsing .bb files # We register any handlers we've found so far here... for var in data.getVar('__BBHANDLERS', False) or []: handlerfn = data.getVarFlag(var, "filename", False) if not handlerfn: parselog.critical("Undefined event handler function '%s'" % var) - sys.exit(1) + raise bb.BBHandledException() handlerln = int(data.getVarFlag(var, "lineno", False)) - bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln) + bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask") or "").split(), handlerfn, handlerln, data) data.setVar('BBINCLUDED',bb.parse.get_file_depends(data)) return data + @staticmethod + def _parse_recipe(bb_data, bbfile, appends, mc, layername): + bb_data.setVar("__BBMULTICONFIG", mc) + bb_data.setVar("FILE_LAYERNAME", layername) + + bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) + bb.parse.cached_mtime_noerror(bbfile_loc) + + if appends: + bb_data.setVar('__BBAPPEND', " ".join(appends)) + + return bb.parse.handle(bbfile, bb_data) + + def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None): + """ + Load and parse one .bb build file + Return the data and whether parsing resulted in the file being skipped + """ + + if virtonly: + (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile) + bb_data = self.mcdata[mc].createCopy() + bb_data.setVar("__ONLYFINALISE", virtual or "default") + return self._parse_recipe(bb_data, bbfile, appends, mc, layername) + + if mc is not None: + bb_data = self.mcdata[mc].createCopy() + return self._parse_recipe(bb_data, bbfile, appends, mc, layername) + + bb_data = self.data.createCopy() + datastores = self._parse_recipe(bb_data, bbfile, appends, '', layername) + + for mc in self.mcdata: + if not mc: + continue + bb_data = self.mcdata[mc].createCopy() + newstores = self._parse_recipe(bb_data, bbfile, appends, mc, layername) + for ns in newstores: + datastores["mc:%s:%s" % (mc, ns)] = newstores[ns] + + return datastores + + def parseRecipe(self, virtualfn, appends, layername): + """ + Return a complete set of data for fn. + To do this, we need to parse the file. + """ + logger.debug("Parsing %s (full)" % virtualfn) + (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn) + datastores = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername) + return datastores[virtual] |