diff options
Diffstat (limited to 'bitbake/lib/bb/data_smart.py')
-rw-r--r-- | bitbake/lib/bb/data_smart.py | 149 |
1 files changed, 111 insertions, 38 deletions
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 8d235da121..0128a5bb17 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py @@ -16,7 +16,10 @@ BitBake build tools. # # Based on functions from the base bb module, Copyright 2003 Holger Schurig -import copy, re, sys, traceback +import builtins +import copy +import re +import sys from collections.abc import MutableMapping import logging import hashlib @@ -29,10 +32,22 @@ logger = logging.getLogger("BitBake.Data") __setvar_keyword__ = [":append", ":prepend", ":remove"] __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") -__expand_python_regexp__ = re.compile(r"\${@.+?}") +__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") __whitespace_split__ = re.compile(r'(\s)') __override_regexp__ = re.compile(r'[a-z0-9]+') +bitbake_renamed_vars = { + "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", + "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", + "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", + "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", + "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", + "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", + "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", + "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", + "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", +} + def infer_caller_details(loginfo, parent = False, varval = True): """Save the caller the trouble of specifying everything.""" # Save effort. @@ -80,10 +95,11 @@ def infer_caller_details(loginfo, parent = False, varval = True): loginfo['func'] = func class VariableParse: - def __init__(self, varname, d, val = None): + def __init__(self, varname, d, unexpanded_value = None, val = None): self.varname = varname self.d = d self.value = val + self.unexpanded_value = unexpanded_value self.references = set() self.execs = set() @@ -107,6 +123,11 @@ class VariableParse: else: code = match.group()[3:-1] + # Do not run code that contains one or more unexpanded variables + # instead return the code with the characters we removed put back + if __expand_var_regexp__.findall(code): + return "${@" + code + "}" + if self.varname: varname = 'Var <%s>' % self.varname else: @@ -132,16 +153,21 @@ class VariableParse: value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) return str(value) - class DataContext(dict): + excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) + def __init__(self, metadata, **kwargs): self.metadata = metadata dict.__init__(self, **kwargs) self['d'] = metadata + self.context = set(bb.utils.get_context()) def __missing__(self, key): + if key in self.excluded or key in self.context: + raise KeyError(key) + value = self.metadata.getVar(key) - if value is None or self.metadata.getVarFlag(key, 'func', False): + if value is None: raise KeyError(key) else: return value @@ -336,6 +362,16 @@ class VariableHistory(object): lines.append(line) return lines + def get_variable_refs(self, var): + """Return a dict of file/line references""" + var_history = self.variable(var) + refs = {} + for event in var_history: + if event['file'] not in refs: + refs[event['file']] = [] + refs[event['file']].append(event['line']) + return refs + def get_variable_items_files(self, var): """ Use variable history to map items added to a list variable and @@ -370,6 +406,23 @@ class VariableHistory(object): else: self.variables[var] = [] +def _print_rename_error(var, loginfo, renamedvars, fullvar=None): + info = "" + if "file" in loginfo: + info = " file: %s" % loginfo["file"] + if "line" in loginfo: + info += " line: %s" % loginfo["line"] + if fullvar and fullvar != var: + info += " referenced as: %s" % fullvar + if info: + info = " (%s)" % info.strip() + renameinfo = renamedvars[var] + if " " in renameinfo: + # A space signals a string to display instead of a rename + bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) + else: + bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) + class DataSmart(MutableMapping): def __init__(self): self.dict = {} @@ -377,6 +430,8 @@ class DataSmart(MutableMapping): self.inchistory = IncludeHistory() self.varhistory = VariableHistory(self) self._tracking = False + self._var_renames = {} + self._var_renames.update(bitbake_renamed_vars) self.expand_cache = {} @@ -398,9 +453,9 @@ class DataSmart(MutableMapping): def expandWithRefs(self, s, varname): if not isinstance(s, str): # sanity check - return VariableParse(varname, self, s) + return VariableParse(varname, self, s, s) - varparse = VariableParse(varname, self) + varparse = VariableParse(varname, self, s) while s.find('${') != -1: olds = s @@ -432,24 +487,19 @@ class DataSmart(MutableMapping): def expand(self, s, varname = None): return self.expandWithRefs(s, varname).value - def finalize(self, parent = False): - return - - def internal_finalize(self, parent = False): - """Performs final steps upon the datastore, including application of overrides""" - self.overrides = None - def need_overrides(self): if self.overrides is not None: return if self.inoverride: return + overrride_stack = [] for count in range(5): self.inoverride = True # Can end up here recursively so setup dummy values self.overrides = [] self.overridesset = set() self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] + overrride_stack.append(self.overrides) self.overridesset = set(self.overrides) self.inoverride = False self.expand_cache = {} @@ -459,7 +509,7 @@ class DataSmart(MutableMapping): self.overrides = newoverrides self.overridesset = set(self.overrides) else: - bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") + bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) def initVar(self, var): self.expand_cache = {} @@ -470,36 +520,44 @@ class DataSmart(MutableMapping): dest = self.dict while dest: if var in dest: - return dest[var], self.overridedata.get(var, None) + return dest[var] if "_data" not in dest: break dest = dest["_data"] - return None, self.overridedata.get(var, None) + return None def _makeShadowCopy(self, var): if var in self.dict: return - local_var, _ = self._findVar(var) + local_var = self._findVar(var) if local_var: self.dict[var] = copy.copy(local_var) else: self.initVar(var) + def hasOverrides(self, var): + return var in self.overridedata def setVar(self, var, value, **loginfo): #print("var=" + str(var) + " val=" + str(value)) - if "_append" in var or "_prepend" in var or "_remove" in var: + if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): info = "%s" % var - if "filename" in loginfo: - info += " file: %s" % loginfo[filename] - if "lineno" in loginfo: - info += " line: %s" % loginfo[lineno] + if "file" in loginfo: + info += " file: %s" % loginfo["file"] + if "line" in loginfo: + info += " line: %s" % loginfo["line"] bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) + shortvar = var.split(":", 1)[0] + if shortvar in self._var_renames: + _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) + # Mark that we have seen a renamed variable + self.setVar("_FAILPARSINGERRORHANDLED", True) + self.expand_cache = {} parsing=False if 'parsing' in loginfo: @@ -581,7 +639,7 @@ class DataSmart(MutableMapping): nextnew.update(vardata.references) nextnew.update(vardata.contains.keys()) new = nextnew - self.internal_finalize(True) + self.overrides = None def _setvar_update_overrides(self, var, **loginfo): # aka pay the cookie monster @@ -621,10 +679,11 @@ class DataSmart(MutableMapping): self.varhistory.record(**loginfo) self.setVar(newkey, val, ignore=True, parsing=True) - for i in (__setvar_keyword__): - src = self.getVarFlag(key, i, False) - if src is None: + srcflags = self.getVarFlags(key, False, True) or {} + for i in srcflags: + if i not in (__setvar_keyword__): continue + src = srcflags[i] dest = self.getVarFlag(newkey, i, False) or [] dest.extend(src) @@ -667,7 +726,7 @@ class DataSmart(MutableMapping): if ':' in var: override = var[var.rfind(':')+1:] shortvar = var[:var.rfind(':')] - while override and override.islower(): + while override and __override_regexp__.match(override): try: if shortvar in self.overridedata: # Force CoW by recreating the list first @@ -685,6 +744,14 @@ class DataSmart(MutableMapping): def setVarFlag(self, var, flag, value, **loginfo): self.expand_cache = {} + if var == "BB_RENAMED_VARIABLES": + self._var_renames[flag] = value + + if var in self._var_renames: + _print_rename_error(var, loginfo, self._var_renames) + # Mark that we have seen a renamed variable + self.setVar("_FAILPARSINGERRORHANDLED", True) + if 'op' not in loginfo: loginfo['op'] = "set" loginfo['flag'] = flag @@ -714,13 +781,18 @@ class DataSmart(MutableMapping): return None cachename = var + "[" + flag + "]" + if not expand and retparser and cachename in self.expand_cache: + return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] + if expand and cachename in self.expand_cache: return self.expand_cache[cachename].value - local_var, overridedata = self._findVar(var) + local_var = self._findVar(var) value = None removes = set() - if flag == "_content" and overridedata is not None and not parsing: + if flag == "_content" and not parsing: + overridedata = self.overridedata.get(var, None) + if flag == "_content" and not parsing and overridedata is not None: match = False active = {} self.need_overrides() @@ -810,7 +882,7 @@ class DataSmart(MutableMapping): expanded_removes[r] = self.expand(r).split() parser.removes = set() - val = "" + val = [] for v in __whitespace_split__.split(parser.value): skip = False for r in removes: @@ -819,8 +891,8 @@ class DataSmart(MutableMapping): skip = True if skip: continue - val = val + v - parser.value = val + val.append(v) + parser.value = "".join(val) if expand: value = parser.value @@ -835,7 +907,7 @@ class DataSmart(MutableMapping): def delVarFlag(self, var, flag, **loginfo): self.expand_cache = {} - local_var, _ = self._findVar(var) + local_var = self._findVar(var) if not local_var: return if not var in self.dict: @@ -878,7 +950,7 @@ class DataSmart(MutableMapping): self.dict[var][i] = flags[i] def getVarFlags(self, var, expand = False, internalflags=False): - local_var, _ = self._findVar(var) + local_var = self._findVar(var) flags = {} if local_var: @@ -924,6 +996,7 @@ class DataSmart(MutableMapping): data.inchistory = self.inchistory.copy() data._tracking = self._tracking + data._var_renames = self._var_renames data.overrides = None data.overridevars = copy.copy(self.overridevars) @@ -946,7 +1019,7 @@ class DataSmart(MutableMapping): value = self.getVar(variable, False) for key in keys: referrervalue = self.getVar(key, False) - if referrervalue and ref in referrervalue: + if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: self.setVar(key, referrervalue.replace(ref, value)) def localkeys(self): @@ -1012,10 +1085,10 @@ class DataSmart(MutableMapping): d = self.createCopy() bb.data.expandKeys(d) - config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split()) + config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) keys = set(key for key in iter(d) if not key.startswith("__")) for key in keys: - if key in config_whitelist: + if key in config_ignore_vars: continue value = d.getVar(key, False) or "" |