diff options
Diffstat (limited to 'bitbake/lib/bb/parse')
| -rw-r--r-- | bitbake/lib/bb/parse/__init__.py | 22 | ||||
| -rw-r--r-- | bitbake/lib/bb/parse/ast.py | 73 | ||||
| -rw-r--r-- | bitbake/lib/bb/parse/parse_py/BBHandler.py | 26 | ||||
| -rw-r--r-- | bitbake/lib/bb/parse/parse_py/ConfHandler.py | 20 |
4 files changed, 120 insertions, 21 deletions
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py index 347609513b..7ffdaa6fd7 100644 --- a/bitbake/lib/bb/parse/__init__.py +++ b/bitbake/lib/bb/parse/__init__.py @@ -49,20 +49,32 @@ class SkipPackage(SkipRecipe): __mtime_cache = {} def cached_mtime(f): if f not in __mtime_cache: - __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + res = os.stat(f) + __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino) return __mtime_cache[f] def cached_mtime_noerror(f): if f not in __mtime_cache: try: - __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + res = os.stat(f) + __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino) except OSError: return 0 return __mtime_cache[f] +def check_mtime(f, mtime): + try: + res = os.stat(f) + current_mtime = (res.st_mtime_ns, res.st_size, res.st_ino) + __mtime_cache[f] = current_mtime + except OSError: + current_mtime = 0 + return current_mtime == mtime + def update_mtime(f): try: - __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] + res = os.stat(f) + __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino) except OSError: if f in __mtime_cache: del __mtime_cache[f] @@ -99,12 +111,12 @@ def supports(fn, data): return 1 return 0 -def handle(fn, data, include = 0): +def handle(fn, data, include=0, baseconfig=False): """Call the handler that is appropriate for this file""" for h in handlers: if h['supports'](fn, data): with data.inchistory.include(fn): - return h['handle'](fn, data, include) + return h['handle'](fn, data, include, baseconfig) raise ParseError("not a BitBake file", fn) def init(fn, data): diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py index 9e0a0f5c98..7581d003fd 100644 --- a/bitbake/lib/bb/parse/ast.py +++ b/bitbake/lib/bb/parse/ast.py @@ -9,6 +9,7 @@ # SPDX-License-Identifier: GPL-2.0-only # +import sys import bb from bb import methodpool from bb.parse import logger @@ -210,10 +211,12 @@ class ExportFuncsNode(AstNode): def eval(self, data): + sentinel = " # Export function set\n" for func in self.n: calledfunc = self.classname + "_" + func - if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False): + basevar = data.getVar(func, False) + if basevar and sentinel not in basevar: continue if data.getVar(func, False): @@ -230,12 +233,11 @@ class ExportFuncsNode(AstNode): data.setVarFlag(func, "lineno", 1) if data.getVarFlag(calledfunc, "python", False): - data.setVar(func, " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) + data.setVar(func, sentinel + " bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True) else: if "-" in self.classname: bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc)) - data.setVar(func, " " + calledfunc + "\n", parsing=True) - data.setVarFlag(func, 'export_func', '1') + data.setVar(func, sentinel + " " + calledfunc + "\n", parsing=True) class AddTaskNode(AstNode): def __init__(self, filename, lineno, func, before, after): @@ -269,6 +271,41 @@ class BBHandlerNode(AstNode): data.setVarFlag(h, "handler", 1) data.setVar('__BBHANDLERS', bbhands) +class PyLibNode(AstNode): + def __init__(self, filename, lineno, libdir, namespace): + AstNode.__init__(self, filename, lineno) + self.libdir = libdir + self.namespace = namespace + + def eval(self, data): + global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split() + for m in global_mods: + if m not in bb.utils._context: + bb.utils._context[m] = __import__(m) + + libdir = data.expand(self.libdir) + if libdir not in sys.path: + sys.path.append(libdir) + try: + bb.utils._context[self.namespace] = __import__(self.namespace) + toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) + for i in toimport: + bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) + mod = getattr(bb.utils._context[self.namespace], i) + fn = getattr(mod, "__file__") + funcs = {} + for f in dir(mod): + if f.startswith("_"): + continue + fcall = getattr(mod, f) + if not callable(fcall): + continue + funcs[f] = fcall + bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i)) + + except AttributeError as e: + bb.error("Error importing OE modules: %s" % str(e)) + class InheritNode(AstNode): def __init__(self, filename, lineno, classes): AstNode.__init__(self, filename, lineno) @@ -277,6 +314,16 @@ class InheritNode(AstNode): def eval(self, data): bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data) +class InheritDeferredNode(AstNode): + def __init__(self, filename, lineno, classes): + AstNode.__init__(self, filename, lineno) + self.inherit = (classes, filename, lineno) + + def eval(self, data): + inherits = data.getVar('__BBDEFINHERITS', False) or [] + inherits.append(self.inherit) + data.setVar('__BBDEFINHERITS', inherits) + def handleInclude(statements, filename, lineno, m, force): statements.append(IncludeNode(filename, lineno, m.group(1), force)) @@ -320,10 +367,17 @@ def handleDelTask(statements, filename, lineno, m): def handleBBHandlers(statements, filename, lineno, m): statements.append(BBHandlerNode(filename, lineno, m.group(1))) +def handlePyLib(statements, filename, lineno, m): + statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2))) + def handleInherit(statements, filename, lineno, m): classes = m.group(1) statements.append(InheritNode(filename, lineno, classes)) +def handleInheritDeferred(statements, filename, lineno, m): + classes = m.group(1) + statements.append(InheritDeferredNode(filename, lineno, classes)) + def runAnonFuncs(d): code = [] for funcname in d.getVar("__BBANONFUNCS", False) or []: @@ -361,6 +415,9 @@ def finalize(fn, d, variant = None): d.setVar('BBINCLUDED', bb.parse.get_file_depends(d)) + if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"): + bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.") + bb.event.fire(bb.event.RecipeParsed(fn), d) finally: bb.event.set_handlers(saved_handlers) @@ -387,6 +444,14 @@ def multi_finalize(fn, d): logger.debug("Appending .bbappend file %s to %s", append, fn) bb.parse.BBHandler.handle(append, d, True) + while True: + inherits = d.getVar('__BBDEFINHERITS', False) or [] + if not inherits: + break + inherit, filename, lineno = inherits.pop(0) + d.setVar('__BBDEFINHERITS', inherits) + bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True) + onlyfinalise = d.getVar("__ONLYFINALISE", False) safe_d = d diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py index 18e6868387..c13e4b9755 100644 --- a/bitbake/lib/bb/parse/parse_py/BBHandler.py +++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py @@ -21,6 +21,7 @@ from .ConfHandler import include, init __func_start_regexp__ = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" ) __inherit_regexp__ = re.compile(r"inherit\s+(.+)" ) +__inherit_def_regexp__ = re.compile(r"inherit_defer\s+(.+)" ) __export_func_regexp__ = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" ) __addtask_regexp__ = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*") __deltask_regexp__ = re.compile(r"deltask\s+(.+)") @@ -33,6 +34,7 @@ __infunc__ = [] __inpython__ = False __body__ = [] __classname__ = "" +__residue__ = [] cached_statements = {} @@ -40,8 +42,10 @@ def supports(fn, d): """Return True if fn has a supported extension""" return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"] -def inherit(files, fn, lineno, d): +def inherit(files, fn, lineno, d, deferred=False): __inherit_cache = d.getVar('__inherit_cache', False) or [] + #if "${" in files and not deferred: + # bb.warn("%s:%s has non deferred conditional inherit" % (fn, lineno)) files = d.expand(files).split() for file in files: classtype = d.getVar("__bbclasstype", False) @@ -77,7 +81,7 @@ def inherit(files, fn, lineno, d): __inherit_cache = d.getVar('__inherit_cache', False) or [] def get_statements(filename, absolute_filename, base_name): - global cached_statements + global cached_statements, __residue__, __body__ try: return cached_statements[absolute_filename] @@ -97,12 +101,17 @@ def get_statements(filename, absolute_filename, base_name): # add a blank line to close out any python definition feeder(lineno, "", filename, base_name, statements, eof=True) + if __residue__: + raise ParseError("Unparsed lines %s: %s" % (filename, str(__residue__)), filename, lineno) + if __body__: + raise ParseError("Unparsed lines from unclosed function %s: %s" % (filename, str(__body__)), filename, lineno) + if filename.endswith(".bbclass") or filename.endswith(".inc"): cached_statements[absolute_filename] = statements return statements -def handle(fn, d, include): - global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__, __classname__ +def handle(fn, d, include, baseconfig=False): + global __infunc__, __body__, __residue__, __classname__ __body__ = [] __infunc__ = [] __classname__ = "" @@ -154,7 +163,7 @@ def handle(fn, d, include): return d def feeder(lineno, s, fn, root, statements, eof=False): - global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__ + global __inpython__, __infunc__, __body__, __residue__, __classname__ # Check tabs in python functions: # - def py_funcname(): covered by __inpython__ @@ -265,7 +274,12 @@ def feeder(lineno, s, fn, root, statements, eof=False): ast.handleInherit(statements, fn, lineno, m) return - return ConfHandler.feeder(lineno, s, fn, statements) + m = __inherit_def_regexp__.match(s) + if m: + ast.handleInheritDeferred(statements, fn, lineno, m) + return + + return ConfHandler.feeder(lineno, s, fn, statements, conffile=False) # Add us to the handlers list from .. import handlers diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py index 451e68dd66..7826dee7d3 100644 --- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py +++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py @@ -21,7 +21,7 @@ __config_regexp__ = re.compile( r""" ^ (?P<exp>export\s+)? (?P<var>[a-zA-Z0-9\-_+.${}/~:]+?) - (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])? + (\[(?P<flag>[a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]*)\])? \s* ( (?P<colon>:=) | @@ -45,7 +45,8 @@ __include_regexp__ = re.compile( r"include\s+(.+)" ) __require_regexp__ = re.compile( r"require\s+(.+)" ) __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" ) __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" ) -__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" ) +__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]+)\]$" ) +__addpylib_regexp__ = re.compile(r"addpylib\s+(.+)\s+(.+)" ) def init(data): return @@ -102,12 +103,12 @@ def include_single_file(parentfn, fn, lineno, data, error_out): # We have an issue where a UI might want to enforce particular settings such as # an empty DISTRO variable. If configuration files do something like assigning # a weak default, it turns out to be very difficult to filter out these changes, -# particularly when the weak default might appear half way though parsing a chain +# particularly when the weak default might appear half way though parsing a chain # of configuration files. We therefore let the UIs hook into configuration file # parsing. This turns out to be a hard problem to solve any other way. confFilters = [] -def handle(fn, data, include): +def handle(fn, data, include, baseconfig=False): init(data) if include == 0: @@ -144,7 +145,7 @@ def handle(fn, data, include): # skip comments if s[0] == '#': continue - feeder(lineno, s, abs_fn, statements) + feeder(lineno, s, abs_fn, statements, baseconfig=baseconfig) # DONE WITH PARSING... time to evaluate data.setVar('FILE', abs_fn) @@ -157,7 +158,9 @@ def handle(fn, data, include): return data -def feeder(lineno, s, fn, statements): +# baseconfig is set for the bblayers/layer.conf cookerdata config parsing +# The function is also used by BBHandler, conffile would be False +def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True): m = __config_regexp__.match(s) if m: groupd = m.groupdict() @@ -189,6 +192,11 @@ def feeder(lineno, s, fn, statements): ast.handleUnsetFlag(statements, fn, lineno, m) return + m = __addpylib_regexp__.match(s) + if baseconfig and conffile and m: + ast.handlePyLib(statements, fn, lineno, m) + return + raise ParseError("unparsed line: '%s'" % s, fn, lineno); # Add us to the handlers list |