diff options
Diffstat (limited to 'bitbake/lib/bb/codeparser.py')
| -rw-r--r-- | bitbake/lib/bb/codeparser.py | 82 |
1 files changed, 61 insertions, 21 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py index 9d66d3ae41..2e8b7ced3c 100644 --- a/bitbake/lib/bb/codeparser.py +++ b/bitbake/lib/bb/codeparser.py @@ -27,6 +27,7 @@ import ast import sys import codegen import logging +import inspect import bb.pysh as pysh import bb.utils, bb.data import hashlib @@ -58,10 +59,40 @@ def check_indent(codestr): return codestr -# A custom getstate/setstate using tuples is actually worth 15% cachesize by -# avoiding duplication of the attribute names! +modulecode_deps = {} +def add_module_functions(fn, functions, namespace): + import os + fstat = os.stat(fn) + fixedhash = fn + ":" + str(fstat.st_size) + ":" + str(fstat.st_mtime) + for f in functions: + name = "%s.%s" % (namespace, f) + parser = PythonParser(name, logger) + try: + parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f) + #bb.warn("Cached %s" % f) + except KeyError: + lines, lineno = inspect.getsourcelines(functions[f]) + src = "".join(lines) + parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f) + #bb.warn("Not cached %s" % f) + execs = parser.execs.copy() + # Expand internal module exec references + for e in parser.execs: + if e in functions: + execs.remove(e) + execs.add(namespace + "." + e) + modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()] + #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, fn, parser.references, parser.execs, parser.var_execs, parser.contains)) + +def update_module_dependencies(d): + for mod in modulecode_deps: + excludes = set((d.getVarFlag(mod, "vardepsexclude") or "").split()) + if excludes: + modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3]] +# A custom getstate/setstate using tuples is actually worth 15% cachesize by +# avoiding duplication of the attribute names! class SetCache(object): def __init__(self): self.setcache = {} @@ -154,12 +185,12 @@ class CodeParserCache(MultiProcessCache): self.shellcachelines[h] = cacheline return cacheline - def init_cache(self, d): + def init_cache(self, cachedir): # Check if we already have the caches if self.pythoncache: return - MultiProcessCache.init_cache(self, d) + MultiProcessCache.init_cache(self, cachedir) # cachedata gets re-assigned in the parent self.pythoncache = self.cachedata[0] @@ -171,8 +202,8 @@ class CodeParserCache(MultiProcessCache): codeparsercache = CodeParserCache() -def parser_cache_init(d): - codeparsercache.init_cache(d) +def parser_cache_init(cachedir): + codeparsercache.init_cache(cachedir) def parser_cache_save(): codeparsercache.save_extras() @@ -225,19 +256,19 @@ class PythonParser(): def visit_Call(self, node): name = self.called_node_name(node.func) if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs): - if isinstance(node.args[0], ast.Str): - varname = node.args[0].s - if name in self.containsfuncs and isinstance(node.args[1], ast.Str): + if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): + varname = node.args[0].value + if name in self.containsfuncs and isinstance(node.args[1], ast.Constant): if varname not in self.contains: self.contains[varname] = set() - self.contains[varname].add(node.args[1].s) - elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str): + self.contains[varname].add(node.args[1].value) + elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Constant): if varname not in self.contains: self.contains[varname] = set() - self.contains[varname].update(node.args[1].s.split()) + self.contains[varname].update(node.args[1].value.split()) elif name.endswith(self.getvarflags): - if isinstance(node.args[1], ast.Str): - self.references.add('%s[%s]' % (varname, node.args[1].s)) + if isinstance(node.args[1], ast.Constant): + self.references.add('%s[%s]' % (varname, node.args[1].value)) else: self.warn(node.func, node.args[1]) else: @@ -245,8 +276,8 @@ class PythonParser(): else: self.warn(node.func, node.args[0]) elif name and name.endswith(".expand"): - if isinstance(node.args[0], ast.Str): - value = node.args[0].s + if isinstance(node.args[0], ast.Constant): + value = node.args[0].value d = bb.data.init() parser = d.expandWithRefs(value, self.name) self.references |= parser.references @@ -256,8 +287,8 @@ class PythonParser(): self.contains[varname] = set() self.contains[varname] |= parser.contains[varname] elif name in self.execfuncs: - if isinstance(node.args[0], ast.Str): - self.var_execs.add(node.args[0].s) + if isinstance(node.args[0], ast.Constant): + self.var_execs.add(node.args[0].value) else: self.warn(node.func, node.args[0]) elif name and isinstance(node.func, (ast.Name, ast.Attribute)): @@ -289,11 +320,17 @@ class PythonParser(): self.unhandled_message = "in call of %s, argument '%s' is not a string literal" self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) - def parse_python(self, node, lineno=0, filename="<string>"): - if not node or not node.strip(): + # For the python module code it is expensive to have the function text so it is + # uses a different fixedhash to cache against. We can take the hit on obtaining the + # text if it isn't in the cache. + def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None): + if not fixedhash and (not node or not node.strip()): return - h = bbhash(str(node)) + if fixedhash: + h = fixedhash + else: + h = bbhash(str(node)) if h in codeparsercache.pythoncache: self.references = set(codeparsercache.pythoncache[h].refs) @@ -311,6 +348,9 @@ class PythonParser(): self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) return + if fixedhash and not node: + raise KeyError + # Need to parse so take the hit on the real log buffer self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log) |