aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-17 16:18:25 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-18 13:59:06 +0000
commitc204cc47d0a319d0435c4a9ba06e241c2e75ff1e (patch)
tree335e80d6b956e0327156aa8eed3811498b735c07
parentfe857e4179355bcfb79303c16baf3ad87fca59a4 (diff)
downloadpoky-c204cc47d0a319d0435c4a9ba06e241c2e75ff1e.tar.gz
poky-c204cc47d0a319d0435c4a9ba06e241c2e75ff1e.tar.bz2
poky-c204cc47d0a319d0435c4a9ba06e241c2e75ff1e.zip
bitbake: cooker/siggen: Empty siggen cache during parsing
When parsing recipes its apparent the memory usage of bitbake rises linearly with number of recipes parsed. It shouldn't. Using tracemalloc (thanks for the tip Joshua Lock) it was clear that the dependency information left behind in siggen was the culprit. Add a new method to allow us to drop this information. We don't need it after the recipe has been parsed and hashes calculated (at runtime its different but only the currently executing task would be in memory). This should give signficant memory usage improvements for bitbake and that in turn should help speed on more constrained systems, as well as when used in multiconfig environments. (Bitbake rev: ef29309d0b512b64d024e383e7baff22c727711c) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py3
-rw-r--r--bitbake/lib/bb/siggen.py11
2 files changed, 13 insertions, 1 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 911805a6e3..d2d308ae20 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1944,7 +1944,8 @@ class Parser(multiprocessing.Process):
except IndexError:
break
result = self.parse(*job)
-
+ # Clear the siggen cache after parsing to control memory usage, its huge
+ bb.parse.siggen.postparsing_clean_cache()
try:
self.results.put(result, timeout=0.25)
except queue.Full:
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index ffd8fcaf36..c2d0c736cf 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -51,6 +51,9 @@ class SignatureGenerator(object):
def finalise(self, fn, d, varient):
return
+ def postparsing_clean_cache(self):
+ return
+
def get_unihash(self, tid):
return self.taskhash[tid]
@@ -188,6 +191,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
for task in taskdeps:
d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task])
+ def postparsing_clean_cache(self):
+ #
+ # After parsing we can remove some things from memory to reduce our memory footprint
+ #
+ self.gendeps = {}
+ self.lookupcache = {}
+ self.taskdeps = {}
+
def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
# Return True if we should keep the dependency, False to drop it
# We only manipulate the dependencies for packages not in the whitelist