summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-02-06 10:53:54 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-15 14:54:34 +0000
commitd66a327fb6189db5de8bc489859235dcba306237 (patch)
tree2c800ad3856e916b25bf2431fff3894abb2ae4f7
parentc97697f442ef74666199b1bc460f6494f437cccc (diff)
downloadpoky-contrib-d66a327fb6189db5de8bc489859235dcba306237.tar.gz
poky-contrib-d66a327fb6189db5de8bc489859235dcba306237.tar.bz2
poky-contrib-d66a327fb6189db5de8bc489859235dcba306237.zip
Add shared make jobserver support
This is a WIP to add a make job server into bitbake. This means the pool of make tokens is central and shared by all tasks executed instead of the current one job pool per task. Currently we can end up with many more make subprocesses executing that is intended or optimal. Implementation wise, make usually uses a pipe for this functionality. Here we substitute a named pipe (fifo) and intercept the make commands, passing in file descriptors to the central fifo. This assumes knowledge of make's internal API, on the plus side it hasn't changed since 1999. TODO: * Remove hardcoded /tmp/makefifo and use something in TMPDIR or similar (alongside the lock file?) * Remove hardcoded make threads number and set from PARALLEL_MAKE * If PARALELL_MAKE = "", don't set MAKEARGS (currently parallelism is set everywhere) (need to check for -j in make commandline) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/bitbake-worker1
-rw-r--r--bitbake/lib/bb/runqueue.py24
-rw-r--r--meta/conf/bitbake.conf2
-rwxr-xr-xscripts/make-intercept/make31
4 files changed, 57 insertions, 1 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 3df783b2ad..5368b7903d 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -202,6 +202,7 @@ def fork_off_task(worker, cfg, data, workerdata, fn, task, taskname, appends, ta
data.setVar("BUILDNAME", workerdata["buildname"])
data.setVar("DATE", workerdata["date"])
data.setVar("TIME", workerdata["time"])
+ data.setVar("BB_MAKEFIFO", workerdata["makefifo"])
bb.parse.siggen.set_taskdata(workerdata["sigdata"])
ret = 0
try:
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 7aca42224a..00d838fffb 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -924,6 +924,7 @@ class RunQueue:
"buildname" : self.cfgData.getVar("BUILDNAME", True),
"date" : self.cfgData.getVar("DATE", True),
"time" : self.cfgData.getVar("TIME", True),
+ "makefifo" : self.cfgData.getVar("BB_MAKEFIFO", True),
}
worker.stdin.write("<cookerconfig>" + pickle.dumps(self.cooker.configuration) + "</cookerconfig>")
@@ -1050,6 +1051,27 @@ class RunQueue:
cache[task] = iscurrent
return iscurrent
+ def setup_make_fifo(self):
+ fifoname = "/tmp/makefifo"
+ self.cfgData.setVar("BB_MAKEFIFO", fifoname)
+ m = re.search(r'-j (\d+)', self.cfgData.getVar("PARALLEL_MAKE", True))
+ if m:
+ threads = int(m.group(1))
+ else:
+ threads = 1
+
+ if os.path.exists(fifoname):
+ os.remove(fifoname)
+ os.mkfifo(fifoname)
+
+ # Has to be open for read and writing
+ self.makereadfd = os.open(fifoname, os.O_RDONLY|os.O_NONBLOCK)
+ self.makewritefd = os.open(fifoname, os.O_WRONLY)
+ wfd = os.fdopen(self.makewritefd, 'w')
+
+ for x in range(0, threads):
+ wfd.write('+')
+
def _execute_runqueue(self):
"""
Run the tasks in a queue prepared by rqdata.prepare()
@@ -1071,6 +1093,8 @@ class RunQueue:
depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
+ self.setup_make_fifo()
+
if self.state is runQueueSceneInit:
dump = self.cooker.configuration.dump_signatures
if dump:
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 61d5bb5a89..41c5815110 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -424,7 +424,7 @@ EXTRA_IMAGEDEPENDS = ""
# Toolchain info.
##################################################################
-PATH_prepend = "${COREBASE}/scripts:${STAGING_BINDIR_TOOLCHAIN}:${STAGING_BINDIR_CROSS}:${STAGING_DIR_NATIVE}${sbindir_native}:${STAGING_BINDIR_NATIVE}:${STAGING_DIR_NATIVE}${base_sbindir_native}:${STAGING_DIR_NATIVE}${base_bindir_native}:"
+PATH_prepend = "${COREBASE}/scripts/make-intercept:${COREBASE}/scripts:${STAGING_BINDIR_TOOLCHAIN}:${STAGING_BINDIR_CROSS}:${STAGING_DIR_NATIVE}${sbindir_native}:${STAGING_BINDIR_NATIVE}:${STAGING_DIR_NATIVE}${base_sbindir_native}:${STAGING_DIR_NATIVE}${base_bindir_native}:"
export PATH
##################################################################
diff --git a/scripts/make-intercept/make b/scripts/make-intercept/make
new file mode 100755
index 0000000000..23cfb11b84
--- /dev/null
+++ b/scripts/make-intercept/make
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+import sys
+import os
+import subprocess
+
+if "BB_MAKEFIFO" in os.environ:
+ fifoname = os.environ["BB_MAKEFIFO"]
+
+ r = os.open(fifoname, os.O_RDONLY|os.O_NONBLOCK)
+ w = os.open(fifoname, os.O_WRONLY)
+ os.close(r)
+ r = os.open(fifoname, os.O_RDONLY)
+
+ seen = False
+ for i in sys.argv:
+ if "-j" in i:
+ seen = True
+ if seen:
+ os.environ["MAKEFLAGS"] = "-j --jobserver-fds=" + str(r) + "," + str(w)
+
+newpath = []
+origpath = os.environ["PATH"].split(":")
+for p in origpath:
+ if "make-intercept" in p:
+ continue
+ newpath.append(p)
+os.environ["PATH"] = ":".join(newpath)
+
+sys.argv[0] = "make"
+
+sys.exit(subprocess.call(sys.argv, shell=False))