diff options
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 99 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/runqueue.py | 89 |
2 files changed, 55 insertions, 133 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index d9a67a3167..02160ef4d7 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -73,7 +73,7 @@ def build_tid(mc, fn, taskname): def pending_hash_index(tid, rqdata): (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) pn = rqdata.dataCaches[mc].pkg_fn[taskfn] - h = rqdata.runtaskentries[tid].hash + h = rqdata.runtaskentries[tid].unihash return pn + ":" + "taskname" + h class RunQueueStats: @@ -2299,9 +2299,6 @@ class RunQueueExecute: for tid in changed: if tid not in self.rqdata.runq_setscene_tids: continue - valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False) - if not valid: - continue if tid in self.runq_running: continue if tid not in self.pending_migrations: @@ -2358,6 +2355,7 @@ class RunQueueExecute: logger.info("Setscene task %s now valid and being rerun" % tid) self.sqdone = False + update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self) if changed: self.holdoff_need_update = True @@ -2674,64 +2672,77 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): rqdata.init_progress_reporter.next_stage() - multiconfigs = set() + sqdata.multiconfigs = set() for tid in sqdata.sq_revdeps: - multiconfigs.add(mc_from_tid(tid)) + sqdata.multiconfigs.add(mc_from_tid(tid)) if len(sqdata.sq_revdeps[tid]) == 0: sqrq.sq_buildable.add(tid) rqdata.init_progress_reporter.finish() - if rq.hashvalidate: - noexec = [] - stamppresent = [] - tocheck = set() + sqdata.noexec = set() + sqdata.stamppresent = set() + sqdata.valid = set() - for tid in sorted(sqdata.sq_revdeps): - (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) + update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq) - taskdep = rqdata.dataCaches[mc].task_deps[taskfn] +def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq): - if 'noexec' in taskdep and taskname in taskdep['noexec']: - noexec.append(tid) - sqrq.sq_task_skip(tid) - bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn) - continue + tocheck = set() - if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache): - logger.debug(2, 'Setscene stamp current for task %s', tid) - stamppresent.append(tid) - sqrq.sq_task_skip(tid) - continue + for tid in sorted(tids): + if tid in sqdata.stamppresent: + sqdata.stamppresent.remove(tid) + if tid in sqdata.valid: + sqdata.valid.remove(tid) - if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache): - logger.debug(2, 'Normal stamp current for task %s', tid) - stamppresent.append(tid) - sqrq.sq_task_skip(tid) - continue + (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) - tocheck.add(tid) + taskdep = rqdata.dataCaches[mc].task_deps[taskfn] - valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False) + if 'noexec' in taskdep and taskname in taskdep['noexec']: + sqdata.noexec.add(tid) + sqrq.sq_task_skip(tid) + bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn) + continue + + if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache): + logger.debug(2, 'Setscene stamp current for task %s', tid) + sqdata.stamppresent.add(tid) + sqrq.sq_task_skip(tid) + continue - valid_new = stamppresent - for v in valid: - valid_new.append(v) + if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache): + logger.debug(2, 'Normal stamp current for task %s', tid) + sqdata.stamppresent.add(tid) + sqrq.sq_task_skip(tid) + continue - hashes = {} - for mc in sorted(multiconfigs): - for tid in sorted(sqdata.sq_revdeps): + tocheck.add(tid) + + sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False) + + sqdata.hashes = {} + for mc in sorted(sqdata.multiconfigs): + for tid in sorted(sqdata.sq_revdeps): if mc_from_tid(tid) != mc: continue - if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered: - sqdata.outrightfail.add(tid) + if tid in sqdata.stamppresent: + continue + if tid in sqdata.valid: + continue + if tid in sqdata.noexec: + continue + if tid in sqrq.scenequeue_notcovered: + continue + sqdata.outrightfail.add(tid) - h = pending_hash_index(tid, rqdata) - if h not in hashes: - hashes[h] = tid - else: - sqrq.sq_deferred[tid] = hashes[h] - bb.warn("Deferring %s after %s" % (tid, hashes[h])) + h = pending_hash_index(tid, rqdata) + if h not in sqdata.hashes: + sqdata.hashes[h] = tid + else: + sqrq.sq_deferred[tid] = sqdata.hashes[h] + bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h])) class TaskFailure(Exception): diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py index cb4d526f13..01b992c47c 100644 --- a/bitbake/lib/bb/tests/runqueue.py +++ b/bitbake/lib/bb/tests/runqueue.py @@ -312,92 +312,3 @@ class RunQueueTests(unittest.TestCase): else: self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i) - @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') - def test_hashserv_partial_match(self): - # e1:do_package matches initial built but not second hash value - with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: - extraenv = { - "BB_HASHSERVE" : "auto", - "BB_SIGNATURE_HANDLER" : "TestEquivHash" - } - cmd = ["bitbake", "a1", "b1"] - setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', - 'populate_sysroot_setscene', 'package_qa_setscene'] - sstatevalid = "" - tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) - expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] - self.assertEqual(set(tasks), set(expected)) - with open(tempdir + "/stamps/a1.do_install.taint", "w") as f: - f.write("d460a29e-903f-4b76-a96b-3bcc22a65994") - with open(tempdir + "/stamps/b1.do_install.taint", "w") as f: - f.write("ed36d46a-2977-458a-b3de-eef885bc1817") - cmd = ["bitbake", "e1"] - sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d" - tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) - expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot', - 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene', - 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene', - 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks] - expected.remove('e1:package') - self.assertEqual(set(tasks), set(expected)) - - @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') - def test_hashserv_partial_match2(self): - # e1:do_package + e1:do_populate_sysroot matches initial built but not second hash value - with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: - extraenv = { - "BB_HASHSERVE" : "auto", - "BB_SIGNATURE_HANDLER" : "TestEquivHash" - } - cmd = ["bitbake", "a1", "b1"] - setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', - 'populate_sysroot_setscene', 'package_qa_setscene'] - sstatevalid = "" - tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) - expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] - self.assertEqual(set(tasks), set(expected)) - with open(tempdir + "/stamps/a1.do_install.taint", "w") as f: - f.write("d460a29e-903f-4b76-a96b-3bcc22a65994") - with open(tempdir + "/stamps/b1.do_install.taint", "w") as f: - f.write("ed36d46a-2977-458a-b3de-eef885bc1817") - cmd = ["bitbake", "e1"] - sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d e1:do_populate_sysroot:ef7dc0e2dd55d0534e75cba50731ff42f949818b6f29a65d72bc05856e56711d" - tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) - expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot', - 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene', - 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene', - 'e1:package_setscene', 'e1:populate_sysroot_setscene', 'e1:build', 'e1:package_qa', 'e1:package_write_rpm', 'e1:package_write_ipk', 'e1:packagedata'] - self.assertEqual(set(tasks), set(expected)) - - @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required') - def test_hashserv_partial_match3(self): - # e1:do_package is valid for a1 but not after b1 - # In former buggy code, this triggered e1:do_fetch, then e1:do_populate_sysroot to run - # with none of the intermediate tasks which is a serious bug - with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir: - extraenv = { - "BB_HASHSERVE" : "auto", - "BB_SIGNATURE_HANDLER" : "TestEquivHash" - } - cmd = ["bitbake", "a1", "b1"] - setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene', - 'populate_sysroot_setscene', 'package_qa_setscene'] - sstatevalid = "" - tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True) - expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks] - self.assertEqual(set(tasks), set(expected)) - with open(tempdir + "/stamps/a1.do_install.taint", "w") as f: - f.write("d460a29e-903f-4b76-a96b-3bcc22a65994") - with open(tempdir + "/stamps/b1.do_install.taint", "w") as f: - f.write("ed36d46a-2977-458a-b3de-eef885bc1817") - cmd = ["bitbake", "e1", "-DD"] - sstatevalid = "e1:do_package:af056eae12a733a6a8c4f4da8c6757e588e13565852c94e2aad4d953a3989c13 e1:do_package:a3677703db82b22d28d57c1820a47851dd780104580863f5bd32e66e003a779d" - tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="e1:fetch b1:install") - expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot', - 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene', - 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene', - 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks] - expected.remove('e1:package') - self.assertEqual(set(tasks), set(expected)) - - |