diff options
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/reproducible.py')
| -rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 128 |
1 files changed, 119 insertions, 9 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index 5d3959be77..be4cdcc429 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py @@ -17,6 +17,57 @@ import stat import os import datetime +# For sample packages, see: +# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/ +# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/ +# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/ +# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/ +# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/ +# (both packages/ and packages-excluded/) +exclude_packages = [ + 'acpica-src', + 'babeltrace2-ptest', + 'bind', + 'bootchart2-doc', + 'epiphany', + 'gcr', + 'glide', + 'go-dep', + 'go-helloworld', + 'go-runtime', + 'go_', + 'gstreamer1.0-python', + 'hwlatdetect', + 'kernel-devsrc', + 'libcap-ng', + 'libjson', + 'libproxy', + 'lttng-tools-dbg', + 'lttng-tools-ptest', + 'ltp', + 'ovmf-shell-efi', + 'parted-ptest', + 'perf', + 'piglit', + 'pybootchartgui', + 'qemu', + 'quilt-ptest', + 'rsync', + 'ruby', + 'stress-ng', + 'systemd-bootchart', + 'systemtap', + 'valgrind-ptest', + 'webkitgtk', + ] + +def is_excluded(package): + package_name = os.path.basename(package) + for i in exclude_packages: + if package_name.startswith(i): + return i + return None + MISSING = 'MISSING' DIFFERENT = 'DIFFERENT' SAME = 'SAME' @@ -39,14 +90,21 @@ class PackageCompareResults(object): self.total = [] self.missing = [] self.different = [] + self.different_excluded = [] self.same = [] + self.active_exclusions = set() def add_result(self, r): self.total.append(r) if r.status == MISSING: self.missing.append(r) elif r.status == DIFFERENT: - self.different.append(r) + exclusion = is_excluded(r.reference) + if exclusion: + self.different_excluded.append(r) + self.active_exclusions.add(exclusion) + else: + self.different.append(r) else: self.same.append(r) @@ -54,10 +112,14 @@ class PackageCompareResults(object): self.total.sort() self.missing.sort() self.different.sort() + self.different_excluded.sort() self.same.sort() def __str__(self): - return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total)) + return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions()) + + def unused_exclusions(self): + return sorted(set(exclude_packages) - self.active_exclusions) def compare_file(reference, test, diffutils_sysroot): result = CompareResult() @@ -68,7 +130,7 @@ def compare_file(reference, test, diffutils_sysroot): result.status = MISSING return result - r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True) + r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False) if r.status: result.status = DIFFERENT @@ -77,9 +139,41 @@ def compare_file(reference, test, diffutils_sysroot): result.status = SAME return result +def run_diffoscope(a_dir, b_dir, html_dir, **kwargs): + return runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir], + **kwargs) + +class DiffoscopeTests(OESelftestTestCase): + diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope") + + def test_diffoscope(self): + bitbake("diffoscope-native -c addto_recipe_sysroot") + diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") + + # Check that diffoscope doesn't return an error when the files compare + # the same (a general check that diffoscope is working) + with tempfile.TemporaryDirectory() as tmpdir: + run_diffoscope('A', 'A', tmpdir, + native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files) + + # Check that diffoscope generates an index.html file when the files are + # different + with tempfile.TemporaryDirectory() as tmpdir: + r = run_diffoscope('A', 'B', tmpdir, + native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files) + + self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected") + self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!") + class ReproducibleTests(OESelftestTestCase): + # Test the reproducibility of whatever is built between sstate_targets and targets + package_classes = ['deb', 'ipk'] - images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline'] + + # targets are the things we want to test the reproducibility of + targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'world'] + # sstate targets are things to pull from sstate to potentially cut build/debugging time + sstate_targets = [] save_results = False if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] @@ -94,7 +188,7 @@ class ReproducibleTests(OESelftestTestCase): def setUpLocal(self): super().setUpLocal() - needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS'] + needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS', 'BB_HASHSERVE'] bb_vars = get_bb_vars(needed_vars) for v in needed_vars: setattr(self, v.lower(), bb_vars[v]) @@ -150,20 +244,29 @@ class ReproducibleTests(OESelftestTestCase): PACKAGE_CLASSES = "{package_classes}" INHIBIT_PACKAGE_STRIP = "1" TMPDIR = "{tmpdir}" + LICENSE_FLAGS_WHITELIST = "commercial" + DISTRO_FEATURES_append = ' systemd pam' ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), tmpdir=tmpdir) if not use_sstate: + if self.sstate_targets: + self.logger.info("Building prebuild for %s (sstate allowed)..." % (name)) + self.write_config(config) + bitbake(' '.join(self.sstate_targets)) + # This config fragment will disable using shared and the sstate # mirror, forcing a complete build from scratch config += textwrap.dedent('''\ SSTATE_DIR = "${TMPDIR}/sstate" - SSTATE_MIRROR = "" + SSTATE_MIRRORS = "file://.*/.*-native.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH" ''') + self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) self.write_config(config) d = get_bb_vars(capture_vars) - bitbake(' '.join(self.images)) + # targets used to be called images + bitbake(' '.join(getattr(self, 'images', self.targets))) return d def test_reproducible_builds(self): @@ -187,6 +290,7 @@ class ReproducibleTests(OESelftestTestCase): self.logger.info('Non-reproducible packages will be copied to %s', save_dir) vars_A = self.do_test_build('reproducibleA', self.build_from_sstate) + vars_B = self.do_test_build('reproducibleB', False) # NOTE: The temp directories from the reproducible build are purposely @@ -201,6 +305,7 @@ class ReproducibleTests(OESelftestTestCase): deploy_A = vars_A['DEPLOY_DIR_' + c.upper()] deploy_B = vars_B['DEPLOY_DIR_' + c.upper()] + self.logger.info('Checking %s packages for differences...' % c) result = self.compare_packages(deploy_A, deploy_B, diffutils_sysroot) self.logger.info('Reproducibility summary for %s: %s' % (c, result)) @@ -209,6 +314,7 @@ class ReproducibleTests(OESelftestTestCase): self.write_package_list(package_class, 'missing', result.missing) self.write_package_list(package_class, 'different', result.different) + self.write_package_list(package_class, 'different_excluded', result.different_excluded) self.write_package_list(package_class, 'same', result.same) if self.save_results: @@ -216,8 +322,12 @@ class ReproducibleTests(OESelftestTestCase): self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) + for d in result.different_excluded: + self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)])) + self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)])) + if result.missing or result.different: - fails.append("The following %s packages are missing or different: %s" % + fails.append("The following %s packages are missing or different and not in exclusion list: %s" % (c, '\n'.join(r.test for r in (result.missing + result.different)))) # Clean up empty directories @@ -232,7 +342,7 @@ class ReproducibleTests(OESelftestTestCase): # Copy jquery to improve the diffoscope output usability self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) - runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', '--html-dir', package_html_dir, 'reproducibleA', 'reproducibleB'], + run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir, native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) if fails: |