diff options
| -rw-r--r-- | classes/swupd-image.bbclass | 50 | ||||
| -rw-r--r-- | classes/swupdbundle.bbclass | 15 | ||||
| -rw-r--r-- | lib/swupd/bundles.py | 21 | ||||
| -rw-r--r-- | lib/swupd/path.py | 24 | ||||
| -rw-r--r-- | lib/swupd/rootfs.py | 6 |
5 files changed, 64 insertions, 52 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass index 466aa46..d422af1 100644 --- a/classes/swupd-image.bbclass +++ b/classes/swupd-image.bbclass @@ -70,10 +70,13 @@ SWUPD_LOG_FN ??= "bbdebug 1" # a non-negative integer that fits in an int. OS_VERSION ??= "${DISTRO_VERSION}" -# We need to preserve xattrs which is only supported by GNU tar >= 1.27 -# to be sure this functionality works as expected use the tar-replacement-native -DEPENDS += "tar-replacement-native" -EXTRANATIVEPATH += "tar-native" +# We need to preserve xattrs, which works with bsdtar out of the box. +# It also has saner file handling (less syscalls per file) than GNU tar. +# Last but not least, GNU tar 1.27.1 had weird problems extracting +# all requested entries with -T from an archive ("Not found in archive" +# errors for entries which were present and could be extraced or listed +# when using simpler file lists). +DEPENDS += "libarchive-native" inherit distro_features_check REQUIRED_DISTRO_FEATURES = "systemd" @@ -95,29 +98,17 @@ python () { # For bundle images, the corresponding bundle name. None in swupd images. bundle_name = d.getVar('BUNDLE_NAME', True) - # We set the path to the rootfs folder of the mega image here so that - # it's simple to refer to later. - megarootfs = d.getVar('IMAGE_ROOTFS', True) + # bundle-<image>-mega archives its rootfs as ${IMAGE_ROOTFS}.tar. + # Every other recipe then can copy (do_stage_swupd_inputs) or + # extract relevant files (do_image/create_rootfs()) without sharing + # the same pseudo database. Not sharing pseudo instances is faster + # and the expensive reading of individual files via pseudo only + # needs to be done once. if havebundles: - megarootfs = megarootfs.replace('/' + pn +'/', '/bundle-%s-mega/' % (pn_base or pn)) - d.setVar('MEGA_IMAGE_ROOTFS', megarootfs) - - # do_stage_swupd_inputs in the main image recipe and do_image in the - # swupd images will copy files from the mega bundle and thus those - # recipes must use the same pseudo database. - # - # All other bundles can use their own pseudo instance, because the - # main image recipe is only interested in file lists, not the actual - # file attributes. - # - # Because real image building via SWUPD_IMAGES can happen also after - # the initial "bitbake <core image>" invocation, we have to keep that - # pseudo database around and cannot delete it. - if pn_base is None or \ - bundle_name is None or \ - bundle_name == 'mega': - pseudo_state = d.expand('${TMPDIR}/work-shared/%s/pseudo') % (pn_base or pn) - d.setVar('PSEUDO_LOCALSTATEDIR', pseudo_state) + mega_rootfs = d.getVar('IMAGE_ROOTFS', True) + mega_rootfs = mega_rootfs.replace('/' + pn +'/', '/bundle-%s-mega/' % (pn_base or pn)) + d.setVar('MEGA_IMAGE_ROOTFS', mega_rootfs) + d.setVar('MEGA_IMAGE_ARCHIVE', mega_rootfs + '.tar') if pn_base is not None: # Swupd images must depend on the mega image having been @@ -284,9 +275,7 @@ addtask do_fetch_swupd_inputs before do_swupd_update # do_swupd_update uses its own pseudo database, for several reasons: # - Performance is better when the pseudo instance is not shared -# with the do_image tasks of other virtual swupd image recipes (those -# tend to run in parallel, because they also depend on -# do_image_complete). +# with other tasks that run in parallel (for example, meta-isafw's do_analyse_image). # - Wiping out the deploy/swupd directory and re-executing do_stage_swupd_inputs/do_swupd_update # really starts from a clean slate. # - The log.do_swupd_update will show commands that can be invoked directly, without @@ -383,9 +372,8 @@ END dir=$(echo $archive | sed -e 's/.tar$//') if [ -e $archive ] && ! [ -d $dir ]; then mkdir -p $dir - # TODO: use bsdtar and auto-detect compression bbnote Unpacking $archive - env $PSEUDO tar --xattrs --xattrs-include='*' -zxf $archive -C $dir + env $PSEUDO bsdtar -xf $archive -C $dir fi done diff --git a/classes/swupdbundle.bbclass b/classes/swupdbundle.bbclass index 72b79fc..c3dd21d 100644 --- a/classes/swupdbundle.bbclass +++ b/classes/swupdbundle.bbclass @@ -19,6 +19,15 @@ # BUNDLE_CONTENTS[bar] = "bar baz quux" # BBCLASSEXTEND = "swupdbundle:foo" +fakeroot do_mega_archive () { + # Extracting files from this archive will be done using file lists which + # do not have a leading ./. Some versions of GNU tar had problems finding + # those files when we stored them with that prefix, so although we now use bsdtar, + # let's keep it consistent (and shorter) and store without the prefix. + bsdtar -zcf ${MEGA_IMAGE_ARCHIVE} -C ${MEGA_IMAGE_ROOTFS} \ + $(ls -1 -a ${MEGA_IMAGE_ROOTFS} | grep -v -e '^\.$' -e '^\.\.$') +} + python swupdbundle_virtclass_handler () { pn = e.data.getVar("PN", True) cls = e.data.getVar("BBEXTENDCURR", True) @@ -43,6 +52,12 @@ python swupdbundle_virtclass_handler () { # Not producing any real images, only the rootfs directory. e.data.setVar("IMAGE_FSTYPES", "") + + # Only the "mega" bundle gets archived in the work directory + # with a custom task. + if bundle == 'mega': + bb.build.addtask('do_mega_archive', 'do_image_complete', 'do_image', e.data) + # Delete the bootimg task as we don't require it for transient images and # its dependent tasks are unlikely to be scheduled due to unsetting # IMAGE_FSTYPES above. diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py index 78ffaa5..8a47a09 100644 --- a/lib/swupd/bundles.py +++ b/lib/swupd/bundles.py @@ -70,24 +70,25 @@ def copy_core_contents(d): corefile + imagesuffix, unwanted_files) + # Create full.tar.gz instead of directory - speeds up + # do_stage_swupd_input from ~11min in the Ostro CI to 6min. + # Where we take the data from depends on whether we have bundles: + # without them, there's also no "mega" bundle and we work + # directly with the rootfs of the main image recipe. havebundles = (d.getVar('SWUPD_BUNDLES', True) or '') != '' - imgrootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) if not havebundles: - imgrootfs = rootfs for suffix in (contentsuffix, imagesuffix): shutil.copy2(corefile + suffix, fullfile + suffix) + bb.debug(1, "Copying from image rootfs (%s) to full bundle (%s)" % (imgrootfs, bundle)) + swupd.path.copyxattrfiles(d, manifest_files, imgrootfs, bundle, True) else: - swupd.utils.create_content_manifests(imgrootfs, + mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) + mega_archive = d.getVar('MEGA_IMAGE_ARCHIVE', True) + swupd.utils.create_content_manifests(mega_rootfs, fullfile + contentsuffix, fullfile + imagesuffix, unwanted_files) - manifest_files = swupd.utils.manifest_to_file_list(fullfile + contentsuffix) + \ - swupd.utils.manifest_to_file_list(fullfile + imagesuffix) - - bb.debug(1, "Copying from image (%s) to full bundle (%s)" % (imgrootfs, bundle)) - # Create full.tar.gz instead of directory - speeds up - # do_stage_swupd_input from ~11min in the Ostro CI to 6min. - swupd.path.copyxattrfiles(d, manifest_files, imgrootfs, bundle, True) + os.link(mega_archive, bundle) def stage_image_bundle_contents(d, bundle): diff --git a/lib/swupd/path.py b/lib/swupd/path.py index 580d0b6..61e979e 100644 --- a/lib/swupd/path.py +++ b/lib/swupd/path.py @@ -9,7 +9,7 @@ def copyxattrtree(src, dst): dst -- the destination to copy to """ import subprocess - cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar -p --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) + cmd = "bsdtar -cf - -C %s . | bsdtar -xf - -C %s" % (src, dst) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) @@ -19,8 +19,8 @@ def copyxattrfiles(d, filelist, src, dst, archive=False): d -- the bitbake data store filelist -- a list of file paths - src -- where to copy the files from - dst -- where to copy the files to + src -- where to copy the files from (directory or archive, auto-detected) + dst -- where to copy the files to (directory or archive, depending on archive parameter) archive -- create archive at dst instead of writing into that directory """ import subprocess @@ -29,19 +29,27 @@ def copyxattrfiles(d, filelist, src, dst, archive=False): bb.utils.mkdirhier(os.path.dirname(dst) if archive else dst) files = sorted(filelist) + fromdir = os.path.isdir(src) workdir = d.getVar('WORKDIR', True) fd, copyfile = tempfile.mkstemp(dir=workdir) os.close(fd) with open(copyfile, 'w') as fdest: - fdest.write('-C%s\n' % src) for f in files: fdest.write('%s\n' % f) - if archive: - cmd = "tar --xattrs --xattrs-include='*' --no-recursion -zcf %s -T %s -p" % (dst, copyfile) + if fromdir: + if archive: + cmd = "bsdtar --no-recursion -C %s -zcf %s -T %s -p" % (src, dst, copyfile) + else: + cmd = "bsdtar --no-recursion -C %s -cf - -T %s -p | bsdtar -p -xf - -C %s" % (src, copyfile, dst) else: - cmd = "tar --xattrs --xattrs-include='*' --no-recursion -cf - -T %s -p | tar -p --xattrs --xattrs-include='*' -xf - -C %s" % (copyfile, dst) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) + if archive: + # archive->archive not needed at the moment and cannot be done easily, + # because although bsdtar supports reading from an archive with @<archive>, + # filtering entries isn't supported in that mode. + bb.fatal('Extracting files from an archive and writing into an archive not implemented yet.') + else: + cmd = "bsdtar --no-recursion -C %s -xf %s -T %s" % (dst, src, copyfile) output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) if output: bb.fatal('Unexpected output from the following command:\n%s\n%s' % (cmd, output)) diff --git a/lib/swupd/rootfs.py b/lib/swupd/rootfs.py index 9cf9b37..b0bed27 100644 --- a/lib/swupd/rootfs.py +++ b/lib/swupd/rootfs.py @@ -65,9 +65,9 @@ def create_rootfs(d): for suffix in suffixes: rootfs_contents.update(manifest_to_file_list(manifest + suffix)) - mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) - bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_rootfs, rootfs)) - copyxattrfiles(d, rootfs_contents, mega_rootfs, rootfs) + mega_archive = d.getVar('MEGA_IMAGE_ARCHIVE', True) + bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_archive, rootfs)) + copyxattrfiles(d, rootfs_contents, mega_archive, rootfs) deploy_dir = d.getVar('IMGDEPLOYDIR', True) link_name = d.getVar('IMAGE_LINK_NAME', True) |