aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-10-05 13:42:11 +0200
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:12:56 +0100
commit7b21305e85feb118c20cfe8871f6c6fac3c1fb0b (patch)
treeb52d2b7825980571bb19b56bcd5afd05ec730b59
parent2ae0c47b2158a162ff8c310a9d25495831a96cc6 (diff)
downloadmeta-swupd-7b21305e85feb118c20cfe8871f6c6fac3c1fb0b.tar.gz
meta-swupd-7b21305e85feb118c20cfe8871f6c6fac3c1fb0b.tar.bz2
meta-swupd-7b21305e85feb118c20cfe8871f6c6fac3c1fb0b.zip
meta-swupd: archive mega rootfs with bsdtar
Instead of granting all virtual images access to the mega rootfs under a shared pseudo instance, archive the mega rootfs in an archive and extract from that the subset of entries that are needed. Sharing pseudo instances is slow: using more than one avoids a potential bottleneck (the pseudo daemon is often 100% busy on a CPU during heavy use). Extracting files with full attributes also is faster when merely sweeping through a tar archive, at least when most of the content is needed. This change therefore increases performance. bsdtar with support for --no-recursive in combination with -x is needed for that. Current bsdtar master does not support that yet, but adding it was easy. GNU tar already supports that, but had bugs in that mode ("Not found in archive" errors for entries that were in the archive). bsdtar is also nicer for other reasons and therefore was extended instead of trying to fix GNU tar: - no need to explicitly add xattrs - guaranteed to auto-detect compression, even when reading from stdin (GNU tar can only do that when working with files); not that relevant here, though - uses less system calls when creating files, which should help a bit with performance under pseudo Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--classes/swupd-image.bbclass50
-rw-r--r--classes/swupdbundle.bbclass15
-rw-r--r--lib/swupd/bundles.py21
-rw-r--r--lib/swupd/path.py24
-rw-r--r--lib/swupd/rootfs.py6
5 files changed, 64 insertions, 52 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass
index 466aa46..d422af1 100644
--- a/classes/swupd-image.bbclass
+++ b/classes/swupd-image.bbclass
@@ -70,10 +70,13 @@ SWUPD_LOG_FN ??= "bbdebug 1"
# a non-negative integer that fits in an int.
OS_VERSION ??= "${DISTRO_VERSION}"
-# We need to preserve xattrs which is only supported by GNU tar >= 1.27
-# to be sure this functionality works as expected use the tar-replacement-native
-DEPENDS += "tar-replacement-native"
-EXTRANATIVEPATH += "tar-native"
+# We need to preserve xattrs, which works with bsdtar out of the box.
+# It also has saner file handling (less syscalls per file) than GNU tar.
+# Last but not least, GNU tar 1.27.1 had weird problems extracting
+# all requested entries with -T from an archive ("Not found in archive"
+# errors for entries which were present and could be extraced or listed
+# when using simpler file lists).
+DEPENDS += "libarchive-native"
inherit distro_features_check
REQUIRED_DISTRO_FEATURES = "systemd"
@@ -95,29 +98,17 @@ python () {
# For bundle images, the corresponding bundle name. None in swupd images.
bundle_name = d.getVar('BUNDLE_NAME', True)
- # We set the path to the rootfs folder of the mega image here so that
- # it's simple to refer to later.
- megarootfs = d.getVar('IMAGE_ROOTFS', True)
+ # bundle-<image>-mega archives its rootfs as ${IMAGE_ROOTFS}.tar.
+ # Every other recipe then can copy (do_stage_swupd_inputs) or
+ # extract relevant files (do_image/create_rootfs()) without sharing
+ # the same pseudo database. Not sharing pseudo instances is faster
+ # and the expensive reading of individual files via pseudo only
+ # needs to be done once.
if havebundles:
- megarootfs = megarootfs.replace('/' + pn +'/', '/bundle-%s-mega/' % (pn_base or pn))
- d.setVar('MEGA_IMAGE_ROOTFS', megarootfs)
-
- # do_stage_swupd_inputs in the main image recipe and do_image in the
- # swupd images will copy files from the mega bundle and thus those
- # recipes must use the same pseudo database.
- #
- # All other bundles can use their own pseudo instance, because the
- # main image recipe is only interested in file lists, not the actual
- # file attributes.
- #
- # Because real image building via SWUPD_IMAGES can happen also after
- # the initial "bitbake <core image>" invocation, we have to keep that
- # pseudo database around and cannot delete it.
- if pn_base is None or \
- bundle_name is None or \
- bundle_name == 'mega':
- pseudo_state = d.expand('${TMPDIR}/work-shared/%s/pseudo') % (pn_base or pn)
- d.setVar('PSEUDO_LOCALSTATEDIR', pseudo_state)
+ mega_rootfs = d.getVar('IMAGE_ROOTFS', True)
+ mega_rootfs = mega_rootfs.replace('/' + pn +'/', '/bundle-%s-mega/' % (pn_base or pn))
+ d.setVar('MEGA_IMAGE_ROOTFS', mega_rootfs)
+ d.setVar('MEGA_IMAGE_ARCHIVE', mega_rootfs + '.tar')
if pn_base is not None:
# Swupd images must depend on the mega image having been
@@ -284,9 +275,7 @@ addtask do_fetch_swupd_inputs before do_swupd_update
# do_swupd_update uses its own pseudo database, for several reasons:
# - Performance is better when the pseudo instance is not shared
-# with the do_image tasks of other virtual swupd image recipes (those
-# tend to run in parallel, because they also depend on
-# do_image_complete).
+# with other tasks that run in parallel (for example, meta-isafw's do_analyse_image).
# - Wiping out the deploy/swupd directory and re-executing do_stage_swupd_inputs/do_swupd_update
# really starts from a clean slate.
# - The log.do_swupd_update will show commands that can be invoked directly, without
@@ -383,9 +372,8 @@ END
dir=$(echo $archive | sed -e 's/.tar$//')
if [ -e $archive ] && ! [ -d $dir ]; then
mkdir -p $dir
- # TODO: use bsdtar and auto-detect compression
bbnote Unpacking $archive
- env $PSEUDO tar --xattrs --xattrs-include='*' -zxf $archive -C $dir
+ env $PSEUDO bsdtar -xf $archive -C $dir
fi
done
diff --git a/classes/swupdbundle.bbclass b/classes/swupdbundle.bbclass
index 72b79fc..c3dd21d 100644
--- a/classes/swupdbundle.bbclass
+++ b/classes/swupdbundle.bbclass
@@ -19,6 +19,15 @@
# BUNDLE_CONTENTS[bar] = "bar baz quux"
# BBCLASSEXTEND = "swupdbundle:foo"
+fakeroot do_mega_archive () {
+ # Extracting files from this archive will be done using file lists which
+ # do not have a leading ./. Some versions of GNU tar had problems finding
+ # those files when we stored them with that prefix, so although we now use bsdtar,
+ # let's keep it consistent (and shorter) and store without the prefix.
+ bsdtar -zcf ${MEGA_IMAGE_ARCHIVE} -C ${MEGA_IMAGE_ROOTFS} \
+ $(ls -1 -a ${MEGA_IMAGE_ROOTFS} | grep -v -e '^\.$' -e '^\.\.$')
+}
+
python swupdbundle_virtclass_handler () {
pn = e.data.getVar("PN", True)
cls = e.data.getVar("BBEXTENDCURR", True)
@@ -43,6 +52,12 @@ python swupdbundle_virtclass_handler () {
# Not producing any real images, only the rootfs directory.
e.data.setVar("IMAGE_FSTYPES", "")
+
+ # Only the "mega" bundle gets archived in the work directory
+ # with a custom task.
+ if bundle == 'mega':
+ bb.build.addtask('do_mega_archive', 'do_image_complete', 'do_image', e.data)
+
# Delete the bootimg task as we don't require it for transient images and
# its dependent tasks are unlikely to be scheduled due to unsetting
# IMAGE_FSTYPES above.
diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index 78ffaa5..8a47a09 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -70,24 +70,25 @@ def copy_core_contents(d):
corefile + imagesuffix,
unwanted_files)
+ # Create full.tar.gz instead of directory - speeds up
+ # do_stage_swupd_input from ~11min in the Ostro CI to 6min.
+ # Where we take the data from depends on whether we have bundles:
+ # without them, there's also no "mega" bundle and we work
+ # directly with the rootfs of the main image recipe.
havebundles = (d.getVar('SWUPD_BUNDLES', True) or '') != ''
- imgrootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
if not havebundles:
- imgrootfs = rootfs
for suffix in (contentsuffix, imagesuffix):
shutil.copy2(corefile + suffix, fullfile + suffix)
+ bb.debug(1, "Copying from image rootfs (%s) to full bundle (%s)" % (imgrootfs, bundle))
+ swupd.path.copyxattrfiles(d, manifest_files, imgrootfs, bundle, True)
else:
- swupd.utils.create_content_manifests(imgrootfs,
+ mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
+ mega_archive = d.getVar('MEGA_IMAGE_ARCHIVE', True)
+ swupd.utils.create_content_manifests(mega_rootfs,
fullfile + contentsuffix,
fullfile + imagesuffix,
unwanted_files)
- manifest_files = swupd.utils.manifest_to_file_list(fullfile + contentsuffix) + \
- swupd.utils.manifest_to_file_list(fullfile + imagesuffix)
-
- bb.debug(1, "Copying from image (%s) to full bundle (%s)" % (imgrootfs, bundle))
- # Create full.tar.gz instead of directory - speeds up
- # do_stage_swupd_input from ~11min in the Ostro CI to 6min.
- swupd.path.copyxattrfiles(d, manifest_files, imgrootfs, bundle, True)
+ os.link(mega_archive, bundle)
def stage_image_bundle_contents(d, bundle):
diff --git a/lib/swupd/path.py b/lib/swupd/path.py
index 580d0b6..61e979e 100644
--- a/lib/swupd/path.py
+++ b/lib/swupd/path.py
@@ -9,7 +9,7 @@ def copyxattrtree(src, dst):
dst -- the destination to copy to
"""
import subprocess
- cmd = "tar --xattrs --xattrs-include='*' -cf - -C %s -p . | tar -p --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst)
+ cmd = "bsdtar -cf - -C %s . | bsdtar -xf - -C %s" % (src, dst)
subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
@@ -19,8 +19,8 @@ def copyxattrfiles(d, filelist, src, dst, archive=False):
d -- the bitbake data store
filelist -- a list of file paths
- src -- where to copy the files from
- dst -- where to copy the files to
+ src -- where to copy the files from (directory or archive, auto-detected)
+ dst -- where to copy the files to (directory or archive, depending on archive parameter)
archive -- create archive at dst instead of writing into that directory
"""
import subprocess
@@ -29,19 +29,27 @@ def copyxattrfiles(d, filelist, src, dst, archive=False):
bb.utils.mkdirhier(os.path.dirname(dst) if archive else dst)
files = sorted(filelist)
+ fromdir = os.path.isdir(src)
workdir = d.getVar('WORKDIR', True)
fd, copyfile = tempfile.mkstemp(dir=workdir)
os.close(fd)
with open(copyfile, 'w') as fdest:
- fdest.write('-C%s\n' % src)
for f in files:
fdest.write('%s\n' % f)
- if archive:
- cmd = "tar --xattrs --xattrs-include='*' --no-recursion -zcf %s -T %s -p" % (dst, copyfile)
+ if fromdir:
+ if archive:
+ cmd = "bsdtar --no-recursion -C %s -zcf %s -T %s -p" % (src, dst, copyfile)
+ else:
+ cmd = "bsdtar --no-recursion -C %s -cf - -T %s -p | bsdtar -p -xf - -C %s" % (src, copyfile, dst)
else:
- cmd = "tar --xattrs --xattrs-include='*' --no-recursion -cf - -T %s -p | tar -p --xattrs --xattrs-include='*' -xf - -C %s" % (copyfile, dst)
- subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
+ if archive:
+ # archive->archive not needed at the moment and cannot be done easily,
+ # because although bsdtar supports reading from an archive with @<archive>,
+ # filtering entries isn't supported in that mode.
+ bb.fatal('Extracting files from an archive and writing into an archive not implemented yet.')
+ else:
+ cmd = "bsdtar --no-recursion -C %s -xf %s -T %s" % (dst, src, copyfile)
output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
if output:
bb.fatal('Unexpected output from the following command:\n%s\n%s' % (cmd, output))
diff --git a/lib/swupd/rootfs.py b/lib/swupd/rootfs.py
index 9cf9b37..b0bed27 100644
--- a/lib/swupd/rootfs.py
+++ b/lib/swupd/rootfs.py
@@ -65,9 +65,9 @@ def create_rootfs(d):
for suffix in suffixes:
rootfs_contents.update(manifest_to_file_list(manifest + suffix))
- mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
- bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_rootfs, rootfs))
- copyxattrfiles(d, rootfs_contents, mega_rootfs, rootfs)
+ mega_archive = d.getVar('MEGA_IMAGE_ARCHIVE', True)
+ bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_archive, rootfs))
+ copyxattrfiles(d, rootfs_contents, mega_archive, rootfs)
deploy_dir = d.getVar('IMGDEPLOYDIR', True)
link_name = d.getVar('IMAGE_LINK_NAME', True)