aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-10-28 10:09:35 +0200
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:12:55 +0100
commite5f6bdd4838d1de88d45819efc941730361b6f2d (patch)
tree3ab57afc5355008cef5077683b59184196dcbe16
parent7a3c43a04465e30b0754fecbc21008984fa92552 (diff)
downloadmeta-swupd-e5f6bdd4838d1de88d45819efc941730361b6f2d.tar.gz
meta-swupd-e5f6bdd4838d1de88d45819efc941730361b6f2d.tar.bz2
meta-swupd-e5f6bdd4838d1de88d45819efc941730361b6f2d.zip
meta-swupd: revise delta computation
This removes the storing of previous build information in sstate. It was conceptually questionable (sstate is a cache which does not need to be backed up, while the information about previous builds is crucial and must not get lost) and not working: - the -map.inc file wasn't actually included anywhere and thus the old build information wasn't getting restored - restoring all previous builds would have made building slower and slower as the number of previous builds grows - the old build information lacked the www/Manifest files that incremental updates need The replacement puts previous build information into the image deploy directory. That's tentative and also not fully working. The automatic selection of old versions to build deltas against also gets replaced with an explicit choice that has to be made by the user of meta-swupd. That's because in practice, incremental updates are more useful when prepared for the releases that actually run on the target device, like major milestones. Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--classes/swupd-image.bbclass186
-rw-r--r--lib/swupd/bundles.py14
2 files changed, 34 insertions, 166 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass
index 67a9316..bc23cce 100644
--- a/classes/swupd-image.bbclass
+++ b/classes/swupd-image.bbclass
@@ -35,9 +35,7 @@ DEPLOY_DIR_SWUPD = "${DEPLOY_DIR}/swupd/${MACHINE}/${SWUPD_IMAGE_PN}"
# User configurable variables to disable all swupd processing or deltapack
# generation.
SWUPD_GENERATE ??= "1"
-SWUPD_DELTAPACKS ??= "1"
-# Create delta packs for N versions back — default 2
-SWUPD_N_DELTAPACK ??= "2"
+SWUPD_DELTAPACK_VERSIONS ??= ""
SWUPD_LOG_FN ??= "bbdebug 1"
@@ -79,12 +77,6 @@ python () {
megarootfs = megarootfs.replace('/' + pn +'/', '/bundle-%s-mega/' % (pn_base or pn))
d.setVar('MEGA_IMAGE_ROOTFS', megarootfs)
- # We need to use a custom manifest filename for stage_swupd_inputs so that
- # the generated sstate can be used to fetch inputs for multiple "releases"
- manfileprefix = d.getVar('SSTATE_MANFILEPREFIX', True)
- manfileprefix = manfileprefix + '-' + ver
- d.setVar('SSTATE_MANFILEPREFIX', manfileprefix)
-
# do_stage_swupd_inputs in the main image recipe and do_image in the
# swupd images will copy files from the mega bundle and thus those
# recipes must use the same pseudo database.
@@ -230,16 +222,9 @@ do_image_append () {
#"
SWUPD_FILE_BLACKLIST ??= ""
-# When set to a non-empty string, the "swupd/image" content for the
-# current build gets archived in the sstate-cache. This is
-# experimental and disabled by default. Using it to create deltas
-# between builds also implies doing some extra work to preserve the
-# SWUPD_SSTATE_MAP map file across builds.
-SWUPD_SSTATE ??= ""
-
-SWUPDIMAGEDIR = "${@ '${WORKDIR}/swupd-image' if '${SWUPD_SSTATE}' else '${DEPLOY_DIR_SWUPD}/image'}"
+SWUPDIMAGEDIR = "${DEPLOY_DIR_SWUPD}/image"
SWUPDMANIFESTDIR = "${WORKDIR}/swupd-manifests"
-SWUPD_SSTATE_MAP = "${DEPLOY_DIR_SWUPD}/${PN}-map.inc"
+
fakeroot python do_stage_swupd_inputs () {
import swupd.bundles
@@ -249,149 +234,12 @@ fakeroot python do_stage_swupd_inputs () {
swupd.bundles.copy_core_contents(d)
swupd.bundles.copy_bundle_contents(d)
-
- # Write information about all known OSV->sstate obj mappings
- mapfile = d.getVar('SWUPD_SSTATE_MAP', True)
- pn = d.getVar('PN', True)
- sstatepkg = d.getVar('SSTATE_PKGNAME', True) + '_stage_swupd_inputs.tgz'
- osv = d.getVar('OS_VERSION', True)
- verdict = {}
- versions = (d.getVar('OS_VERSION_SSTATE_MAP_%s' % pn, True) or '').split()
- for version in versions:
- ver, pkg = version.split('=')
- verdict[ver] = pkg
- verdict[osv] = sstatepkg
-
- with open(mapfile, 'w') as f:
- f.write('OS_VERSION_SSTATE_MAP_%s = "\\\n' % pn)
- for ver, pkg in verdict.items():
- f.write(' %s=%s \\\n' % (ver, pkg))
- f.write(' "\n')
-
- bb.debug(3, 'Writing mapfile to %s' % mapfile)
+ swupd.bundles.copy_old_versions(d)
}
addtask stage_swupd_inputs after do_image before do_swupd_update
do_stage_swupd_inputs[dirs] = "${SWUPDIMAGEDIR} ${SWUPDMANIFESTDIR} ${DEPLOY_DIR_SWUPD}/maps/"
do_stage_swupd_inputs[depends] += "virtual/fakeroot-native:do_populate_sysroot"
-SSTATETASKS += "${@ 'do_stage_swupd_inputs' if '${SWUPD_SSTATE}' else ''}"
-do_stage_swupd_inputs[sstate-inputdirs] = "${SWUPDIMAGEDIR}/${OS_VERSION} ${SWUPDMANIFESTDIR}"
-do_stage_swupd_inputs[sstate-outputdirs] = "${DEPLOY_DIR_SWUPD}/image/${OS_VERSION} ${DEPLOY_DIR_IMAGE}"
-
-python swupd_fix_manifest_link() {
- """
- Ensure the manifest symlink points to the latest version of the manifest,
- not the most recently staged.
- """
- import glob
-
- sourcedir = d.getVar('SWUPDMANIFESTDIR', True)
- destdir = d.getVar('DEPLOY_DIR_IMAGE', True)
- links = []
- # Find symlinks in SWUPDMANIFESTDIR
- for f in os.listdir(sourcedir):
- if os.path.islink(os.path.join(sourcedir, f)):
- links.append(f)
-
- for link in links:
- target = None
- latest = None
- # Extract a pattern for glob:
- # core-image-minimal-qemux86.manifest ->
- # core-image-minimal-qemux86-20160602082427.rootfs.manifest
- components = link.split('.')
- prefix = components[0]
- suffix = components[1]
- pattern = prefix + '-*.' + suffix
- # Find files matching the pattern in DEPLOY_DIR_IMAGE
- for f in glob.glob(destdir+'/'+pattern):
- # Find the most recent file matching that pattern
- fname = os.path.basename(f)
- date = f.split('-')[-1].split('.')[0]
- if not latest or latest < date:
- target = f
- # Update the symlink
- lnk = os.path.join(destdir, link)
- os.remove(lnk)
- bb.debug(3, 'Updating link %s to %s' % (lnk, target))
- os.symlink(target, lnk)
-}
-
-python do_stage_swupd_inputs_setscene () {
- if d.getVar('PN_BASE', True):
- bb.debug(2, 'Skipping update input staging from sstate for non-base image %s' % d.getVar('PN', True))
- return
-
- sstate_setscene(d)
-}
-addtask do_stage_swupd_inputs_setscene
-do_stage_swupd_inputs_setscene[dirs] = "${SWUPDIMAGEDIR} ${DEPLOY_DIR_SWUPD}/image/ ${SWUPDMANIFESTDIR} ${DEPLOY_DIR_IMAGE}"
-do_stage_swupd_inputs_setscene[postfuncs] += "swupd_fix_manifest_link "
-
-fakeroot python do_fetch_swupd_inputs () {
- import subprocess
- import swupd.path
-
- if d.getVar('PN_BASE', True):
- bb.debug(2, 'Skipping update input fetching for non-base image %s' % d.getVar('PN', True))
- return
-
- fetchlist = {}
- pn = d.getVar('PN', True)
- currv = d.getVar('OS_VERSION', True)
- maplist = (d.getVar('OS_VERSION_SSTATE_MAP_%s' % pn, True) or '').split()
- for map in maplist:
- osv, pkg = map.split('=')
- if osv == currv:
- continue
- fetchlist[osv] = pkg
-
- workdir = d.expand('${WORKDIR}/fetched-inputs')
- bb.utils.mkdirhier(workdir)
-
- deploydirswupd = d.getVar('DEPLOY_DIR_SWUPD', True)
- deploydirimage = d.getVar('DEPLOY_DIR_IMAGE', True)
- sstatedir = d.getVar('SSTATE_DIR', True)
- # For each identified input sstate object, try and ensure we have the
- # object file available
- for osv, pkg in fetchlist.items():
- # Don't try and fetch & unpack the sstate for a version directory
- # which already exists
- imgdst = os.path.join(deploydirswupd, 'image', osv)
- if os.path.exists(imgdst):
- continue
-
- sstatefetch = pkg
- sstatepkg = '%s/%s' % (sstatedir, pkg)
-
- bb.debug(1, 'Preparing sstate package %s' % sstatepkg)
-
- if not os.path.exists(sstatepkg):
- bb.debug(2, 'Fetching object %s from mirror' % sstatepkg)
- pstaging_fetch(sstatefetch, sstatepkg, d)
-
- if not os.path.isfile(sstatepkg):
- bb.debug(2, "Shared state package %s is not available" % sstatepkg)
- continue
-
- # We now have a copy of the sstate for a do_stage_swupd_inputs
- # version let's "install" it. We have two directories:
- # $osv: should be extracted to ${DEPLOY_DIR_SWUPD}/image/$osv
- # swupd-manifests: should be extracted to ${DEPLOY_DIR_IMAGE}
- src = os.path.join(workdir, osv)
- bb.utils.mkdirhier(src)
-
- bb.debug(2, 'Unpacking sstate object %s in %s' % (sstatepkg, src))
- cmd = 'cd %s && tar -xvzf %s' % (src, sstatepkg)
- subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
- bb.utils.mkdirhier(imgdst)
- swupd.path.copyxattrtree('%s/%s/' % (src, osv), imgdst)
- swupd.path.copyxattrtree('%s/swupd-manifests/' % src, deploydirimage)
-}
-addtask fetch_swupd_inputs before do_swupd_update
-do_fetch_swupd_inputs[dirs] = "${DEPLOY_DIR_SWUPD}/maps ${DEPLOY_DIR_SWUPD}/image"
-do_fetch_swupd_inputs[depends] += "virtual/fakeroot-native:do_populate_sysroot"
-
SWUPD_FORMAT ??= "3"
# do_swupd_update uses its own pseudo database, for several reasons:
# - Performance is better when the pseudo instance is not shared
@@ -464,6 +312,12 @@ END
if [ -e ${DEPLOY_DIR_SWUPD}/image/latest.version ]; then
PREVREL=`cat ${DEPLOY_DIR_SWUPD}/image/latest.version`
else
+ # TODO: locate information about latest version from online www update repo
+ # and download the relevant files. That makes swupd_create_fullfiles
+ # a lot faster because it allows reusing existing, unmodified files.
+ # Saves a lot of space, too, because the new Manifest files then merely
+ # point to the older version (no entry in ${DEPLOY_DIR_SWUPD}/www/${OS_VERSION}/files,
+ # not even a link).
bbdebug 2 "Stubbing out empty latest.version file"
touch ${DEPLOY_DIR_SWUPD}/image/latest.version
PREVREL="0"
@@ -517,20 +371,15 @@ END
invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout -S ${DEPLOY_DIR_SWUPD} 0 ${OS_VERSION} $bndl
done
- # Generate delta-packs going back SWUPD_N_DELTAPACK versions
+ # Generate delta-packs against previous versions chosen by our caller.
# env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd
- if [ ${SWUPD_DELTAPACKS} -eq 1 -a ${SWUPD_N_DELTAPACK} -gt 0 -a $PREVREL -gt 0 ]; then
+ for prevver in ${SWUPD_DELTAPACK_VERSIONS}; do
for bndl in ${ALL_BUNDLES}; do
bndlcnt=0
- # Build list of previous versions and pick the last n ones to build
- # deltas against. Ignore the latest one, which is the one we build
- # right now.
- ls -d -1 ${DEPLOY_DIR_SWUPD}/image/*/$bndl | sed -e 's;${DEPLOY_DIR_SWUPD}/image/\([^/]*\)/.*;\1;' | grep -e '^[0-9]*$' | sort -n | head -n -1 | tail -n ${SWUPD_N_DELTAPACK} | while read prevver; do
- ${SWUPD_LOG_FN} "Generating delta pack from $prevver to ${OS_VERSION} for $bndl"
- invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl
- done
+ ${SWUPD_LOG_FN} "Generating delta pack from $prevver to ${OS_VERSION} for $bndl"
+ invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl
done
- fi
+ done
# Write version to www/version/format${SWUPD_FORMAT}/latest and image/latest.version
bbdebug 2 "Writing latest file"
@@ -538,6 +387,11 @@ END
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version
# env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd
+
+ # Archive the files of the current build which will be needed in the future
+ # for a <current version> -> <future version> delta computation. We exclude
+ # the expanded "full" rootfs, because we already have "full.tar".
+ (cd ${DEPLOY_DIR_SWUPD}; tar -zcf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}-${OS_VERSION}-swupd.tar --exclude=full --exclude=Manifest.*.tar image/${OS_VERSION} www/${OS_VERSION}/Manifest.*)
}
SWUPDDEPENDS = "\
diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index abfd08c..611f59d 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -1,3 +1,4 @@
+import glob
import subprocess
import shutil
from oe.package_manager import RpmPM
@@ -145,3 +146,16 @@ def copy_bundle_contents(d):
bundles = (d.getVar('SWUPD_EMPTY_BUNDLES', True) or '').split()
for bndl in bundles:
stage_empty_bundle(d, bndl)
+
+def copy_old_versions(d):
+ for prevver in d.getVar('SWUPD_DELTAPACK_VERSIONS', True).split():
+ if not os.path.exists(os.path.join(d.expand('${DEPLOY_DIR_SWUPD}/image'), prevver)):
+ pattern = d.expand('${DEPLOY_DIR_IMAGE}/${IMAGE_BASENAME}*-%s-swupd.tar' % prevver)
+ prevver_tar = glob.glob(pattern)
+ if not prevver_tar or len(prevver_tar) > 1 or not os.path.exists(prevver_tar[0]):
+ bb.fatal("Creating swupd delta packs against %s is not possible because %s is not available." %
+ (prevver, pattern))
+ cmd = ['tar', '-C', d.getVar('DEPLOY_DIR_SWUPD', True), '-xf', prevver_tar[0]]
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ if output:
+ bb.fatal('Unexpected output from the following command:\n%s\n%s' % (cmd, output))