aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-09-30 09:16:31 +0200
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:12:55 +0100
commitb641cc200acdcd4df6cb299acd10b6be165eb951 (patch)
treeff38012abc3c64a7dfd2e2d063fe6d510cec86e0
parentbcede1b94951641536d3e5318406bcec7dab2213 (diff)
downloadmeta-swupd-b641cc200acdcd4df6cb299acd10b6be165eb951.tar.gz
meta-swupd-b641cc200acdcd4df6cb299acd10b6be165eb951.tar.bz2
meta-swupd-b641cc200acdcd4df6cb299acd10b6be165eb951.zip
meta-swupd: avoid splitting up mega rootfs
Creating individual bundle directories as input for swupd is a waste of resources and time, because swupd is just going to recreate the "full" tree anyway. With an improved swupd-server, we can just copy the full tree once and then define the content of each bundle with a text file. This replaces the "files-in-image" files. Those were used only by meta-swupd before. They were renamed because they not only list files, but also directories. "content" is a bit more neutral. Creating them is now done in pure Python and integrated with the SWUPD_FILE_BLACKLIST mechanism. That way, the content files are correct right away, which allows removing the post-processing code (for example, sanitise_file_list()). The special mode of obtaining bundle content from the package manager instead of a full rootfs gets dropped for now. If that mode can be shown to be noticably faster then full rootfs creation, then it can be re-added such that it also only produces a content file for the bundle. Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--classes/swupd-image.bbclass29
-rw-r--r--lib/swupd/bundles.py194
-rw-r--r--lib/swupd/rootfs.py16
-rw-r--r--lib/swupd/utils.py43
4 files changed, 67 insertions, 215 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass
index 98bfa7b..8110e99 100644
--- a/classes/swupd-image.bbclass
+++ b/classes/swupd-image.bbclass
@@ -16,8 +16,7 @@
# See docs/Guide.md for more information.
DEPLOY_DIR_SWUPDBASE = "${DEPLOY_DIR}/swupd/${MACHINE}"
-SWUPD_ROOTFS_MANIFEST_SUFFIX = "-files-in-image.txt"
-SWUPD_ROOTFS_MANIFEST = "${PN}${SWUPD_ROOTFS_MANIFEST_SUFFIX}"
+SWUPD_ROOTFS_MANIFEST_SUFFIX = ".content.txt"
# User configurable variables to disable all swupd processing or deltapack
# generation.
@@ -130,17 +129,13 @@ python () {
for bndl in bundles:
check_reserved_name(bndl)
- # Generate virtual images for each bundle which adds IMAGE_FEATURES as
- # we can't easily determine which packages to install in order to satisfy
- # the dependecies of an IMAGE_FEATURES
+ # Generate virtual images for all bundles.
for bndl in bundles:
- features = d.getVarFlag('BUNDLE_FEATURES', bndl, True)
- if features:
- extended.append('swupdbundle:%s' % bndl)
- dep = ' bundle-%s-%s:do_image_complete' % (pn, bndl)
- # do_stage_swupd_inputs will try and utilise artefacts of the bundle
- # image build, so must depend on it having completed
- d.appendVarFlag('do_stage_swupd_inputs', 'depends', dep)
+ extended.append('swupdbundle:%s' % bndl)
+ dep = ' bundle-%s-%s:do_image_complete' % (pn, bndl)
+ # do_stage_swupd_inputs will try and utilise artefacts of the bundle
+ # image build, so must depend on it having completed
+ d.appendVarFlag('do_stage_swupd_inputs', 'depends', dep)
if havebundles:
extended.append('swupdbundle:mega')
@@ -427,17 +422,17 @@ END
done
${SWUPD_LOG_FN} "Generating update from $PREVREL to ${OS_VERSION}"
- bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd
+ # bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd
echo ${STAGING_BINDIR_NATIVE}/swupd_create_update -S ${DEPLOY_DIR_SWUPD} --osversion ${OS_VERSION} --format ${SWUPD_FORMAT}
time ${STAGING_BINDIR_NATIVE}/swupd_create_update -S ${DEPLOY_DIR_SWUPD} --osversion ${OS_VERSION} --format ${SWUPD_FORMAT}
${SWUPD_LOG_FN} "Generating fullfiles for ${OS_VERSION}"
- bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd
+ # bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd
echo ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles -S ${DEPLOY_DIR_SWUPD} ${OS_VERSION}
time ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles -S ${DEPLOY_DIR_SWUPD} ${OS_VERSION}
${SWUPD_LOG_FN} "Generating zero packs, this can take some time."
- bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd
+ # bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd
for bndl in ${ALL_BUNDLES}; do
${SWUPD_LOG_FN} "Generating zero pack for $bndl"
echo ${STAGING_BINDIR_NATIVE}/swupd_make_pack -S ${DEPLOY_DIR_SWUPD} 0 ${OS_VERSION} $bndl
@@ -445,7 +440,7 @@ END
done
# Generate delta-packs going back SWUPD_N_DELTAPACK versions
- bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd
+ # bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd
if [ ${SWUPD_DELTAPACKS} -eq 1 -a ${SWUPD_N_DELTAPACK} -gt 0 -a $PREVREL -gt 0 ]; then
for bndl in ${ALL_BUNDLES}; do
bndlcnt=0
@@ -465,7 +460,7 @@ END
mkdir -p ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version
- bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd
+ # bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd
}
SWUPDDEPENDS = "\
diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index 3b8be32..c75347b 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -42,145 +42,62 @@ def get_bundle_packages(d, bundle):
return pkgs
+def create_content_manifest(dir, outfile, blacklist):
+ """
+ Iterate over the content of the directory, remove entries listed in the blacklist
+ (for example, /etc/machine-id), and write the full paths of the remaining
+ entries (without leading ./ or /) to the file named in outfile. All directories
+ are explicitly listed.
+ """
+ bb.debug(3, 'Creating %s from directory %s, excluding %s' % (outfile, dir, blacklist))
+ cwd = os.getcwd()
+ try:
+ os.chdir(dir)
+ with open(outfile, 'w') as f:
+ for root, dirs, files in os.walk('.'):
+ for entry in dirs + files:
+ # strip the leading ./
+ fullpath = os.path.join(root, entry)[2:]
+ if not ('/' + fullpath) in blacklist:
+ f.write(fullpath + '\n')
+ finally:
+ os.chdir(cwd)
+
+
def copy_core_contents(d):
"""
- Copy the os-core contents from the mega image to swupd's image directory
+ Determine the os-core contents and copy the mega image to swupd's image directory.
d -- the bitbake datastore
"""
- outfile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/${SWUPD_ROOTFS_MANIFEST}')
- bundledir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/${BUNDLE_NAME}/')
+ corefile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}')
+ fullfile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full${SWUPD_ROOTFS_MANIFEST_SUFFIX}')
+ bundledir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full/')
rootfs = d.getVar('IMAGE_ROOTFS', True)
- # Generate a manifest of the bundle contents for pruning
+ # Generate a manifest of the bundle content.
bb.utils.mkdirhier(bundledir)
- manifest_cmd = 'cd %s && find . ! -path . > %s' % (rootfs, outfile)
- subprocess.call(manifest_cmd, shell=True, stderr=subprocess.STDOUT)
-
- manifest_files = swupd.utils.manifest_to_file_list(outfile)
- bundle_file_contents = []
- # Don't copy files which should not be included in the swupd manifests
unwanted_files = (d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split()
- # The manifest files have a leading . before the /
- for f in manifest_files:
- if f[1:] not in unwanted_files:
- bundle_file_contents.append(f[2:])
- bb.debug(1, 'os-core contains %s items' % len(bundle_file_contents))
+ create_content_manifest(rootfs, corefile, unwanted_files)
+
havebundles = (d.getVar('SWUPD_BUNDLES', True) or '') != ''
imgrootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
if not havebundles:
imgrootfs = rootfs
- bb.debug(1, "Copying from image (%s) to os-core bundle dir (%s)" % (imgrootfs, bundledir))
- swupd.path.copyxattrfiles(d, bundle_file_contents, imgrootfs, bundledir)
-
-
-def stage_package_bundle_contents(d, bundle):
- """
- stage the contents of a bundle using the package manager
-
- For each bundle we have already included their contents in the mega-image,
- thus we should be able to determine which packages were generated for that
- bundles features and contents through the generated dependency data. Thus:
- 1) determine the package manager and instantiate a PM object
- 2) collect a list of package names for each bundle
- 3) install the packages for the bundle into:
- ${SWUPDIMAGEDIR}/${OS_VERSION}/$bndl
-
- d -- the bitbake datastore
- bundle -- the name of the bundle to be staged
- """
- bb.debug(1, 'Staging bundle contents for %s' % bundle)
- dest = d.expand("${SWUPDIMAGEDIR}/${OS_VERSION}/%s/" % bundle)
- pm = swupd.utils.get_package_manager(d, dest)
-
- pkgs = get_bundle_packages(d, bundle)
- pm.install(pkgs)
- # We don't want package manager artefacts left in the bundle 'chroot'
- pm.remove_packaging_data()
- # Remove any empty directories installed by the package manager, so as not
- # to pollute the 'chroot'
- swupd.path.remove_empty_directories(dest)
-
- # Create the swupd bundle manifest
- create_bundle_manifest(d, bundle, dest)
-
- # Generate a manifest of files in the bundle
- imagename = d.getVar('PN_BASE', True)
- if not imagename:
- imagename = d.getVar('IMAGE_BASENAME', True)
- manfile = d.expand("${SWUPDIMAGEDIR}/${OS_VERSION}/bundle-%s-%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % (imagename, bundle)
- bb.debug(3, 'Writing bundle file manifest %s' % manfile)
- cmd = 'cd %s && find . ! -path . > %s' % (dest, manfile)
- subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
-
- # Generate a manifest of packages in the bundle, we need this so that we
- # can compose a complete list of packages installed in any bundle images
- mandir = d.getVar('SWUPDMANIFESTDIR', True)
- manf = d.expand('${IMAGE_NAME}.rootfs.manifest')
- manf = manf.replace(imagename, 'bundle-%s-%s' % (imagename, bundle))
- manfile = mandir + '/' + manf
- bb.debug(3, 'Writing bundle package manifest %s' % manfile)
- installed = image_list_installed_packages(d, dest)
- with open(manfile, 'w+') as manifest:
- manifest.write(format_pkg_list(installed, "ver"))
- manifest.write('\n')
-
- # Also write the manifest symlink
- if os.path.exists(manfile):
- deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True)
- link_name = d.getVar('IMAGE_LINK_NAME', True)
- manifest_link = deploy_dir + '/' + link_name + '.manifest'
- if os.path.lexists(manifest_link):
- if d.getVar('RM_OLD_IMAGE', True) == "1" and \
- os.path.exists(os.path.realpath(manifest_link)):
- os.remove(os.path.realpath(manifest_link))
- os.remove(manifest_link)
- bb.debug(3, 'Linking bundle package manifest from %s to %s' % (manfile, manifest_link))
- os.symlink(os.path.basename(manfile), manifest_link)
-
-
-def recopy_package_bundle_contents(d, bundle):
- """
- recopy the contents of a bundle from the mega image rootfs
-
- d -- the bitbake datastore
- bundle -- the name of the bundle to be staged
- """
- bb.debug(2, 'Re-copying files for package based bundle %s' % bundle)
- bundlecontents = []
- bundlebase = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/')
- bundledir = bundlebase + bundle
- bb.debug(3, 'Scanning %s for bundle files' % bundledir)
+ manifest_files = swupd.utils.manifest_to_file_list(corefile)
+ with open(fullfile, 'w') as f:
+ f.write('\n'.join(manifest_files))
+ else:
+ create_content_manifest(imgrootfs, fullfile, unwanted_files)
+ manifest_files = swupd.utils.manifest_to_file_list(fullfile)
- # Don't copy files which should not be included in the swupd manifests
- unwanted_files = (d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split()
- def add_target_to_contents(root, file):
- # Compose a full path to the file
- tgt = os.path.join(root, file)
- # then strip out the prefix so it's just the target path
- tgt = tgt.replace(bundledir, '')
- if tgt not in unwanted_files:
- bundlecontents.append(tgt)
-
- for root, directories, files in os.walk(bundledir):
- for file in files:
- add_target_to_contents(root, file)
- for dir in directories:
- add_target_to_contents(root, dir)
-
- bundle_files = swupd.utils.sanitise_file_list(bundlecontents)
-
- # remove the current file contents of the bundle directory
- bb.debug(2, 'About to rm %s' % bundledir)
- oe.path.remove(bundledir)
- # copy over the required files from the megarootfs
- megarootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
- swupd.path.copyxattrfiles(d, bundle_files, megarootfs, bundledir)
+ bb.debug(1, "Copying from image (%s) to full bundle dir (%s)" % (imgrootfs, bundledir))
+ swupd.path.copyxattrfiles(d, manifest_files, imgrootfs, bundledir)
-def copy_image_bundle_contents(d, bundle):
+def stage_image_bundle_contents(d, bundle):
"""
- Copy bundle contents which aren't part of os-core from the mega-image rootfs
+ Determine bundle contents which aren't part of os-core from the mega-image rootfs
For an image-based bundle, generate a list of files which exist in the
bundle but not os-core and stage those files from the mega image rootfs to
@@ -189,35 +106,23 @@ def copy_image_bundle_contents(d, bundle):
d -- the bitbake datastore
bundle -- the name of the bundle to be staged
"""
- bb.debug(2, 'Re-copying files for image based bundle %s' % bundle)
# Construct paths to manifest files and directories
pn = d.getVar('PN', True)
manifest_path = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/')
- base_manifest_name = d.getVar('SWUPD_ROOTFS_MANIFEST', True)
- image_manifest_name = base_manifest_name.replace(pn, 'bundle-%s-%s' % (pn, bundle))
+ base_manifest_name = d.expand('os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}')
+ image_manifest_name = base_manifest_name.replace('os-core', bundle, 1)
base_manifest = manifest_path + base_manifest_name
image_manifest = manifest_path + image_manifest_name
megarootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
imagesrc = megarootfs.replace('mega', bundle)
- # Generate the manifest of the bundle image's file contents
+ # Generate the manifest of the bundle image's file contents,
+ # excluding blacklisted files and the content of the os-core.
bb.debug(3, 'Writing bundle image file manifest %s' % image_manifest)
- cmd = 'cd %s && find . ! -path . > %s' % (imagesrc, image_manifest)
- subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
-
- # Get a list of files in the bundle image which aren't in the base (os-core)
- bb.debug(3, 'Comparing manifest %s to %s' %(base_manifest, image_manifest))
- bundle_file_contents = swupd.utils.unique_contents(base_manifest, image_manifest)
- bb.debug(3, '%s has %s unique contents' % (bundle, len(bundle_file_contents)))
-
- bundle_files = swupd.utils.sanitise_file_list(bundle_file_contents)
- bb.debug(3, 'Sanitised file list for %s has %s contents' % (bundle, len(bundle_files)))
-
- # Finally, copy over the unique bundle contents
- bundledir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/%s/' % bundle)
- swupd.path.copyxattrfiles(d, bundle_files, megarootfs, bundledir)
-
+ unwanted_files = set((d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split())
+ unwanted_files.update(['/' + x for x in swupd.utils.manifest_to_file_list(base_manifest)])
+ create_content_manifest(imagesrc, image_manifest, unwanted_files)
def stage_empty_bundle(d, bundle):
"""
@@ -244,12 +149,7 @@ def copy_bundle_contents(d):
bb.debug(1, 'Copying contents of bundles for %s from mega image rootfs' % d.getVar('PN', True))
bundles = (d.getVar('SWUPD_BUNDLES', True) or '').split()
for bndl in bundles:
- features = d.getVarFlag('BUNDLE_FEATURES', bndl, True)
- if features:
- copy_image_bundle_contents(d, bndl)
- else:
- stage_package_bundle_contents(d, bndl)
- recopy_package_bundle_contents(d, bndl)
+ stage_image_bundle_contents(d, bndl)
bundles = (d.getVar('SWUPD_EMPTY_BUNDLES', True) or '').split()
for bndl in bundles:
stage_empty_bundle(d, bndl)
diff --git a/lib/swupd/rootfs.py b/lib/swupd/rootfs.py
index 307044d..0ed742d 100644
--- a/lib/swupd/rootfs.py
+++ b/lib/swupd/rootfs.py
@@ -46,6 +46,7 @@ def create_rootfs(d):
outfile = d.expand('${WORKDIR}/orig-rootfs-manifest.txt')
rootfs = d.getVar('IMAGE_ROOTFS', True)
# Generate a manifest of the current file contents
+ # TODO: use the same common utility method
manifest_cmd = 'cd %s && find . ! -path . > %s' % (rootfs, outfile)
subprocess.call(manifest_cmd, shell=True, stderr=subprocess.STDOUT)
# Remove the current rootfs contents
@@ -55,18 +56,17 @@ def create_rootfs(d):
# clean up
os.unlink(outfile)
else: # non-base image, i.e. swupdimage
- manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/${PN_BASE}${SWUPD_ROOTFS_MANIFEST_SUFFIX}")
- for entry in manifest_to_file_list(manifest):
- rootfs_contents.append(entry[2:])
+ manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}")
+ rootfs_contents.extend(manifest_to_file_list(manifest))
bb.debug(3, 'rootfs_contents has %s entries' % (len(rootfs_contents)))
for bundle in imagebundles:
- manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/bundle-${PN_BASE}-%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % bundle
- for entry in manifest_to_file_list(manifest):
- rootfs_contents.append(entry[2:])
+ manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % bundle
+ rootfs_contents.extend(manifest_to_file_list(manifest))
- bb.debug(2, 'Re-copying rootfs contents from mega image')
- copyxattrfiles(d, rootfs_contents, d.getVar('MEGA_IMAGE_ROOTFS', True), rootfs)
+ mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
+ bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_rootfs, rootfs))
+ copyxattrfiles(d, rootfs_contents, mega_rootfs, rootfs)
deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True)
link_name = d.getVar('IMAGE_LINK_NAME', True)
diff --git a/lib/swupd/utils.py b/lib/swupd/utils.py
index b7bfc8e..0190821 100644
--- a/lib/swupd/utils.py
+++ b/lib/swupd/utils.py
@@ -1,48 +1,5 @@
-def sanitise_file_list(filelist):
- """
- expand a list of paths ensuring each component is represented in the list
-
- We need to ensure that each component of every file path to be copied is
- present in the list.
- This is because when copying the file contents using copyxattrfiles() any
- intermediate path components which aren't explicitly specified will be
- automatically created (instead of being copied) and thus end up with the
- default permissions for newly created directories -- this will likely lead
- to hash mismatches in the swupd Manifests and verification failures.
- We also take the step of removing a leading / from the path, as required
- by copyxattrfiles()
-
- filelist -- the list of files to expand
- """
- sanitised = set()
- rootrepr = ['', '.', '/']
-
- def addpathcomponents(path):
- """
- add each component of path to the file list
-
- path -- the path to add compoents from
- """
- dirname = os.path.dirname(path)
- while dirname:
- # If the directory is a representation of / then we're done
- if dirname in rootrepr:
- break
- sanitised.add(dirname[1:])
- # Process the next component of the path
- dirname = os.path.dirname(dirname)
-
- for f in filelist:
- # Ensure every component of the path is included in the file list
- addpathcomponents(f)
- # Remove / prefix for passing to tar
- sanitised.add(f[1:])
-
- return sorted(sanitised)
-
-
def manifest_to_file_list(manifest_fn):
"""
open a manifest file and read it into a list