diff options
| -rw-r--r-- | classes/swupd-image.bbclass | 29 | ||||
| -rw-r--r-- | lib/swupd/bundles.py | 194 | ||||
| -rw-r--r-- | lib/swupd/rootfs.py | 16 | ||||
| -rw-r--r-- | lib/swupd/utils.py | 43 |
4 files changed, 67 insertions, 215 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass index 98bfa7b..8110e99 100644 --- a/classes/swupd-image.bbclass +++ b/classes/swupd-image.bbclass @@ -16,8 +16,7 @@ # See docs/Guide.md for more information. DEPLOY_DIR_SWUPDBASE = "${DEPLOY_DIR}/swupd/${MACHINE}" -SWUPD_ROOTFS_MANIFEST_SUFFIX = "-files-in-image.txt" -SWUPD_ROOTFS_MANIFEST = "${PN}${SWUPD_ROOTFS_MANIFEST_SUFFIX}" +SWUPD_ROOTFS_MANIFEST_SUFFIX = ".content.txt" # User configurable variables to disable all swupd processing or deltapack # generation. @@ -130,17 +129,13 @@ python () { for bndl in bundles: check_reserved_name(bndl) - # Generate virtual images for each bundle which adds IMAGE_FEATURES as - # we can't easily determine which packages to install in order to satisfy - # the dependecies of an IMAGE_FEATURES + # Generate virtual images for all bundles. for bndl in bundles: - features = d.getVarFlag('BUNDLE_FEATURES', bndl, True) - if features: - extended.append('swupdbundle:%s' % bndl) - dep = ' bundle-%s-%s:do_image_complete' % (pn, bndl) - # do_stage_swupd_inputs will try and utilise artefacts of the bundle - # image build, so must depend on it having completed - d.appendVarFlag('do_stage_swupd_inputs', 'depends', dep) + extended.append('swupdbundle:%s' % bndl) + dep = ' bundle-%s-%s:do_image_complete' % (pn, bndl) + # do_stage_swupd_inputs will try and utilise artefacts of the bundle + # image build, so must depend on it having completed + d.appendVarFlag('do_stage_swupd_inputs', 'depends', dep) if havebundles: extended.append('swupdbundle:mega') @@ -427,17 +422,17 @@ END done ${SWUPD_LOG_FN} "Generating update from $PREVREL to ${OS_VERSION}" - bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd + # bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd echo ${STAGING_BINDIR_NATIVE}/swupd_create_update -S ${DEPLOY_DIR_SWUPD} --osversion ${OS_VERSION} --format ${SWUPD_FORMAT} time ${STAGING_BINDIR_NATIVE}/swupd_create_update -S ${DEPLOY_DIR_SWUPD} --osversion ${OS_VERSION} --format ${SWUPD_FORMAT} ${SWUPD_LOG_FN} "Generating fullfiles for ${OS_VERSION}" - bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd + # bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd echo ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles -S ${DEPLOY_DIR_SWUPD} ${OS_VERSION} time ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles -S ${DEPLOY_DIR_SWUPD} ${OS_VERSION} ${SWUPD_LOG_FN} "Generating zero packs, this can take some time." - bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd + # bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd for bndl in ${ALL_BUNDLES}; do ${SWUPD_LOG_FN} "Generating zero pack for $bndl" echo ${STAGING_BINDIR_NATIVE}/swupd_make_pack -S ${DEPLOY_DIR_SWUPD} 0 ${OS_VERSION} $bndl @@ -445,7 +440,7 @@ END done # Generate delta-packs going back SWUPD_N_DELTAPACK versions - bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd + # bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd if [ ${SWUPD_DELTAPACKS} -eq 1 -a ${SWUPD_N_DELTAPACK} -gt 0 -a $PREVREL -gt 0 ]; then for bndl in ${ALL_BUNDLES}; do bndlcnt=0 @@ -465,7 +460,7 @@ END mkdir -p ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT} echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version - bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd + # bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd } SWUPDDEPENDS = "\ diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py index 3b8be32..c75347b 100644 --- a/lib/swupd/bundles.py +++ b/lib/swupd/bundles.py @@ -42,145 +42,62 @@ def get_bundle_packages(d, bundle): return pkgs +def create_content_manifest(dir, outfile, blacklist): + """ + Iterate over the content of the directory, remove entries listed in the blacklist + (for example, /etc/machine-id), and write the full paths of the remaining + entries (without leading ./ or /) to the file named in outfile. All directories + are explicitly listed. + """ + bb.debug(3, 'Creating %s from directory %s, excluding %s' % (outfile, dir, blacklist)) + cwd = os.getcwd() + try: + os.chdir(dir) + with open(outfile, 'w') as f: + for root, dirs, files in os.walk('.'): + for entry in dirs + files: + # strip the leading ./ + fullpath = os.path.join(root, entry)[2:] + if not ('/' + fullpath) in blacklist: + f.write(fullpath + '\n') + finally: + os.chdir(cwd) + + def copy_core_contents(d): """ - Copy the os-core contents from the mega image to swupd's image directory + Determine the os-core contents and copy the mega image to swupd's image directory. d -- the bitbake datastore """ - outfile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/${SWUPD_ROOTFS_MANIFEST}') - bundledir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/${BUNDLE_NAME}/') + corefile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}') + fullfile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full${SWUPD_ROOTFS_MANIFEST_SUFFIX}') + bundledir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full/') rootfs = d.getVar('IMAGE_ROOTFS', True) - # Generate a manifest of the bundle contents for pruning + # Generate a manifest of the bundle content. bb.utils.mkdirhier(bundledir) - manifest_cmd = 'cd %s && find . ! -path . > %s' % (rootfs, outfile) - subprocess.call(manifest_cmd, shell=True, stderr=subprocess.STDOUT) - - manifest_files = swupd.utils.manifest_to_file_list(outfile) - bundle_file_contents = [] - # Don't copy files which should not be included in the swupd manifests unwanted_files = (d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split() - # The manifest files have a leading . before the / - for f in manifest_files: - if f[1:] not in unwanted_files: - bundle_file_contents.append(f[2:]) - bb.debug(1, 'os-core contains %s items' % len(bundle_file_contents)) + create_content_manifest(rootfs, corefile, unwanted_files) + havebundles = (d.getVar('SWUPD_BUNDLES', True) or '') != '' imgrootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) if not havebundles: imgrootfs = rootfs - bb.debug(1, "Copying from image (%s) to os-core bundle dir (%s)" % (imgrootfs, bundledir)) - swupd.path.copyxattrfiles(d, bundle_file_contents, imgrootfs, bundledir) - - -def stage_package_bundle_contents(d, bundle): - """ - stage the contents of a bundle using the package manager - - For each bundle we have already included their contents in the mega-image, - thus we should be able to determine which packages were generated for that - bundles features and contents through the generated dependency data. Thus: - 1) determine the package manager and instantiate a PM object - 2) collect a list of package names for each bundle - 3) install the packages for the bundle into: - ${SWUPDIMAGEDIR}/${OS_VERSION}/$bndl - - d -- the bitbake datastore - bundle -- the name of the bundle to be staged - """ - bb.debug(1, 'Staging bundle contents for %s' % bundle) - dest = d.expand("${SWUPDIMAGEDIR}/${OS_VERSION}/%s/" % bundle) - pm = swupd.utils.get_package_manager(d, dest) - - pkgs = get_bundle_packages(d, bundle) - pm.install(pkgs) - # We don't want package manager artefacts left in the bundle 'chroot' - pm.remove_packaging_data() - # Remove any empty directories installed by the package manager, so as not - # to pollute the 'chroot' - swupd.path.remove_empty_directories(dest) - - # Create the swupd bundle manifest - create_bundle_manifest(d, bundle, dest) - - # Generate a manifest of files in the bundle - imagename = d.getVar('PN_BASE', True) - if not imagename: - imagename = d.getVar('IMAGE_BASENAME', True) - manfile = d.expand("${SWUPDIMAGEDIR}/${OS_VERSION}/bundle-%s-%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % (imagename, bundle) - bb.debug(3, 'Writing bundle file manifest %s' % manfile) - cmd = 'cd %s && find . ! -path . > %s' % (dest, manfile) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) - - # Generate a manifest of packages in the bundle, we need this so that we - # can compose a complete list of packages installed in any bundle images - mandir = d.getVar('SWUPDMANIFESTDIR', True) - manf = d.expand('${IMAGE_NAME}.rootfs.manifest') - manf = manf.replace(imagename, 'bundle-%s-%s' % (imagename, bundle)) - manfile = mandir + '/' + manf - bb.debug(3, 'Writing bundle package manifest %s' % manfile) - installed = image_list_installed_packages(d, dest) - with open(manfile, 'w+') as manifest: - manifest.write(format_pkg_list(installed, "ver")) - manifest.write('\n') - - # Also write the manifest symlink - if os.path.exists(manfile): - deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True) - link_name = d.getVar('IMAGE_LINK_NAME', True) - manifest_link = deploy_dir + '/' + link_name + '.manifest' - if os.path.lexists(manifest_link): - if d.getVar('RM_OLD_IMAGE', True) == "1" and \ - os.path.exists(os.path.realpath(manifest_link)): - os.remove(os.path.realpath(manifest_link)) - os.remove(manifest_link) - bb.debug(3, 'Linking bundle package manifest from %s to %s' % (manfile, manifest_link)) - os.symlink(os.path.basename(manfile), manifest_link) - - -def recopy_package_bundle_contents(d, bundle): - """ - recopy the contents of a bundle from the mega image rootfs - - d -- the bitbake datastore - bundle -- the name of the bundle to be staged - """ - bb.debug(2, 'Re-copying files for package based bundle %s' % bundle) - bundlecontents = [] - bundlebase = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/') - bundledir = bundlebase + bundle - bb.debug(3, 'Scanning %s for bundle files' % bundledir) + manifest_files = swupd.utils.manifest_to_file_list(corefile) + with open(fullfile, 'w') as f: + f.write('\n'.join(manifest_files)) + else: + create_content_manifest(imgrootfs, fullfile, unwanted_files) + manifest_files = swupd.utils.manifest_to_file_list(fullfile) - # Don't copy files which should not be included in the swupd manifests - unwanted_files = (d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split() - def add_target_to_contents(root, file): - # Compose a full path to the file - tgt = os.path.join(root, file) - # then strip out the prefix so it's just the target path - tgt = tgt.replace(bundledir, '') - if tgt not in unwanted_files: - bundlecontents.append(tgt) - - for root, directories, files in os.walk(bundledir): - for file in files: - add_target_to_contents(root, file) - for dir in directories: - add_target_to_contents(root, dir) - - bundle_files = swupd.utils.sanitise_file_list(bundlecontents) - - # remove the current file contents of the bundle directory - bb.debug(2, 'About to rm %s' % bundledir) - oe.path.remove(bundledir) - # copy over the required files from the megarootfs - megarootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) - swupd.path.copyxattrfiles(d, bundle_files, megarootfs, bundledir) + bb.debug(1, "Copying from image (%s) to full bundle dir (%s)" % (imgrootfs, bundledir)) + swupd.path.copyxattrfiles(d, manifest_files, imgrootfs, bundledir) -def copy_image_bundle_contents(d, bundle): +def stage_image_bundle_contents(d, bundle): """ - Copy bundle contents which aren't part of os-core from the mega-image rootfs + Determine bundle contents which aren't part of os-core from the mega-image rootfs For an image-based bundle, generate a list of files which exist in the bundle but not os-core and stage those files from the mega image rootfs to @@ -189,35 +106,23 @@ def copy_image_bundle_contents(d, bundle): d -- the bitbake datastore bundle -- the name of the bundle to be staged """ - bb.debug(2, 'Re-copying files for image based bundle %s' % bundle) # Construct paths to manifest files and directories pn = d.getVar('PN', True) manifest_path = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/') - base_manifest_name = d.getVar('SWUPD_ROOTFS_MANIFEST', True) - image_manifest_name = base_manifest_name.replace(pn, 'bundle-%s-%s' % (pn, bundle)) + base_manifest_name = d.expand('os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}') + image_manifest_name = base_manifest_name.replace('os-core', bundle, 1) base_manifest = manifest_path + base_manifest_name image_manifest = manifest_path + image_manifest_name megarootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) imagesrc = megarootfs.replace('mega', bundle) - # Generate the manifest of the bundle image's file contents + # Generate the manifest of the bundle image's file contents, + # excluding blacklisted files and the content of the os-core. bb.debug(3, 'Writing bundle image file manifest %s' % image_manifest) - cmd = 'cd %s && find . ! -path . > %s' % (imagesrc, image_manifest) - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) - - # Get a list of files in the bundle image which aren't in the base (os-core) - bb.debug(3, 'Comparing manifest %s to %s' %(base_manifest, image_manifest)) - bundle_file_contents = swupd.utils.unique_contents(base_manifest, image_manifest) - bb.debug(3, '%s has %s unique contents' % (bundle, len(bundle_file_contents))) - - bundle_files = swupd.utils.sanitise_file_list(bundle_file_contents) - bb.debug(3, 'Sanitised file list for %s has %s contents' % (bundle, len(bundle_files))) - - # Finally, copy over the unique bundle contents - bundledir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/%s/' % bundle) - swupd.path.copyxattrfiles(d, bundle_files, megarootfs, bundledir) - + unwanted_files = set((d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split()) + unwanted_files.update(['/' + x for x in swupd.utils.manifest_to_file_list(base_manifest)]) + create_content_manifest(imagesrc, image_manifest, unwanted_files) def stage_empty_bundle(d, bundle): """ @@ -244,12 +149,7 @@ def copy_bundle_contents(d): bb.debug(1, 'Copying contents of bundles for %s from mega image rootfs' % d.getVar('PN', True)) bundles = (d.getVar('SWUPD_BUNDLES', True) or '').split() for bndl in bundles: - features = d.getVarFlag('BUNDLE_FEATURES', bndl, True) - if features: - copy_image_bundle_contents(d, bndl) - else: - stage_package_bundle_contents(d, bndl) - recopy_package_bundle_contents(d, bndl) + stage_image_bundle_contents(d, bndl) bundles = (d.getVar('SWUPD_EMPTY_BUNDLES', True) or '').split() for bndl in bundles: stage_empty_bundle(d, bndl) diff --git a/lib/swupd/rootfs.py b/lib/swupd/rootfs.py index 307044d..0ed742d 100644 --- a/lib/swupd/rootfs.py +++ b/lib/swupd/rootfs.py @@ -46,6 +46,7 @@ def create_rootfs(d): outfile = d.expand('${WORKDIR}/orig-rootfs-manifest.txt') rootfs = d.getVar('IMAGE_ROOTFS', True) # Generate a manifest of the current file contents + # TODO: use the same common utility method manifest_cmd = 'cd %s && find . ! -path . > %s' % (rootfs, outfile) subprocess.call(manifest_cmd, shell=True, stderr=subprocess.STDOUT) # Remove the current rootfs contents @@ -55,18 +56,17 @@ def create_rootfs(d): # clean up os.unlink(outfile) else: # non-base image, i.e. swupdimage - manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/${PN_BASE}${SWUPD_ROOTFS_MANIFEST_SUFFIX}") - for entry in manifest_to_file_list(manifest): - rootfs_contents.append(entry[2:]) + manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}") + rootfs_contents.extend(manifest_to_file_list(manifest)) bb.debug(3, 'rootfs_contents has %s entries' % (len(rootfs_contents))) for bundle in imagebundles: - manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/bundle-${PN_BASE}-%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % bundle - for entry in manifest_to_file_list(manifest): - rootfs_contents.append(entry[2:]) + manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % bundle + rootfs_contents.extend(manifest_to_file_list(manifest)) - bb.debug(2, 'Re-copying rootfs contents from mega image') - copyxattrfiles(d, rootfs_contents, d.getVar('MEGA_IMAGE_ROOTFS', True), rootfs) + mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True) + bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_rootfs, rootfs)) + copyxattrfiles(d, rootfs_contents, mega_rootfs, rootfs) deploy_dir = d.getVar('DEPLOY_DIR_IMAGE', True) link_name = d.getVar('IMAGE_LINK_NAME', True) diff --git a/lib/swupd/utils.py b/lib/swupd/utils.py index b7bfc8e..0190821 100644 --- a/lib/swupd/utils.py +++ b/lib/swupd/utils.py @@ -1,48 +1,5 @@ -def sanitise_file_list(filelist): - """ - expand a list of paths ensuring each component is represented in the list - - We need to ensure that each component of every file path to be copied is - present in the list. - This is because when copying the file contents using copyxattrfiles() any - intermediate path components which aren't explicitly specified will be - automatically created (instead of being copied) and thus end up with the - default permissions for newly created directories -- this will likely lead - to hash mismatches in the swupd Manifests and verification failures. - We also take the step of removing a leading / from the path, as required - by copyxattrfiles() - - filelist -- the list of files to expand - """ - sanitised = set() - rootrepr = ['', '.', '/'] - - def addpathcomponents(path): - """ - add each component of path to the file list - - path -- the path to add compoents from - """ - dirname = os.path.dirname(path) - while dirname: - # If the directory is a representation of / then we're done - if dirname in rootrepr: - break - sanitised.add(dirname[1:]) - # Process the next component of the path - dirname = os.path.dirname(dirname) - - for f in filelist: - # Ensure every component of the path is included in the file list - addpathcomponents(f) - # Remove / prefix for passing to tar - sanitised.add(f[1:]) - - return sorted(sanitised) - - def manifest_to_file_list(manifest_fn): """ open a manifest file and read it into a list |