aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-10-04 13:19:13 +0200
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:12:55 +0100
commitaf2753627478992328221b369d4eddce74efe475 (patch)
tree6031da5835d095339058f17c44aaedfaee2400db
parentacc51c5cfd9694227772e6dd814063e4870f6659 (diff)
downloadmeta-swupd-af2753627478992328221b369d4eddce74efe475.tar.gz
meta-swupd-af2753627478992328221b369d4eddce74efe475.tar.bz2
meta-swupd-af2753627478992328221b369d4eddce74efe475.zip
fix meta-swupd: avoid splitting up mega rootfs
When creating bundle images, we need to know and copy also the entries that we exclude from processing by swupd-server. This could be done with a more complex syntax for the .content.txt files, but that would also make the swupd-server patches more complicated. Instead, an .extra-content.txt gets written alongside the .content.text and meta-swupd uses that when copying files into images. Due to the way how this is implemented, the .extra-content.txt of bundles also lists the files that were excluded from the bundle because they were already in the os-core. This may or may not be desirable. This change also includes some other improvements (consistent use of the helper method, sorting the content of the file lists). Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--classes/swupd-image.bbclass7
-rw-r--r--lib/swupd/bundles.py65
-rw-r--r--lib/swupd/rootfs.py27
-rw-r--r--lib/swupd/utils.py24
4 files changed, 72 insertions, 51 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass
index 3336dab..64b84ce 100644
--- a/classes/swupd-image.bbclass
+++ b/classes/swupd-image.bbclass
@@ -16,7 +16,14 @@
# See docs/Guide.md for more information.
DEPLOY_DIR_SWUPDBASE = "${DEPLOY_DIR}/swupd/${MACHINE}"
+# Created for each bundle (including os-core) and the "full" directory,
+# describing files and directories that swupd-server needs to include in the update
+# mechanism (i.e. without SWUPD_FILE_BLACKLIST entries). Used by swupd-server.
SWUPD_ROOTFS_MANIFEST_SUFFIX = ".content.txt"
+# Additional entries which need to be in images (for example, /etc/machine-id, but
+# that are excluded from the update mechanism. Ignored by swupd-server,
+# used by swupdimage.bbclass.
+SWUPD_IMAGE_MANIFEST_SUFFIX = ".extra-content.txt"
# User configurable variables to disable all swupd processing or deltapack
# generation.
diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index 34e5a37..abfd08c 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -1,4 +1,5 @@
import subprocess
+import shutil
from oe.package_manager import RpmPM
from oe.package_manager import OpkgPM
from oe.package_manager import DpkgPM
@@ -42,28 +43,6 @@ def get_bundle_packages(d, bundle):
return pkgs
-def create_content_manifest(dir, outfile, blacklist):
- """
- Iterate over the content of the directory, remove entries listed in the blacklist
- (for example, /etc/machine-id), and write the full paths of the remaining
- entries (without leading ./ or /) to the file named in outfile. All directories
- are explicitly listed.
- """
- bb.debug(3, 'Creating %s from directory %s, excluding %s' % (outfile, dir, blacklist))
- cwd = os.getcwd()
- try:
- os.chdir(dir)
- with open(outfile, 'w') as f:
- for root, dirs, files in os.walk('.'):
- for entry in dirs + files:
- # strip the leading ./
- fullpath = os.path.join(root, entry)[2:]
- if not ('/' + fullpath) in blacklist:
- f.write(fullpath + '\n')
- finally:
- os.chdir(cwd)
-
-
def copy_core_contents(d):
"""
Determine the os-core contents and copy the mega image to swupd's image directory.
@@ -71,26 +50,34 @@ def copy_core_contents(d):
d -- the bitbake datastore
"""
imagedir = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}')
- corefile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}')
- fullfile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full${SWUPD_ROOTFS_MANIFEST_SUFFIX}')
+ corefile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/os-core')
+ contentsuffix = d.getVar('SWUPD_ROOTFS_MANIFEST_SUFFIX', True)
+ imagesuffix = d.getVar('SWUPD_IMAGE_MANIFEST_SUFFIX', True)
+ fullfile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full')
bundle = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/full.tar')
rootfs = d.getVar('IMAGE_ROOTFS', True)
# Generate a manifest of the bundle content.
bb.utils.mkdirhier(imagedir)
unwanted_files = (d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split()
- create_content_manifest(rootfs, corefile, unwanted_files)
+ swupd.utils.create_content_manifests(rootfs,
+ corefile + contentsuffix,
+ corefile + imagesuffix,
+ unwanted_files)
havebundles = (d.getVar('SWUPD_BUNDLES', True) or '') != ''
imgrootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
if not havebundles:
imgrootfs = rootfs
- manifest_files = swupd.utils.manifest_to_file_list(corefile)
- with open(fullfile, 'w') as f:
- f.write('\n'.join(manifest_files))
+ for suffix in (contentsuffix, imagesuffix):
+ shutil.copy2(corefile + suffix, fullfile + suffix)
else:
- create_content_manifest(imgrootfs, fullfile, unwanted_files)
- manifest_files = swupd.utils.manifest_to_file_list(fullfile)
+ swupd.utils.create_content_manifests(imgrootfs,
+ fullfile + contentsuffix,
+ fullfile + imagesuffix,
+ unwanted_files)
+ manifest_files = swupd.utils.manifest_to_file_list(fullfile + contentsuffix) + \
+ swupd.utils.manifest_to_file_list(fullfile + imagesuffix)
bb.debug(1, "Copying from image (%s) to full bundle (%s)" % (imgrootfs, bundle))
# Create full.tar.gz instead of directory - speeds up
@@ -112,20 +99,22 @@ def stage_image_bundle_contents(d, bundle):
# Construct paths to manifest files and directories
pn = d.getVar('PN', True)
- manifest_path = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/')
- base_manifest_name = d.expand('os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}')
- image_manifest_name = base_manifest_name.replace('os-core', bundle, 1)
- base_manifest = manifest_path + base_manifest_name
- image_manifest = manifest_path + image_manifest_name
+ corefile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/os-core')
+ bundlefile = d.expand('${SWUPDIMAGEDIR}/${OS_VERSION}/') + bundle
+ contentsuffix = d.getVar('SWUPD_ROOTFS_MANIFEST_SUFFIX', True)
+ imagesuffix = d.getVar('SWUPD_IMAGE_MANIFEST_SUFFIX', True)
megarootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
imagesrc = megarootfs.replace('mega', bundle)
# Generate the manifest of the bundle image's file contents,
# excluding blacklisted files and the content of the os-core.
- bb.debug(3, 'Writing bundle image file manifest %s' % image_manifest)
+ bb.debug(3, 'Writing bundle image file manifests %s' % bundlefile)
unwanted_files = set((d.getVar('SWUPD_FILE_BLACKLIST', True) or '').split())
- unwanted_files.update(['/' + x for x in swupd.utils.manifest_to_file_list(base_manifest)])
- create_content_manifest(imagesrc, image_manifest, unwanted_files)
+ unwanted_files.update(['/' + x for x in swupd.utils.manifest_to_file_list(corefile + contentsuffix)])
+ swupd.utils.create_content_manifests(imagesrc,
+ bundlefile + contentsuffix,
+ bundlefile + imagesuffix,
+ unwanted_files)
def stage_empty_bundle(d, bundle):
"""
diff --git a/lib/swupd/rootfs.py b/lib/swupd/rootfs.py
index 53cef55..9cf9b37 100644
--- a/lib/swupd/rootfs.py
+++ b/lib/swupd/rootfs.py
@@ -1,7 +1,7 @@
import os
import bb
import oe.path
-from swupd.utils import manifest_to_file_list
+from swupd.utils import manifest_to_file_list, create_content_manifests
from swupd.path import copyxattrfiles
@@ -30,11 +30,15 @@ def create_rootfs(d):
bb.debug(2, 'Skipping swupd_create_rootfs(), original rootfs can be used as no additional bundles are defined')
return
+ contentsuffix = d.getVar('SWUPD_ROOTFS_MANIFEST_SUFFIX', True)
+ imagesuffix = d.getVar('SWUPD_IMAGE_MANIFEST_SUFFIX', True)
+ suffixes = (contentsuffix, imagesuffix)
+
# Sanity checking was already done in swupdimage.bbclass.
# Here we can simply use the settings.
imagebundles = d.getVarFlag('SWUPD_IMAGES', imageext, True).split() if imageext else []
rootfs = d.getVar('IMAGE_ROOTFS', True)
- rootfs_contents = []
+ rootfs_contents = set()
if not pn_base: # the base image
import subprocess
@@ -46,23 +50,20 @@ def create_rootfs(d):
outfile = d.expand('${WORKDIR}/orig-rootfs-manifest.txt')
rootfs = d.getVar('IMAGE_ROOTFS', True)
# Generate a manifest of the current file contents
- # TODO: use the same common utility method
- manifest_cmd = 'cd %s && find . ! -path . > %s' % (rootfs, outfile)
- subprocess.call(manifest_cmd, shell=True, stderr=subprocess.STDOUT)
- # Remove the current rootfs contents
- oe.path.remove('%s/*' % rootfs)
- for entry in manifest_to_file_list(outfile):
- rootfs_contents.append(entry[2:])
+ create_content_manifests(rootfs, outfile, None, [])
+ rootfs_contents.update(manifest_to_file_list(outfile))
# clean up
os.unlink(outfile)
else: # non-base image, i.e. swupdimage
- manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/os-core${SWUPD_ROOTFS_MANIFEST_SUFFIX}")
- rootfs_contents.extend(manifest_to_file_list(manifest))
+ manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/os-core")
+ for suffix in suffixes:
+ rootfs_contents.update(manifest_to_file_list(manifest + suffix))
bb.debug(3, 'rootfs_contents has %s entries' % (len(rootfs_contents)))
for bundle in imagebundles:
- manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/%s${SWUPD_ROOTFS_MANIFEST_SUFFIX}") % bundle
- rootfs_contents.extend(manifest_to_file_list(manifest))
+ manifest = d.expand("${DEPLOY_DIR_SWUPD}/image/${OS_VERSION}/") + bundle
+ for suffix in suffixes:
+ rootfs_contents.update(manifest_to_file_list(manifest + suffix))
mega_rootfs = d.getVar('MEGA_IMAGE_ROOTFS', True)
bb.debug(2, 'Re-copying rootfs contents from mega image %s to %s' % (mega_rootfs, rootfs))
diff --git a/lib/swupd/utils.py b/lib/swupd/utils.py
index 0190821..0ffd9c5 100644
--- a/lib/swupd/utils.py
+++ b/lib/swupd/utils.py
@@ -13,6 +13,30 @@ def manifest_to_file_list(manifest_fn):
return image_manifest_list
+def create_content_manifests(dir, included, excluded, blacklist):
+ """
+ Iterate over the content of the directory, decide which entries are
+ included in the swupd update mechanism and write the full paths of the remaining
+ entries (without leading ./ or /) to the respective file. All directories
+ are explicitly listed.
+ """
+ bb.debug(3, 'Creating %s and %s from directory %s, excluding %s' %
+ (included, excluded, dir, blacklist))
+ cwd = os.getcwd()
+ try:
+ os.chdir(dir)
+ with open(included, 'w') as i:
+ with open(excluded or '/dev/null', 'w') as e:
+ for root, dirs, files in os.walk('.'):
+ # Strip the leading ./ that we get in root from os.walk('.').
+ root = root[2:]
+ for entry in sorted(dirs + files):
+ fullpath = os.path.join(root, entry)
+ out = e if blacklist and ('/' + fullpath) in blacklist else i
+ out.write(fullpath + '\n')
+ finally:
+ os.chdir(cwd)
+
def delta_contents(difflist):
"""
Generate a list of files which exist in the bundle image but not the base