aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-11-17 11:49:36 +0100
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:28:11 +0100
commit550c29471ca6654edca2b9b692d6a2b85159789b (patch)
tree1a0e5ade3febecf31a37df05971611b9f638e0cf
parent76d34f41347e782411f737b03acb7046b535bf7c (diff)
downloadmeta-swupd-550c29471ca6654edca2b9b692d6a2b85159789b.tar.gz
meta-swupd-550c29471ca6654edca2b9b692d6a2b85159789b.tar.bz2
meta-swupd-550c29471ca6654edca2b9b692d6a2b85159789b.zip
do_swupd_update: support format changes
There are reasons for format changes, the upcoming tools update being one of them. When the format changes, swupd-image.bbclass must build two OS versions from the same rootfs: once with the old format, once with the new format. OS_VERSION is used as number for the new format, OS_VERSION - 1 for the old one. OS_VERSION must be high enough such that OS_VERSION - 1 is still available. Usually it is, but there's also a sanity check for that. When changing the format because of a change in the tools, then both old and new swupd-server are needed, so now recipe and installed files include the tool format version. Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--classes/swupd-image.bbclass224
-rw-r--r--recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch16
-rw-r--r--recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch186
-rw-r--r--recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch56
-rw-r--r--recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch184
-rw-r--r--recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch118
-rw-r--r--recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch167
-rw-r--r--recipes-core/swupd-client/swupd-client_git.bb31
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch257
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch (renamed from recipes-core/swupd-server/swupd-server/0001-delta.c-fix-xattr-test-after-patching.patch)0
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch349
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch200
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch42
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch38
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch60
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch246
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch56
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch30
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch89
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch100
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch626
-rw-r--r--recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch (renamed from recipes-core/swupd-server/swupd-server/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch)0
-rw-r--r--recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch41
-rw-r--r--recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch86
-rw-r--r--recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch2
-rw-r--r--recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch2
-rw-r--r--recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch2
-rw-r--r--recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch37
-rw-r--r--recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch2
-rw-r--r--recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch91
-rw-r--r--recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch26
-rw-r--r--recipes-core/swupd-server/swupd-server_3.2.5.bb60
-rw-r--r--recipes-core/swupd-server/swupd-server_git.bb24
33 files changed, 2609 insertions, 839 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass
index 0cac12d..5ba9cfb 100644
--- a/classes/swupd-image.bbclass
+++ b/classes/swupd-image.bbclass
@@ -32,11 +32,47 @@ SWUPD_IMAGE_PN = "${@ d.getVar('PN_BASE', True) or d.getVar('PN', True)}"
# to be published will be in the "www" sub-directory.
DEPLOY_DIR_SWUPD = "${DEPLOY_DIR}/swupd/${MACHINE}/${SWUPD_IMAGE_PN}"
-# The current format has to match the the source code of the
-# swupd-client that is in the image. This recipe picks a suitable
-# swupd-client via the client's RPROVIDES.
-SWUPD_FORMAT ??= "3"
-IMAGE_INSTALL_append = " swupd-client-format${SWUPD_FORMAT}"
+# The "format" needs to be bumped for different reasons:
+# - the output of the swupd-server changes in a way that
+# a swupd-client currently installed on devices will not
+# understand it (example: changing file names or using
+# a new compression method for archives)
+# - the content of the distro changes such that a device
+# cannot update directly to the latest build (example:
+# the distro changes the boot loader and some swupd postinst
+# helper which knows about that change must be installed on
+# the device first before actually switching)
+#
+# meta-swupd handles the first case with SWUPD_TOOLS_FORMAT.
+# The default value matches the default versions of the swupd-server
+# and swupd-client. Distros can override this if they need to pick
+# non-default versions of the tools, but that is not tested.
+#
+# Distros need to handle the second case by preparing and releasing
+# a build that devices can update to (i.e. the version URL the devices
+# check must have that update), then make the incompatible change and
+# in the next build bump the SWUPD_DISTRO_FORMAT.
+#
+# In both cases, SWUPD_FORMAT gets bumped. meta-swupd notices that
+# and then prepares a special transitional update:
+# - the rootfs is configured to use the new SWUUPD_FORMAT and
+# OS_VERSION
+# - a fake OS_VERSION-1 release is built using a swupd-server that is
+# compatible with the swupd-client before the bump
+# - the OS_VERSION release then is the first one using the new format
+#
+# This way, devices are forced to update to OS_VERSION-1 because that
+# will forever be the "latest" version for their current format.
+# Once they have updated, the device really is on OS_VERSION, configured
+# to use the new format, and the next update check will see future
+# releases again.
+#
+# For this to work, "swupd-client" should always be invoked without
+# explicit format parameter.
+SWUPD_TOOLS_FORMAT ?= "4"
+SWUPD_DISTRO_FORMAT ?= "0"
+SWUPD_FORMAT = "${@ str(int('${SWUPD_TOOLS_FORMAT}') + int('${SWUPD_DISTRO_FORMAT}')) }"
+IMAGE_INSTALL_append = " swupd-client-format${SWUPD_TOOLS_FORMAT}"
# The information about where to find version information and actual
# content is needed in several places:
@@ -70,6 +106,11 @@ SWUPD_LOG_FN ??= "bbdebug 1"
# a non-negative integer that fits in an int.
OS_VERSION ??= "${DISTRO_VERSION}"
+# When doing format changes, this version number is used for the intermediate
+# release. Default is OS_VERSION - 1. There's a separate sanity check for
+# OS_VERSION below, so this code should always work.
+OS_VERSION_INTERIM ?= "${@ ${OS_VERSION} - 1 }"
+
# We need to preserve xattrs, which works with bsdtar out of the box.
# It also has saner file handling (less syscalls per file) than GNU tar.
# Last but not least, GNU tar 1.27.1 had weird problems extracting
@@ -325,6 +366,65 @@ do_swupd_update () {
fi
fi
+ swupd_format_of_version () {
+ if [ ! -f ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM ]; then
+ bbfatal "Cannot determine swupd format of $1, ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM not found."
+ exit 1
+ fi
+ format=`head -1 ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM | perl -n -e '/^MANIFEST\s(\d+)$/ && print $1'`
+ if [ ! "$format" ]; then
+ bbfatal "Cannot determine swupd format of $1, ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM does not have MANIFEST with format number in first line."
+ exit 1
+ fi
+ echo $format
+ }
+
+ # do_fetch_swupd_inputs() creates this file when a content
+ # URL was set, so creating an empty file shouldn't be necessary
+ # in most cases. Also determine whether we are switching
+ # formats.
+ #
+ # When the new format is different compared to what was used by
+ # latest.version, then swupd-server will automatically ignore
+ # the old content. That includes the case where tool format
+ # hasn't changed and only the distro format was bumped. In that
+ # case, reusing old content would be possible, but swupd-server
+ # would have to be improved to know that.
+ if [ -e ${DEPLOY_DIR_SWUPD}/image/latest.version ]; then
+ PREVREL=`cat ${DEPLOY_DIR_SWUPD}/image/latest.version`
+ if [ ! -e ${DEPLOY_DIR_SWUPD}/www/$PREVREL/Manifest.MoM ]; then
+ bbfatal "${DEPLOY_DIR_SWUPD}/image/latest.version specifies $PREVREL as last version, but there is no corresponding ${DEPLOY_DIR_SWUPD}/www/$PREVREL/Manifest.MoM."
+ exit 1
+ fi
+ PREVFORMAT=`swupd_format_of_version $PREVREL`
+ if [ ! "$PREVFORMAT" ]; then
+ bbfatal "Format number not found in first line of ${DEPLOY_DIR_SWUPD}/www/$PREVREL/Manifest.MoM"
+ exit 1
+ fi
+ # For now assume that SWUPD_DISTRO_FORMAT is always 0 and that thus
+ # $PREVFORMAT also is the format of the previous tools.
+ PREVTOOLSFORMAT=$PREVFORMAT
+
+ if [ $PREVFORMAT -ne ${SWUPD_FORMAT} ] && [ $PREVREL -ge ${OS_VERSION_INTERIM} ]; then
+ bbfatal "Building two releases because of a format change, so OS_VERSION - 1 = ${OS_VERSION_INTERIM} must be higher than last version $PREVREL."
+ elif [ $PREVREL -ge ${OS_VERSION} ]; then
+ bbfatal "OS_VERSION = ${OS_VERSION} must be higher than last version $PREVREL."
+ exit 1
+ fi
+ else
+ bbdebug 2 "Stubbing out empty latest.version file"
+ touch ${DEPLOY_DIR_SWUPD}/image/latest.version
+ PREVREL="0"
+ PREVFORMAT=${SWUPD_FORMAT}
+ PREVTOOLSFORMAT=${SWUPD_FORMAT}
+ fi
+
+ # swupd-server >= 3.2.8 uses a different name. Support old and new names
+ # via symlinking.
+ ln -sf latest.version ${DEPLOY_DIR_SWUPD}/image/LAST_VER
+
+ ${SWUPD_LOG_FN} "Generating update from $PREVREL (format $PREVFORMAT) to ${OS_VERSION} (format ${SWUPD_FORMAT})"
+
# Generate swupd-server configuration
bbdebug 2 "Writing ${DEPLOY_DIR_SWUPD}/server.ini"
if [ -e "${DEPLOY_DIR_SWUPD}/server.ini" ]; then
@@ -337,17 +437,6 @@ outputdir=${DEPLOY_DIR_SWUPD}/www/
emptydir=${DEPLOY_DIR_SWUPD}/empty/
END
- # do_fetch_swupd_inputs() creates this file when a content
- # URL was set, so creating an empty file shouldn't be necessary
- # in most cases.
- if [ -e ${DEPLOY_DIR_SWUPD}/image/latest.version ]; then
- PREVREL=`cat ${DEPLOY_DIR_SWUPD}/image/latest.version`
- else
- bbdebug 2 "Stubbing out empty latest.version file"
- touch ${DEPLOY_DIR_SWUPD}/image/latest.version
- PREVREL="0"
- fi
-
GROUPS_INI="${DEPLOY_DIR_SWUPD}/groups.ini"
bbdebug 2 "Writing ${GROUPS_INI}"
if [ -e "${DEPLOY_DIR_SWUPD}/groups.ini" ]; then
@@ -400,64 +489,91 @@ END
done
}
- ${SWUPD_LOG_FN} "Generating update from $PREVREL to ${OS_VERSION}"
- # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd
- invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_create_update --log-stdout -S ${DEPLOY_DIR_SWUPD} --osversion ${OS_VERSION} --format ${SWUPD_FORMAT}
-
- ${SWUPD_LOG_FN} "Generating fullfiles for ${OS_VERSION}"
- # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd
- invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles --log-stdout -S ${DEPLOY_DIR_SWUPD} ${OS_VERSION}
-
if [ "${SWUPD_CONTENT_URL}" ]; then
content_url_parameter="--content-url ${SWUPD_CONTENT_URL}"
else
content_url_parameter=""
fi
- ${SWUPD_LOG_FN} "Generating zero packs, this can take some time."
- # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd
- # Generating zero packs isn't parallelized internally. Mostly it just
- # spends its time compressing a single tar archive. Therefore we parallelize
- # by forking each command and then waiting for all of them to complete.
- jobs=""
- for bndl in ${ALL_BUNDLES}; do
- ${SWUPD_LOG_FN} "Generating zero pack for $bndl"
- # The zero packs are used by the swupd client when adding bundles.
- # The zero pack for os-core is not needed by the swupd client itself;
- # in Clear Linux OS it is used by the installer. We could use some
- # space by skipping the os-core zero bundle, but for now it gets
- # generated, just in case that it has some future use.
- invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} 0 ${OS_VERSION} $bndl &
- jobs="$jobs $!"
- done
+ create_version () {
+ swupd_format=$1
+ tool_format=$2
+ os_version=$3
+
+ # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd
+ invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_create_update_$tool_format --log-stdout -S ${DEPLOY_DIR_SWUPD} --osversion $os_version --format $swupd_format
+
+ ${SWUPD_LOG_FN} "Generating fullfiles for $os_version"
+ # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd
+ invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles_$tool_format --log-stdout -S ${DEPLOY_DIR_SWUPD} $os_version
- # Generate delta-packs against previous versions chosen by our caller.
- # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd
- for prevver in ${SWUPD_DELTAPACK_VERSIONS}; do
+ ${SWUPD_LOG_FN} "Generating zero packs, this can take some time."
+ # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd
+ # Generating zero packs isn't parallelized internally. Mostly it just
+ # spends its time compressing a single tar archive. Therefore we parallelize
+ # by forking each command and then waiting for all of them to complete.
+ jobs=""
for bndl in ${ALL_BUNDLES}; do
- bndlcnt=0
- ${SWUPD_LOG_FN} "Generating delta pack from $prevver to ${OS_VERSION} for $bndl"
- invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl | sed -u -e "s/^/$prevver $bndl: /" &
+ # The zero packs are used by the swupd client when adding bundles.
+ # The zero pack for os-core is not needed by the swupd client itself;
+ # in Clear Linux OS it is used by the installer. We could use some
+ # space by skipping the os-core zero bundle, but for now it gets
+ # generated, just in case that it has some future use.
+ invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack_$tool_format --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} 0 $os_version $bndl | sed -u -e "s/^/$bndl: /" &
jobs="$jobs $!"
done
- done
- waitall $jobs
+ # Generate delta-packs against previous versions chosen by our caller,
+ # if possible. Different formats make this useless because the previous
+ # version won't be able to update to the new version directly.
+ # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd
+ for prevver in ${SWUPD_DELTAPACK_VERSIONS}; do
+ old_swupd_format=`swupd_format_of_version $prevver`
+ if [ $old_swupd_format -eq $swupd_format ]; then
+ for bndl in ${ALL_BUNDLES}; do
+ ${SWUPD_LOG_FN} "Generating delta pack from $prevver to $os_version for $bndl"
+ invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack_$tool_format --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} $prevver $os_version $bndl | sed -u -e "s/^/$prevver $bndl: /" &
+ jobs="$jobs $!"
+ done
+ fi
+ done
+
+ waitall $jobs
- # Write version to www/version/format${SWUPD_FORMAT}/latest and image/latest.version
- bbdebug 2 "Writing latest file"
- mkdir -p ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}
- echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest
+ # Write version to www/version/format$swupd_format/latest.
+ bbdebug 2 "Writing latest file"
+ mkdir -p ${DEPLOY_DIR_SWUPD}/www/version/format$swupd_format
+ echo $os_version > ${DEPLOY_DIR_SWUPD}/www/version/format$swupd_format/latest
+ # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd
+ }
+
+ if [ $PREVFORMAT -ne ${SWUPD_FORMAT} ]; then
+ # Exact same content (including the OS_VERSION in the os-release file),
+ # just different tool and/or format in the manifests.
+ ln -sf ${OS_VERSION} ${DEPLOY_DIR_SWUPD}/image/${OS_VERSION_INTERIM}
+ echo $PREVREL > ${DEPLOY_DIR_SWUPD}/image/latest.version
+ create_version $PREVFORMAT $PREVTOOLSFORMAT ${OS_VERSION_INTERIM}
+ fi
+ echo $PREVREL > ${DEPLOY_DIR_SWUPD}/image/latest.version
+ create_version ${SWUPD_FORMAT} ${SWUPD_TOOLS_FORMAT} ${OS_VERSION}
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version
- # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd
}
SWUPDDEPENDS = "\
virtual/fakeroot-native:do_populate_sysroot \
rsync-native:do_populate_sysroot \
bsdiff-native:do_populate_sysroot \
- swupd-server-native:do_populate_sysroot \
"
+
+# We don't know exactly which formats will be in use during
+# do_swupd_update. It depends on the content of the update
+# repo, which is unavailable when dependencies are evaluated
+# in preparation of the build.
+#
+# For now we simply build all supported server versions.
+SWUPD_SERVER_FORMATS = "3 4"
+SWUPDDEPENDS += "${@ ' '.join(['swupd-server-format%s-native:do_populate_sysroot' % x for x in '${SWUPD_SERVER_FORMATS}'.split()])}"
+
addtask swupd_update after do_image_complete before do_build
do_swupd_update[depends] = "${SWUPDDEPENDS}"
diff --git a/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch b/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch
index 649f7da..06f138c 100644
--- a/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch
+++ b/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch
@@ -1,22 +1,24 @@
-From d648898c8f9823a8d511507f93390079954bc24a Mon Sep 17 00:00:00 2001
+From 72bcbe256a6612954ea24175538660864e65e26d Mon Sep 17 00:00:00 2001
From: Joshua Lock <joshua.g.lock@intel.com>
Date: Mon, 18 Apr 2016 13:30:18 +0100
-Subject: [PATCH] Add configure option to re-enable updating of config files
+Subject: [PATCH 2/4] Add configure option to re-enable updating of config
+ files
Upstream-Status: Pending
Signed-off-by: Joshua Lock <joshua.g.lock@intel.com>
+
---
configure.ac | 7 +++++++
src/heuristics.c | 5 +++--
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/configure.ac b/configure.ac
-index b1905f0..4a5c029 100644
+index ee05258..5cbc450 100644
--- a/configure.ac
+++ b/configure.ac
-@@ -63,6 +63,13 @@ AS_IF([test "x$enable_bsdtar" = "xyes" ],
- [AC_DEFINE(SWUPD_WITHOUT_BSDTAR, 1, [Use default tar])]
+@@ -153,6 +153,13 @@ AS_IF([test "x$enable_tar_selinux" = "xyes"],
+ [TARSELINUX=no]
)
+AC_ARG_ENABLE(
@@ -30,7 +32,7 @@ index b1905f0..4a5c029 100644
[path to systemd system service dir @<:@default=/usr/lib/systemd/system@:>@]), [unitpath=${withval}],
[unitpath="$($PKG_CONFIG --variable=systemdsystemunitdir systemd)"])
diff --git a/src/heuristics.c b/src/heuristics.c
-index e7c0632..c488e17 100644
+index 1b5d6c9..a2615f3 100644
--- a/src/heuristics.c
+++ b/src/heuristics.c
@@ -27,6 +27,7 @@
@@ -53,5 +55,5 @@ index e7c0632..c488e17 100644
is_state(file->filename) || // ideally we trust the manifest but short term reapply check here
(file->is_boot && file->is_deleted) ||
--
-2.5.5
+2.1.4
diff --git a/recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch b/recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch
deleted file mode 100644
index 59875b7..0000000
--- a/recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch
+++ /dev/null
@@ -1,186 +0,0 @@
-From 9bc713b7ed0dba91304c5d7ed4905f5924ad8e42 Mon Sep 17 00:00:00 2001
-From: Patrick Ohly <patrick.ohly@intel.com>
-Date: Thu, 14 Apr 2016 11:03:31 +0200
-Subject: [PATCH 1/3] downloads: minimize syscalls to improve performance
-
-The previous approach was to open/fdopen/fclose the file for each
-chunk that gets passed from curl. This incurrs a huge performance hit
-when close() triggers a hashing of the file content on systems where
-integrity protection via IMA is enabled.
-
-Now the file is opened only once and kept open until the download is
-complete. In addition, the unnecessary usage of C file IO is avoided.
-
-The semantic is changed as little as possible:
-- file gets created only after the first chunk of data arrived
-- file descriptors do not leak to child processes (O_CLOEXEC)
-- data gets appended to existing files (via O_APPEND, used
- to keep the code simple and avoid an additional lseek)
-- data gets flushed explicitly for each chunk (via fdatasync(),
- which somewhat approximates the effect that an explicit
- close() may have had)
-
-As an additional improvement, failures during close() are checked. To
-keep error handling as much as before, the completion function which has
-the close() takes the current curl error code and replaces it if it
-encounters a write error.
-
-[v2 of the patch with fixes by Dmitry Rozhkov, see https://github.com/pohly/swupd-client/pull/1]
-
-Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
----
- include/swupd.h | 3 +++
- src/curl.c | 63 ++++++++++++++++++++++++++++++++++++++++-----------------
- src/download.c | 8 +++++++-
- 3 files changed, 54 insertions(+), 20 deletions(-)
-
-diff --git a/include/swupd.h b/include/swupd.h
-index 14e65ab..3bac8b2 100644
---- a/include/swupd.h
-+++ b/include/swupd.h
-@@ -89,6 +89,7 @@ struct file {
- int last_change;
- struct update_stat stat;
-
-+ unsigned int fd_valid : 1;
- unsigned int is_dir : 1;
- unsigned int is_file : 1;
- unsigned int is_link : 1;
-@@ -109,6 +110,7 @@ struct file {
-
- char *staging; /* output name used during download & staging */
- CURL *curl; /* curl handle if downloading */
-+ int fd; /* file written into during downloading, unset when fd_valid is false */
- };
-
- extern bool download_only;
-@@ -199,6 +201,7 @@ extern void swupd_curl_set_current_version(int v);
- extern void swupd_curl_set_requested_version(int v);
- extern double swupd_query_url_content_size(char *url);
- extern size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata);
-+extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file);
- extern int swupd_curl_get_file(const char *url, char *filename, struct file *file,
- struct version_container *tmp_version, bool pack);
- #define SWUPD_CURL_LOW_SPEED_LIMIT 1
-diff --git a/src/curl.c b/src/curl.c
-index b14193b..cab1ef2 100644
---- a/src/curl.c
-+++ b/src/curl.c
-@@ -165,35 +165,57 @@ size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata)
- const char *outfile;
- int fd;
- FILE *f;
-- size_t written;
-+ size_t written, remaining;
-
- outfile = file->staging;
-+ if (file->fd_valid) {
-+ fd = file->fd;
-+ } else {
-+ fd = open(outfile, O_CREAT | O_RDWR | O_CLOEXEC | O_APPEND, 00600);
-+ if (fd < 0) {
-+ fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
-+ outfile, strerror(errno));
-+ return -1;
-+ }
-+ file->fd = fd;
-+ file->fd_valid = 1;
-+ }
-
-- fd = open(outfile, O_CREAT | O_RDWR, 00600);
-- if (fd < 0) {
-- printf("Error: Cannot open %s for write: %s\n",
-- outfile, strerror(errno));
-- return -1;
-+ /* handle short writes with repeated write() calls */
-+ for (remaining = size * nmemb; remaining; remaining -= written) {
-+ written = write(fd, ptr, size*nmemb);
-+ if (written < 0) {
-+ if (errno == EINTR) {
-+ written = 0;
-+ continue;
-+ }
-+ fprintf(stderr, "write error \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
-+ outfile, strerror(errno));
-+ return -1;
-+ }
- }
-
-- f = fdopen(fd, "a");
-- if (!f) {
-- printf("Error: Cannot fdopen %s for write: %s\n",
-- outfile, strerror(errno));
-- close(fd);
-+ if (fdatasync(fd)) {
-+ fprintf(stderr, "fdatasync \\*outfile=\"%s\",strerror=\"%s\"*\\\n", outfile, strerror(errno));
- return -1;
- }
-
-- written = fwrite(ptr, size * nmemb, 1, f);
--
-- fflush(f);
-- fclose(f);
-+ return size*nmemb;
-+}
-
-- if (written != 1) {
-- return -1;
-+CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file)
-+{
-+ if (file->fd_valid) {
-+ if (close(file->fd)) {
-+ fprintf(stderr, "Cannot close file after write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
-+ file->staging, strerror(errno));
-+ if (curl_ret == CURLE_OK) {
-+ curl_ret = CURLE_WRITE_ERROR;
-+ }
-+ }
-+ file->fd_valid = 0;
- }
--
-- return size * nmemb;
-+ return curl_ret;
- }
-
- /* Download a single file SYNCHRONOUSLY
-@@ -281,6 +303,9 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file,
- }
-
- exit:
-+ if (local) {
-+ curl_ret = swupd_download_file_complete(curl_ret, local);
-+ }
- if (curl_ret == CURLE_OK) {
- /* curl command succeeded, download might've failed, let our caller handle */
- switch (ret) {
-diff --git a/src/download.c b/src/download.c
-index 6d81d81..c4a7a07 100644
---- a/src/download.c
-+++ b/src/download.c
-@@ -164,6 +164,7 @@ static void free_curl_list_data(void *data)
- {
- struct file *file = (struct file *)data;
- CURL *curl = file->curl;
-+ (void) swupd_download_file_complete(CURLE_OK, file);
- if (curl != NULL) {
- curl_multi_remove_handle(mcurl, curl);
- curl_easy_cleanup(curl);
-@@ -368,9 +369,14 @@ static int perform_curl_io_and_complete(int *left)
- continue;
- }
-
-+ /* Get error code from easy handle and augment it if
-+ * completing the download encounters further problems. */
-+ curl_ret = msg->data.result;
-+ curl_ret = swupd_download_file_complete(curl_ret, file);
-+
- /* The easy handle may have an error set, even if the server returns
- * HTTP 200, so retry the download for this case. */
-- if (ret == 200 && msg->data.result != CURLE_OK) {
-+ if (ret == 200 && curl_ret != CURLE_OK) {
- printf("Error for %s download: %s\n", file->hash,
- curl_easy_strerror(msg->data.result));
- failed = list_prepend_data(failed, file);
---
-2.1.4
-
diff --git a/recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch b/recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch
new file mode 100644
index 0000000..b3539df
--- /dev/null
+++ b/recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch
@@ -0,0 +1,56 @@
+From a8c7a7a93e19b328a1a6e8114f21b2bff9ad4f69 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Thu, 17 Nov 2016 18:00:46 +0100
+Subject: [PATCH] fix --enable-xattr
+
+Commit fc0f570d added a check that prevents using --enable-xattr
+together with --enable-bsdtar, perhaps because it was assumed that this
+wouldn't work because there is no special tar option as in the GNU tar
+case.
+
+But that combination works fine for (and is needed by) Ostro OS with
+IMA and Smack xattrs, so the check needs to be removed.
+
+Besides that, enabling xattrs also had no effect because xattrs.c
+never got to see the SWUPD_WITH_XATTRS define due to not including
+config.h.
+
+Upstream-Status: Backported [https://github.com/clearlinux/swupd-client/commit/a2b80dc6958b78885ec395f22d34996e71a5f58a]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ configure.ac | 5 +----
+ src/xattrs.c | 1 +
+ 2 files changed, 2 insertions(+), 4 deletions(-)
+
+diff --git a/configure.ac b/configure.ac
+index 5cbc450..2166fd0 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -135,10 +135,7 @@ AC_ARG_ENABLE(
+ AS_HELP_STRING([--enable-xattr],[Use extended file attributes (unused by default)])
+ )
+ AS_IF([test "x$enable_xattr" = "xyes"],
+- [AC_DEFINE(SWUPD_WITH_XATTRS,1,[Use extended file attributes])
+- AS_IF(test "x$enable_bsdtar" = "xyes",
+- echo "Options --enable-bsdtar and --enable-xattr are incompatible" >&2
+- AS_EXIT(1))],
++ [AC_DEFINE(SWUPD_WITH_XATTRS,1,[Use extended file attributes])],
+ [XATTR=no]
+ )
+ TARSELINUX="yes"
+diff --git a/src/xattrs.c b/src/xattrs.c
+index 36d4241..bbca237 100644
+--- a/src/xattrs.c
++++ b/src/xattrs.c
+@@ -27,6 +27,7 @@
+ #include <string.h>
+ #include <sys/xattr.h>
+
++#include "config.h"
+ #include "swupd.h"
+ #include "xattrs.h"
+
+--
+2.1.4
+
diff --git a/recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch b/recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch
deleted file mode 100644
index 4d8339a..0000000
--- a/recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch
+++ /dev/null
@@ -1,184 +0,0 @@
-From 26c603ad25469d3e37fc00b78ad161b34093f5fa Mon Sep 17 00:00:00 2001
-From: Patrick Ohly <patrick.ohly@intel.com>
-Date: Tue, 15 Nov 2016 15:01:38 +0100
-Subject: [PATCH 2/3] downloads: open FILE in advance and use default write
- handler
-
-Now that the number of pending downloads is kept below a certain limit
-(see poll_fewer_than()) it is possible to open files before starting
-the transfer. Using the default curl write handler and explicit
-open/close of the file makes the code simpler.
-
-Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
----
- include/swupd.h | 5 ++---
- src/curl.c | 59 +++++++++++++--------------------------------------------
- src/download.c | 6 ++++--
- 3 files changed, 19 insertions(+), 51 deletions(-)
-
-diff --git a/include/swupd.h b/include/swupd.h
-index 3bac8b2..ed4b82c 100644
---- a/include/swupd.h
-+++ b/include/swupd.h
-@@ -89,7 +89,6 @@ struct file {
- int last_change;
- struct update_stat stat;
-
-- unsigned int fd_valid : 1;
- unsigned int is_dir : 1;
- unsigned int is_file : 1;
- unsigned int is_link : 1;
-@@ -110,7 +109,7 @@ struct file {
-
- char *staging; /* output name used during download & staging */
- CURL *curl; /* curl handle if downloading */
-- int fd; /* file written into during downloading, unset when fd_valid is false */
-+ FILE *fh; /* file written into during downloading */
- };
-
- extern bool download_only;
-@@ -200,7 +199,7 @@ extern void swupd_curl_cleanup(void);
- extern void swupd_curl_set_current_version(int v);
- extern void swupd_curl_set_requested_version(int v);
- extern double swupd_query_url_content_size(char *url);
--extern size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata);
-+extern CURLcode swupd_download_file_start(struct file *file);
- extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file);
- extern int swupd_curl_get_file(const char *url, char *filename, struct file *file,
- struct version_container *tmp_version, bool pack);
-diff --git a/src/curl.c b/src/curl.c
-index cab1ef2..009fdd5 100644
---- a/src/curl.c
-+++ b/src/curl.c
-@@ -158,62 +158,28 @@ static size_t swupd_download_version_to_memory(void *ptr, size_t size, size_t nm
- return data_len;
- }
-
--/* curl easy CURLOPT_WRITEFUNCTION callback */
--size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata)
-+CURLcode swupd_download_file_start(struct file *file)
- {
-- struct file *file = (struct file *)userdata;
-- const char *outfile;
-- int fd;
-- FILE *f;
-- size_t written, remaining;
--
-- outfile = file->staging;
-- if (file->fd_valid) {
-- fd = file->fd;
-- } else {
-- fd = open(outfile, O_CREAT | O_RDWR | O_CLOEXEC | O_APPEND, 00600);
-- if (fd < 0) {
-- fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
-- outfile, strerror(errno));
-- return -1;
-- }
-- file->fd = fd;
-- file->fd_valid = 1;
-- }
--
-- /* handle short writes with repeated write() calls */
-- for (remaining = size * nmemb; remaining; remaining -= written) {
-- written = write(fd, ptr, size*nmemb);
-- if (written < 0) {
-- if (errno == EINTR) {
-- written = 0;
-- continue;
-- }
-- fprintf(stderr, "write error \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
-- outfile, strerror(errno));
-- return -1;
-- }
-+ file->fh = fopen(file->staging, "w");
-+ if (!file->fh) {
-+ fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
-+ file->staging, strerror(errno));
-+ return CURLE_WRITE_ERROR;
- }
--
-- if (fdatasync(fd)) {
-- fprintf(stderr, "fdatasync \\*outfile=\"%s\",strerror=\"%s\"*\\\n", outfile, strerror(errno));
-- return -1;
-- }
--
-- return size*nmemb;
-+ return CURLE_OK;
- }
-
- CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file)
- {
-- if (file->fd_valid) {
-- if (close(file->fd)) {
-+ if (file->fh) {
-+ if (fclose(file->fh)) {
- fprintf(stderr, "Cannot close file after write \\*outfile=\"%s\",strerror=\"%s\"*\\\n",
- file->staging, strerror(errno));
- if (curl_ret == CURLE_OK) {
- curl_ret = CURLE_WRITE_ERROR;
- }
- }
-- file->fd_valid = 0;
-+ file->fh = NULL;
- }
- return curl_ret;
- }
-@@ -246,6 +212,7 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file,
-
- if (file) {
- local = file;
-+ local->fh = NULL;
- } else {
- local = calloc(1, sizeof(struct file));
- if (!local) {
-@@ -266,11 +233,11 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file,
- if (curl_ret != CURLE_OK) {
- goto exit;
- }
-- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, swupd_download_file);
-+ curl_ret = swupd_download_file_start(local);
- if (curl_ret != CURLE_OK) {
- goto exit;
- }
-- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)local);
-+ curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)local->fh);
- if (curl_ret != CURLE_OK) {
- goto exit;
- }
-diff --git a/src/download.c b/src/download.c
-index c4a7a07..397d56c 100644
---- a/src/download.c
-+++ b/src/download.c
-@@ -475,6 +475,7 @@ void full_download(struct file *file)
- CURLMcode curlm_ret = CURLM_OK;
- CURLcode curl_ret = CURLE_OK;
-
-+ file->fh = NULL;
- ret = swupd_curl_hashmap_insert(file);
- if (ret > 0) { /* no download needed */
- /* File already exists - report success */
-@@ -510,11 +511,11 @@ void full_download(struct file *file)
- if (curl_ret != CURLE_OK) {
- goto out_bad;
- }
-- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, swupd_download_file);
-+ curl_ret = swupd_download_file_start(file);
- if (curl_ret != CURLE_OK) {
- goto out_bad;
- }
-- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file);
-+ curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file->fh);
- if (curl_ret != CURLE_OK) {
- goto out_bad;
- }
-@@ -536,6 +537,7 @@ void full_download(struct file *file)
- goto out_good;
-
- out_bad:
-+ (void) swupd_download_file_complete(CURLE_OK, file);
- failed = list_prepend_data(failed, file);
- if (curl != NULL) {
- /* Must remove handle out of multi queue first!*/
---
-2.1.4
-
diff --git a/recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch b/recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch
deleted file mode 100644
index 4326a58..0000000
--- a/recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch
+++ /dev/null
@@ -1,118 +0,0 @@
-From 6792cfef0ebfbe83e41bc81df6bc675604d7c943 Mon Sep 17 00:00:00 2001
-From: Dmitry Rozhkov <dmitry.rozhkov@linux.intel.com>
-Date: Tue, 16 Aug 2016 10:55:15 +0300
-Subject: [PATCH] Make pinned pubkey configurable
-
-The server may move to a new location where a different pubkey
-needs to be used and the hardcoded one won't work.
-
-This patch makes pinned pubkey configurable.
-
-Upstream-Status: Submitted [https://github.com/clearlinux/swupd-client/pull/110]
-
-Signed-off-by: Dmitry Rozhkov <dmitry.rozhkov@linux.intel.com>
----
- configure.ac | 2 ++
- include/swupd.h | 1 +
- src/curl.c | 2 +-
- src/globals.c | 14 ++++++++++++++
- 4 files changed, 18 insertions(+), 1 deletion(-)
-
-diff --git a/configure.ac b/configure.ac
-index 83007aa..883553a 100644
---- a/configure.ac
-+++ b/configure.ac
-@@ -114,6 +114,7 @@ AH_TEMPLATE([LOCK_DIR],[Directory for lock file])
- AH_TEMPLATE([BUNDLES_DIR],[Directory to use for bundles])
- AH_TEMPLATE([UPDATE_CA_CERTS_PATH],[Location of CA certificates])
- AH_TEMPLATE([MOTD_FILE],[motd file path])
-+AH_TEMPLATE([PINNED_PUBKEY_PATH],[Path to pinned public key])
-
- if test "$enable_linux_rootfs_build" = "yes"; then
- AC_DEFINE([SWUPD_LINUX_ROOTFS],1)
-@@ -124,6 +125,7 @@ if test "$enable_linux_rootfs_build" = "yes"; then
- AC_DEFINE([BUNDLES_DIR],["/usr/share/clear/bundles"])
- AC_DEFINE_UNQUOTED([UPDATE_CA_CERTS_PATH],["$certs_path"])
- AC_DEFINE([MOTD_FILE],["/usr/lib/motd.d/001-new-release"])
-+ AC_DEFINE_UNQUOTED([PINNED_PUBKEY_PATH],["${certs_path}/425b0f6b.key"])
- else
- AC_MSG_ERROR([Unknown build variant])
- fi
-diff --git a/include/swupd.h b/include/swupd.h
-index 5c722d3..3658dc8 100644
---- a/include/swupd.h
-+++ b/include/swupd.h
-@@ -132,6 +132,7 @@ extern void *tm_dlhandle;
- extern char *bundle_to_add;
- extern struct timeval start_time;
- extern char *state_dir;
-+extern char *pinned_pubkey_path;
-
- extern char *version_url;
- extern char *content_url;
-diff --git a/src/curl.c b/src/curl.c
-index 6b6099f..b14193b 100644
---- a/src/curl.c
-+++ b/src/curl.c
-@@ -447,7 +447,7 @@ static CURLcode swupd_curl_set_security_opts(CURL *curl)
- goto exit;
- }
-
-- curl_ret = curl_easy_setopt(curl, CURLOPT_PINNEDPUBLICKEY, "/usr/share/clear/update-ca/425b0f6b.key");
-+ curl_ret = curl_easy_setopt(curl, CURLOPT_PINNEDPUBLICKEY, pinned_pubkey_path);
- if (curl_ret != CURLE_OK) {
- goto exit;
- }
-diff --git a/src/globals.c b/src/globals.c
-index f2f1200..d0858df 100644
---- a/src/globals.c
-+++ b/src/globals.c
-@@ -51,6 +51,7 @@ char *mounted_dirs = NULL;
- char *bundle_to_add = NULL;
- struct timeval start_time;
- char *state_dir = NULL;
-+char *pinned_pubkey_path = NULL;
-
- /* NOTE: Today the content and version server urls are the same in
- * all cases. It is highly likely these will eventually differ, eg:
-@@ -68,6 +69,7 @@ long update_server_port = -1;
- static const char *default_version_url_path = "/usr/share/defaults/swupd/versionurl";
- static const char *default_content_url_path = "/usr/share/defaults/swupd/contenturl";
- static const char *default_format_path = "/usr/share/defaults/swupd/format";
-+static const char *default_pinnedpubkey_path = "/usr/share/defaults/swupd/pinnedpubkey";
-
- static int set_default_value(char **global, const char *path)
- {
-@@ -194,6 +196,16 @@ bool set_state_dir(char *path)
- return true;
- }
-
-+void set_pinned_pubkey_path()
-+{
-+ int ret;
-+
-+ ret = set_default_value(&pinned_pubkey_path, default_pinnedpubkey_path);
-+ if (ret < 0) {
-+ string_or_die(&pinned_pubkey_path, "%s", PINNED_PUBKEY_PATH);
-+ }
-+}
-+
- bool set_format_string(char *userinput)
- {
- int ret;
-@@ -322,6 +334,7 @@ bool init_globals(void)
- (void)set_format_string(NULL);
- set_version_url(NULL);
- set_content_url(NULL);
-+ set_pinned_pubkey_path();
-
- /* must set this global after version_url and content_url */
- set_local_download();
-@@ -337,6 +350,7 @@ void free_globals(void)
- free(format_string);
- free(mounted_dirs);
- free(state_dir);
-+ free(pinned_pubkey_path);
- if (bundle_to_add != NULL) {
- free(bundle_to_add);
- }
diff --git a/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch b/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch
index 7410b1d..b3dd47b 100644
--- a/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch
+++ b/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch
@@ -1,7 +1,7 @@
-From c16e1e7fc16933669ed4be63858edd4082509183 Mon Sep 17 00:00:00 2001
+From cc44bbfb2eaa90284a67ad6d42706e6433abd7ff Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Thu, 3 Nov 2016 11:47:53 +0100
-Subject: [PATCH] ignore xattrs when verifying Manifest files
+Subject: [PATCH 1/3] verify_file: ignore xattrs when verifying Manifest files
When IMA or Smack are active on the client, the downloaded Manifest
files will be assigned certain xattrs (security.ima
@@ -12,170 +12,33 @@ Manifest hashes even if they existed (see write_manifest_plain() in
src/manifest.c).
Therefore the client must ignore xattrs when verifying Manifest files.
-This is the only place where verification gets relaxed. All other locations
-still use xattrs, just as before.
+
+Upstream-Status: Backported [https://github.com/clearlinux/swupd-client/commit/09c26658d346cdd80ea54188d991db3493983176]
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
- include/swupd.h | 2 +-
- src/delta.c | 2 +-
- src/download.c | 6 +++---
- src/hash.c | 6 +++---
- src/helpers.c | 2 +-
- src/manifest.c | 2 +-
- src/scripts.c | 2 +-
- src/verify.c | 4 ++--
- 8 files changed, 13 insertions(+), 13 deletions(-)
+ src/hash.c | 9 ++++++++-
-diff --git a/include/swupd.h b/include/swupd.h
-index e1e1f3d..14e65ab 100644
---- a/include/swupd.h
-+++ b/include/swupd.h
-@@ -225,7 +225,7 @@ extern struct list *recurse_manifest(struct manifest *manifest, const char *comp
- extern struct list *consolidate_files(struct list *files);
- extern void debug_write_manifest(struct manifest *manifest, char *filename);
- extern void populate_file_struct(struct file *file, char *filename);
--extern bool verify_file(struct file *file, char *filename);
-+extern bool verify_file(struct file *file, char *filename, bool use_xattrs);
- extern int verify_bundle_hash(struct manifest *manifest, struct file *bundle);
- extern void unlink_all_staged_content(struct file *file);
- extern void link_renames(struct list *newfiles, struct manifest *from_manifest);
-diff --git a/src/delta.c b/src/delta.c
-index 8172b67..317adde 100644
---- a/src/delta.c
-+++ b/src/delta.c
-@@ -109,7 +109,7 @@ static void do_delta(struct file *file)
- }
- xattrs_copy(origin, filename);
-
-- if (!verify_file(file, filename)) {
-+ if (!verify_file(file, filename, true)) {
- unlink_all_staged_content(file);
- goto out;
- }
-diff --git a/src/download.c b/src/download.c
-index 9ea957d..6d81d81 100644
---- a/src/download.c
-+++ b/src/download.c
-@@ -98,7 +98,7 @@ static int swupd_curl_hashmap_insert(struct file *file)
- string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash);
-
- if (lstat(targetfile, &stat) == 0) {
-- if (verify_file(file, targetfile)) {
-+ if (verify_file(file, targetfile, true)) {
- free(targetfile);
- pthread_mutex_unlock(&bucket->mutex);
- return 1;
-@@ -260,7 +260,7 @@ int untar_full_download(void *data)
- * NOTE: this should NEVER happen given the checking that happens
- * ahead of queueing a download. But... */
- if (lstat(targetfile, &stat) == 0) {
-- if (verify_file(file, targetfile)) {
-+ if (verify_file(file, targetfile, true)) {
- unlink(tar_dotfile);
- unlink(tarfile);
- free(tar_dotfile);
-@@ -316,7 +316,7 @@ int untar_full_download(void *data)
- }
-
- err = lstat(targetfile, &stat);
-- if (!err && !verify_file(file, targetfile)) {
-+ if (!err && !verify_file(file, targetfile, true)) {
- /* Download was successful but the hash was bad. This is fatal*/
- printf("Error: File content hash mismatch for %s (bad server data?)\n", targetfile);
- exit(EXIT_FAILURE);
diff --git a/src/hash.c b/src/hash.c
-index 34da6eb..00a6802 100644
+index 1e61454..9553644 100644
--- a/src/hash.c
+++ b/src/hash.c
-@@ -226,7 +226,7 @@ int compute_hash(struct file *file, char *filename)
- return 0;
- }
-
--bool verify_file(struct file *file, char *filename)
-+bool verify_file(struct file *file, char *filename, bool use_xattrs)
- {
- struct file *local = calloc(1, sizeof(struct file));
-
-@@ -235,7 +235,7 @@ bool verify_file(struct file *file, char *filename)
+@@ -236,7 +236,14 @@ bool verify_file(struct file *file, char *filename)
}
local->filename = file->filename;
- local->use_xattrs = true;
-+ local->use_xattrs = use_xattrs;
++ /*
++ * xattrs are currently not supported for manifest files.
++ * They are data files produced by the swupd-server and
++ * therefore do not have any of the xattrs normally
++ * set for the actual system files (like security.ima
++ * when using IMA or security.SMACK64 when using Smack).
++ */
++ local->use_xattrs = !file->is_manifest;
populate_file_struct(local, filename);
if (compute_hash(local, filename) != 0) {
-@@ -275,7 +275,7 @@ int verify_bundle_hash(struct manifest *manifest, struct file *bundle)
- string_or_die(&local, "%s/%i/Manifest.%s", state_dir,
- current->last_change, current->filename);
-
-- if (!verify_file(bundle, local)) {
-+ if (!verify_file(bundle, local, false)) {
- printf("Warning: hash check failed for Manifest.%s\n",
- current->filename);
- ret = 0;
-diff --git a/src/helpers.c b/src/helpers.c
-index e71688c..01fd4a3 100644
---- a/src/helpers.c
-+++ b/src/helpers.c
-@@ -787,7 +787,7 @@ int verify_fix_path(char *targetpath, struct manifest *target_MoM)
-
- ret = stat(target, &sb);
- if (ret == 0) {
-- if (verify_file(file, target)) {
-+ if (verify_file(file, target, true)) {
- continue;
- }
- printf("Hash did not match for path : %s\n", path);
-diff --git a/src/manifest.c b/src/manifest.c
-index 2b57d3d..ee6d29a 100644
---- a/src/manifest.c
-+++ b/src/manifest.c
-@@ -674,7 +674,7 @@ struct list *create_update_list(struct manifest *current, struct manifest *serve
- if (fullname == NULL) {
- abort();
- }
-- if (verify_file(file, fullname)) {
-+ if (verify_file(file, fullname, true)) {
- free(fullname);
- continue;
- }
-diff --git a/src/scripts.c b/src/scripts.c
-index 59417af..c2157f7 100644
---- a/src/scripts.c
-+++ b/src/scripts.c
-@@ -127,7 +127,7 @@ void run_preupdate_scripts(struct manifest *manifest)
- }
-
- /* Check that system file matches file in manifest */
-- if (verify_file(file, script)) {
-+ if (verify_file(file, script, true)) {
- system(script);
- break;
- }
-diff --git a/src/verify.c b/src/verify.c
-index 1514988..eaf9dd8 100644
---- a/src/verify.c
-+++ b/src/verify.c
-@@ -462,7 +462,7 @@ static void deal_with_hash_mismatches(struct manifest *official_manifest, bool r
- if (fullname == NULL) {
- abort();
- }
-- if (verify_file(file, fullname)) {
-+ if (verify_file(file, fullname, true)) {
- free(fullname);
- continue;
- } else {
-@@ -483,7 +483,7 @@ static void deal_with_hash_mismatches(struct manifest *official_manifest, bool r
- }
-
- /* at the end of all this, verify the hash again to judge success */
-- if (verify_file(file, fullname)) {
-+ if (verify_file(file, fullname, true)) {
- file_fixed_count++;
- printf("\tfixed\n");
- } else {
--
2.1.4
diff --git a/recipes-core/swupd-client/swupd-client_git.bb b/recipes-core/swupd-client/swupd-client_git.bb
index fd73223..d603cb0 100644
--- a/recipes-core/swupd-client/swupd-client_git.bb
+++ b/recipes-core/swupd-client/swupd-client_git.bb
@@ -5,17 +5,14 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=f8d90fb802930e30e49c39c8126a959e"
DEPENDS = "glib-2.0 curl openssl libarchive bsdiff"
-PV = "3.6.0+git${SRCPV}"
-SRC_URI = "\
- git://github.com/clearlinux/swupd-client.git;protocol=https \
- file://Change-systemctl-path-to-OE-systemctl-path.patch \
- file://0001-Add-configure-option-to-re-enable-updating-of-config.patch \
- file://Make-pinned-pubkey-configurable.patch \
- file://ignore-xattrs-when-verifying-Manifest-files.patch \
- file://0001-downloads-minimize-syscalls-to-improve-performance.patch \
- file://0002-downloads-open-FILE-in-advance-and-use-default-write.patch \
-"
-SRCREV = "f4000c5b22be47ec1af2f8748fd71a36148b5dc4"
+PV = "3.7.2+git${SRCPV}"
+SRC_URI = "git://github.com/clearlinux/swupd-client.git;protocol=https \
+ file://Change-systemctl-path-to-OE-systemctl-path.patch \
+ file://0001-Add-configure-option-to-re-enable-updating-of-config.patch \
+ file://ignore-xattrs-when-verifying-Manifest-files.patch \
+ file://0001-fix-enable-xattr.patch \
+ "
+SRCREV = "b43ad9748ea690f69f924b6cae9a83c3886514b6"
S = "${WORKDIR}/git"
@@ -30,18 +27,18 @@ RRECOMMENDS_${PN}_class-target = "os-release"
# and bump the number by one for each update of the recipe where we
# switch to a source that has a format change.
#
-# To switch to a client with a new format also update SWUPD_FORMAT in
+# To switch to a client with a new format also update SWUPD_TOOLS_FORMAT in
# swupd-image.bbclass.
-RPROVIDES_${PN} = "swupd-client-format3"
+SWUPD_CLIENT_FORMAT = "4"
+RPROVIDES_${PN} = "swupd-client-format${SWUPD_CLIENT_FORMAT}"
-# TODO: we inherit autotools-brokensep because the Makefile calls a perl script
-# in ${S} during one of its steps.
-inherit pkgconfig autotools-brokensep systemd
+inherit pkgconfig autotools systemd
EXTRA_OECONF = "\
--with-systemdsystemunitdir=${systemd_system_unitdir} \
--enable-bsdtar \
--disable-tests \
+ --enable-xattr \
"
PACKAGECONFIG ??= "stateless"
@@ -53,7 +50,7 @@ FILES_${PN} += "\
/var/lib/swupd \
"
-SYSTEMD_SERVICE_${PN} = "check-update.timer check-update.service"
+SYSTEMD_SERVICE_${PN} = "check-update.timer check-update.service swupd-update.timer swupd-update.service"
SYSTEMD_AUTO_ENABLE_${PN} = "disable"
BBCLASSEXTEND = "native"
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch
new file mode 100644
index 0000000..fd91b78
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch
@@ -0,0 +1,257 @@
+From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Wed, 16 Nov 2016 14:26:30 +0100
+Subject: [PATCH 1/3] create_pack: rely less on previous builds
+
+When a file has not been modified in the current build, then by
+definition the current copy of the file is the same as in the build
+were it was last changed and thus it does not matter whether we use
+<current build>/full/<file> or <last change>/full/<file>. But using
+the current copy is better for a CI system which starts without local
+access to older rootfs directories. It might also be a bit more
+efficient (file access less scattered between different "full"
+directories).
+
+Staging directories is better than staging .tar archives containing
+those directories for the same reason (the .tar archive might not be
+available in the CI system) and probably also improves efficiency (no
+need to invoke bsdtar just to create a directory; impact not
+measured).
+
+Also fix a slight flaw in the "target file exists already" handling:
+when that occured for whatever reason (likely only during manual
+debugging), the code would add the original fullfile .tar although it
+isn't needed. Clearing the "ret" variable in that particular error
+case avoids that.
+
+make_pack_full_files() and make_final_pack() used the exact same code
+for populating the "staged" directory. Now that common code is in
+stage_entry().
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ include/swupd.h | 2 +-
+ src/delta.c | 4 +--
+ src/pack.c | 105 +++++++++++++++++++++++++++++++-------------------------
+ 3 files changed, 62 insertions(+), 49 deletions(-)
+
+diff --git a/include/swupd.h b/include/swupd.h
+index c1c0e96..cf384e3 100644
+--- a/include/swupd.h
++++ b/include/swupd.h
+@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest);
+
+ extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
+ extern void link_renames(GList *newfiles, struct manifest *from_manifest);
+-extern void __create_delta(struct file *file, int from_version);
++extern void __create_delta(struct file *file, int from_version, int to_version);
+
+ extern void account_delta_hit(void);
+ extern void account_delta_miss(void);
+diff --git a/src/delta.c b/src/delta.c
+index 7e978b1..8fff4c9 100644
+--- a/src/delta.c
++++ b/src/delta.c
+@@ -35,7 +35,7 @@
+ #include "swupd.h"
+ #include "xattrs.h"
+
+-void __create_delta(struct file *file, int from_version)
++void __create_delta(struct file *file, int from_version, int to_version)
+ {
+ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL;
+ char *tmpdir = NULL;
+@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version)
+ }
+
+ conf = config_image_base();
+- string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename);
++ string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename);
+
+ string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
+
+diff --git a/src/pack.c b/src/pack.c
+index 984c2d6..ccb28bd 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -37,6 +37,7 @@
+ #include <unistd.h>
+
+ #include "swupd.h"
++#include "xattrs.h"
+
+ static void empty_pack_stage(int full, int from_version, int to_version, char *module)
+ {
+@@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack)
+ link_renames(pack->end_manifest->files, manifest);
+ }
+
++static int stage_entry(struct file *file,
++ const char *fullfrom, const char *fullto,
++ const char *tarfrom, const char *tarto,
++ const char *packname)
++{
++ int ret;
++
++ /* Prefer to hardlink uncompressed files or replicate
++ * directories first, and fall back to the compressed
++ * versions if that failed.
++ */
++ if (!file->is_dir) {
++ ret = link(fullfrom, fullto);
++ if (ret && errno == EEXIST) {
++ ret = 0;
++ } else if (ret) {
++ LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno);
++ }
++ } else {
++ /* Replicate directory. */
++ struct stat st;
++ if ((stat(fullfrom, &st) ||
++ mkdir(fullto, 0) ||
++ chmod(fullto, st.st_mode) ||
++ chown(fullto, st.st_uid, st.st_gid) ||
++ (xattrs_copy(fullfrom, fullto), false)) &&
++ errno != EEXIST) {
++ LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno);
++ rmdir(fullto);
++ ret = -1;
++ } else {
++ ret = 0;
++ }
++ }
++
++ if (ret) {
++ ret = link(tarfrom, tarto);
++ if (ret && errno != EEXIST) {
++ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno);
++ }
++ }
++
++ return ret;
++}
++
+ static void make_pack_full_files(struct packdata *pack)
+ {
+ GList *item;
+@@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack)
+ char *fullfrom, *fullto;
+
+ /* hardlink each file that is in <end> but not in <X> */
+- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename);
+ string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+ string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash);
+ string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+
+- ret = -1;
+- errno = 0;
+-
+- /* Prefer to hardlink uncompressed files (excluding
+- * directories) first, and fall back to the compressed
+- * versions if the hardlink fails.
++ /* Prefer to hardlink uncompressed files or replicate
++ * directories first, and fall back to the compressed
++ * versions if that failed.
+ */
+- if (!file->is_dir) {
+- ret = link(fullfrom, fullto);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno);
+- }
+- }
+- if (ret) {
+- ret = link(from, to);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno);
+- }
+- }
++ ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile");
+
+ if (ret == 0) {
+ pack->fullcount++;
+@@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
+ return files;
+ }
+
+-static void create_delta(gpointer data, __unused__ gpointer user_data)
++static void create_delta(gpointer data, gpointer user_data)
+ {
+ struct file *file = data;
++ int *to_version = user_data;
+
+ /* if the file was not found in the from version, skip delta creation */
+ if (file->peer) {
+- __create_delta(file, file->peer->last_change);
++ __create_delta(file, file->peer->last_change, *to_version);
+ }
+ }
+
+-static void make_pack_deltas(GList *files)
++static void make_pack_deltas(GList *files, int to_version)
+ {
+ GThreadPool *threadpool;
+ GList *item;
+@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files)
+ sysconf(_SC_NPROCESSORS_ONLN);
+
+ LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
+- threadpool = g_thread_pool_new(create_delta, NULL,
++ threadpool = g_thread_pool_new(create_delta, &to_version,
+ numthreads, FALSE, NULL);
+
+ item = g_list_first(files);
+@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack)
+ file->last_change, file->hash);
+ string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename);
+ string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+
+@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack)
+ }
+ }
+ } else {
+- ret = -1;
+- errno = 0;
+-
+- /* Prefer to hardlink uncompressed files (excluding
+- * directories) first, and fall back to the compressed
+- * versions if the hardlink fails.
+- */
+- if (!file->is_dir) {
+- ret = link(fullfrom, fullto);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno);
+- }
+- }
+-
+- if (ret) {
+- ret = link(tarfrom, tarto);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno);
+- }
+- }
+-
++ ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final");
+ if (ret == 0) {
+ pack->fullcount++;
+ }
+@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack)
+
+ /* step 2: consolidate delta list & create all delta files*/
+ delta_list = consolidate_packs_delta_files(delta_list, pack);
+- make_pack_deltas(delta_list);
++ make_pack_deltas(delta_list, pack->to);
+ g_list_free(delta_list);
+
+ /* step 3: complete pack creation */
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server/0001-delta.c-fix-xattr-test-after-patching.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch
index 78fb85a..78fb85a 100644
--- a/recipes-core/swupd-server/swupd-server/0001-delta.c-fix-xattr-test-after-patching.patch
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch
new file mode 100644
index 0000000..5920afc
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch
@@ -0,0 +1,349 @@
+From e1f0d54a940eb7d04e2fbd59bd995a819331425f Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Fri, 30 Sep 2016 08:42:08 +0200
+Subject: [PATCH 1/2] swupd-create-update: alternative input layout
+
+In Ostro OS, we already have a "full" directory with all files.
+Splitting it up into bundles just so that swupd-create-update can
+reconstruct the "full" directory is a waste of IO, and noticably slow
+when run under pseudo.
+
+To streamline the required work, a new layout for the "image" input
+directory gets introduced:
+- The "full" directory gets created by the caller before invoking
+ swupd-create-update.
+- For each bundle, instead of a <bundle> directory, there is a
+ <bundle>.content.txt file, listing all entries (including directories)
+ of the bundle.
+
+The traditional mode of operation still works as before because each operation
+which normally works with a bundle directory checks whether there is such a
+directory and if not, switches to the new mode.
+
+That way it is even possible to mix the two modes, i.e. replacing only
+some bundles with a content list, although that's probably not all
+that useful.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/55]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/analyze_fs.c | 171 ++++++++++++++++++++++++++++++++++++++--------------
+ src/chroot.c | 24 +++++---
+ src/create_update.c | 6 +-
+ src/fullfiles.c | 2 +
+ 4 files changed, 150 insertions(+), 53 deletions(-)
+
+diff --git a/src/analyze_fs.c b/src/analyze_fs.c
+index 0f16343..d534587 100644
+--- a/src/analyze_fs.c
++++ b/src/analyze_fs.c
+@@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data)
+ /* disallow characters which can do unexpected things when the filename is
+ * used on a tar command line via system("tar [args] filename [more args]");
+ */
+-static bool illegal_characters(char *filename)
++static bool illegal_characters(const char *filename)
+ {
+ char c;
+ int i;
+@@ -301,25 +301,145 @@ static bool illegal_characters(char *filename)
+ return false;
+ }
+
++static void add_file(struct manifest *manifest,
++ const char *entry_name,
++ char *sub_filename,
++ char *fullname,
++ bool do_hash)
++{
++ GError *err = NULL;
++ struct file *file;
++
++ if (illegal_characters(entry_name)) {
++ printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
++ free(sub_filename);
++ free(fullname);
++ return;
++ }
++
++ file = calloc(1, sizeof(struct file));
++ assert(file);
++
++ file->last_change = manifest->version;
++ file->filename = sub_filename;
++
++ populate_file_struct(file, fullname);
++ if (file->is_deleted) {
++ /*
++ * populate_file_struct() logs a stat() failure, but
++ * does not abort. When adding files that should
++ * exist, this case is an error.
++ */
++ LOG(NULL, "file not found", "%s", fullname);
++ assert(0);
++ }
++
++
++ /* if for some reason there is a file in the official build
++ * which should not be included in the Manifest, then open a bug
++ * to get it removed, and work around its presence by
++ * excluding it here, eg:
++ if (strncmp(file->filename, "/dev/", 5) == 0) {
++ continue;
++ }
++ */
++
++ if (do_hash) {
++ /* compute the hash from a thread */
++ int ret;
++ ret = g_thread_pool_push(threadpool, file, &err);
++ if (ret == FALSE) {
++ printf("GThread hash computation push error\n");
++ printf("%s\n", err->message);
++ assert(0);
++ }
++ }
++ manifest->files = g_list_prepend(manifest->files, file);
++ manifest->count++;
++}
++
++
+ static void iterate_directory(struct manifest *manifest, char *pathprefix,
+ char *subpath, bool do_hash)
+ {
+ DIR *dir;
+ struct dirent *entry;
+ char *fullpath;
+- int ret;
+- GError *err = NULL;
+
+ string_or_die(&fullpath, "%s/%s", pathprefix, subpath);
+
+ dir = opendir(fullpath);
+ if (!dir) {
++ FILE *content;
++ int len;
++ free(fullpath);
++ if (errno != ENOENT) {
++ return;
++ }
++ /*
++ * If there is a <dir>.content.txt instead of
++ * the actual directory, then read that
++ * file. It has a list of path names,
++ * including all directories. The
++ * corresponding file system entry is then
++ * expected to be in a pre-populated "full"
++ * directory.
++ */
++ if (subpath[0]) {
++ string_or_die(&fullpath, "%s/%s.content.txt", pathprefix, len, subpath);
++ } else {
++ string_or_die(&fullpath, "%s.content.txt", pathprefix);
++ }
++ content = fopen(fullpath, "r");
+ free(fullpath);
++ fullpath = NULL;
++ if (content) {
++ char *line = NULL;
++ size_t len = 0;
++ ssize_t read;
++ const char *full;
++ int full_len;
++ /*
++ * determine path to "full" directory: it is assumed to be alongside
++ * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit,
++ * so we have to strip the last path component.
++ */
++ full = strrchr(pathprefix, '/');
++ if (full) {
++ full_len = full - pathprefix + 1;
++ full = pathprefix;
++ } else {
++ full = "";
++ full_len = 0;
++ }
++ while ((read = getline(&line, &len, content)) != -1) {
++ if (read) {
++ const char *entry_name = strrchr(line, '/');
++ if (entry_name) {
++ entry_name++;
++ } else {
++ entry_name = line;
++ }
++ if (line[read - 1] == '\n') {
++ line[read - 1] = 0;
++ }
++ string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line);
++ add_file(manifest,
++ entry_name,
++ strdup(line),
++ fullpath,
++ do_hash);
++ }
++ }
++ free(line);
++ }
++
++ // If both directory and content file are missing, silently (?)
++ // don't add anything to the manifest.
+ return;
+ }
+
+ while (dir) {
+- struct file *file;
+ char *sub_filename;
+ char *fullname;
+
+@@ -334,50 +454,13 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
+ }
+
+ string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
+-
+- if (illegal_characters(entry->d_name)) {
+- printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+- free(sub_filename);
+- continue;
+- }
+-
+- file = calloc(1, sizeof(struct file));
+- if (!file) {
+- break;
+- }
+-
+- file->last_change = manifest->version;
+- file->filename = sub_filename;
+-
+ string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
+- populate_file_struct(file, fullname);
+- free(fullname);
+-
+ if (entry->d_type == DT_DIR) {
+- iterate_directory(manifest, pathprefix, file->filename, do_hash);
++ iterate_directory(manifest, pathprefix, sub_filename, do_hash);
+ }
++ /* takes ownership of the strings */
++ add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
+
+- /* if for some reason there is a file in the official build
+- * which should not be included in the Manifest, then open a bug
+- * to get it removed, and work around its presence by
+- * excluding it here, eg:
+- if (strncmp(file->filename, "/dev/", 5) == 0) {
+- continue;
+- }
+- */
+-
+- if (do_hash) {
+- /* compute the hash from a thread */
+- ret = g_thread_pool_push(threadpool, file, &err);
+- if (ret == FALSE) {
+- printf("GThread hash computation push error\n");
+- printf("%s\n", err->message);
+- closedir(dir);
+- return;
+- }
+- }
+- manifest->files = g_list_prepend(manifest->files, file);
+- manifest->count++;
+ }
+ closedir(dir);
+ free(fullpath);
+diff --git a/src/chroot.c b/src/chroot.c
+index 32ed997..f3832e1 100644
+--- a/src/chroot.c
++++ b/src/chroot.c
+@@ -39,15 +39,21 @@ void chroot_create_full(int newversion)
+ char *full_dir;
+
+ string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion);
++ if (!access(full_dir, R_OK|X_OK)) {
++ free(full_dir);
++ return;
++ }
+
+ g_mkdir_with_parents(full_dir, S_IRWXU);
+
+ /* start with base */
+- LOG(NULL, "Copying chroot os-core to full", "");
+ string_or_die(&param, "%s/%i/os-core/", image_dir, newversion);
+- char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
+- if (system_argv(rsynccmd) != 0) {
+- assert(0);
++ if (!access(param, F_OK)) {
++ LOG(NULL, "Copying chroot os-core to full", "");
++ char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
++ if (system_argv(rsynccmd) != 0) {
++ assert(0);
++ }
+ }
+ free(param);
+
+@@ -58,11 +64,13 @@ void chroot_create_full(int newversion)
+ break;
+ }
+
+- LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
+ string_or_die(&param, "%s/%i/%s/", image_dir, newversion, group);
+- char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
+- if (system_argv(rsynccmd) != 0) {
+- assert(0);
++ if (!access(param, F_OK)) {
++ LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
++ char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
++ if (system_argv(rsynccmd) != 0) {
++ assert(0);
++ }
+ }
+ free(param);
+ }
+diff --git a/src/create_update.c b/src/create_update.c
+index 766609b..74d5376 100644
+--- a/src/create_update.c
++++ b/src/create_update.c
+@@ -141,6 +141,7 @@ static bool parse_options(int argc, char **argv)
+ static void populate_dirs(int version)
+ {
+ char *newversiondir;
++ char *newversiondircontent = NULL;
+
+ string_or_die(&newversiondir, "%s/%d", image_dir, version);
+
+@@ -182,9 +183,11 @@ static void populate_dirs(int version)
+ }
+
+ string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group);
++ string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group);
+
+ /* Create the bundle directory(s) as needed */
+- if (access(newversiondir, F_OK | R_OK) != 0) {
++ if (access(newversiondir, F_OK | R_OK) != 0 &&
++ access(newversiondircontent, F_OK | R_OK) != 0) {
+ printf("%s does not exist...creating\n", group);
+ if (mkdir(newversiondir, 0755) != 0) {
+ printf("Failed to create %s subdirectory\n", group);
+@@ -193,6 +196,7 @@ static void populate_dirs(int version)
+ }
+ }
+ free(newversiondir);
++ free(newversiondircontent);
+ }
+
+ static int check_build_env(void)
+diff --git a/src/fullfiles.c b/src/fullfiles.c
+index 9fdfc2f..214b7b4 100644
+--- a/src/fullfiles.c
++++ b/src/fullfiles.c
+@@ -136,6 +136,8 @@ static void create_fullfile(struct file *file)
+ }
+ done += curr;
+ }
++ close(fd);
++ fd = -1;
+ }
+
+ for (int i = 0; compression_filters[i]; i++) {
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch
new file mode 100644
index 0000000..56ea17d
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch
@@ -0,0 +1,200 @@
+From 2e3eb8abcdef496d0ce30d03da7befcb9978aceb Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Sat, 1 Oct 2016 13:51:02 +0200
+Subject: [PATCH 2/2] add logging to stdout
+
+When a CI system (like the one from Ostro) captures the output of
+commands, but not necessarily intermediate log files, then it is
+useful to also log to stdout. Another use case is calling the tools
+interactively during development.
+
+The new --log-stdout option in all three commands enables logging to
+stdout in addition to the traditional log files.
+
+The implementation recycles the existing init_log_stdout() (not used
+before) and gives it the slightly different meaning of "also log to
+stdout".
+
+Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/72dd27a886ad9b2c66bb7cdb5c4cadb24f783654]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/create_update.c | 5 +++++
+ src/log.c | 27 +++++++++++++++++----------
+ src/make_fullfiles.c | 5 +++++
+ src/make_packs.c | 5 +++++
+ 4 files changed, 32 insertions(+), 10 deletions(-)
+
+diff --git a/src/create_update.c b/src/create_update.c
+index 74d5376..f1c840d 100644
+--- a/src/create_update.c
++++ b/src/create_update.c
+@@ -50,6 +50,7 @@ static void banner(void)
+ static const struct option prog_opts[] = {
+ { "help", no_argument, 0, 'h' },
+ { "version", no_argument, 0, 'v' },
++ { "log-stdout", no_argument, 0, 'l' },
+ { "osversion", required_argument, 0, 'o' },
+ { "minversion", required_argument, 0, 'm' },
+ { "format", required_argument, 0, 'F' },
+@@ -68,6 +69,7 @@ static void print_help(const char *name)
+ printf(" -v, --version Show software version\n");
+ printf("\n");
+ printf("Application Options:\n");
++ printf(" -l, --log-stdout Write log messages also to stdout\n");
+ printf(" -o, --osversion The OS version for which to create an update\n");
+ printf(" -m, --minversion Optional minimum file version to write into manifests per file\n");
+ printf(" -F, --format Format number for the update\n");
+@@ -87,6 +89,9 @@ static bool parse_options(int argc, char **argv)
+ case 'h':
+ print_help(argv[0]);
+ return false;
++ case 'l':
++ init_log_stdout();
++ break;
+ case 'v':
+ banner();
+ return false;
+diff --git a/src/log.c b/src/log.c
+index e8bf9c3..45a4d66 100644
+--- a/src/log.c
++++ b/src/log.c
+@@ -33,7 +33,7 @@
+
+ #include "swupd.h"
+
+-static FILE *logfile;
++static FILE *logfile[2];
+
+ static struct timeval start_time;
+
+@@ -41,13 +41,13 @@ void init_log(const char *prefix, const char *bundle, int start, int end)
+ {
+ char *filename;
+ string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end);
+- logfile = fopen(filename, "w");
++ logfile[0] = fopen(filename, "w");
+ free(filename);
+ gettimeofday(&start_time, NULL);
+ }
+ void init_log_stdout(void)
+ {
+- logfile = stdout;
++ logfile[1] = stdout;
+ gettimeofday(&start_time, NULL);
+ }
+
+@@ -91,8 +91,9 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
+ char *logstring = NULL;
+ char filebuf[4096];
+ char filebuf2[4096];
++ int i;
+
+- if (!logfile) {
++ if (!logfile[0] && !logfile[1]) {
+ return;
+ }
+
+@@ -119,12 +120,16 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con
+ strcat(filebuf2, " ");
+ }
+
+- fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
+- (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
++ for (i = 0; i < 2; i++) {
++ if (logfile[i]) {
++ fprintf(logfile[i], "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n",
++ (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf);
++ fflush(logfile[i]);
++ }
++ }
+
+ free(logstring);
+ free(buf);
+- fflush(logfile);
+ }
+
+ void close_log(int version, int exit_status)
+@@ -133,7 +138,7 @@ void close_log(int version, int exit_status)
+ int t_sec;
+ int t_msec;
+
+- if (!logfile) {
++ if (!logfile[0] && !logfile[1]) {
+ return;
+ }
+
+@@ -159,6 +164,8 @@ void close_log(int version, int exit_status)
+ printf("Update build failed for version %i\n", version);
+ }
+
+- fclose(logfile);
+- logfile = NULL;
++ if (logfile[0]) {
++ fclose(logfile[0]);
++ logfile[0] = NULL;
++ }
+ }
+diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c
+index 2a1e2e9..0216e91 100644
+--- a/src/make_fullfiles.c
++++ b/src/make_fullfiles.c
+@@ -33,6 +33,7 @@
+
+ static const struct option prog_opts[] = {
+ { "help", no_argument, 0, 'h' },
++ { "log-stdout", no_argument, 0, 'l' },
+ { "statedir", required_argument, 0, 'S' },
+ { 0, 0, 0, 0 }
+ };
+@@ -43,6 +44,7 @@ static void usage(const char *name)
+ printf(" %s <version>\n\n", name);
+ printf("Help options:\n");
+ printf(" -h, --help Show help options\n");
++ printf(" -l, --log-stdout Write log messages also to stdout\n");
+ printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
+ printf("\n");
+ }
+@@ -57,6 +59,9 @@ static bool parse_options(int argc, char **argv)
+ case 'h':
+ usage(argv[0]);
+ return false;
++ case 'l':
++ init_log_stdout();
++ break;
+ case 'S':
+ if (!optarg || !set_state_dir(optarg)) {
+ printf("Invalid --statedir argument '%s'\n\n", optarg);
+diff --git a/src/make_packs.c b/src/make_packs.c
+index 8560b3f..2d3e25e 100644
+--- a/src/make_packs.c
++++ b/src/make_packs.c
+@@ -46,6 +46,7 @@ static void banner(void)
+
+ static const struct option prog_opts[] = {
+ { "help", no_argument, 0, 'h' },
++ { "log-stdout", no_argument, 0, 'l' },
+ { "statedir", required_argument, 0, 'S' },
+ { "signcontent", no_argument, 0, 's' },
+ { 0, 0, 0, 0 }
+@@ -57,6 +58,7 @@ static void usage(const char *name)
+ printf(" %s <start version> <latest version> <bundle>\n\n", name);
+ printf("Help options:\n");
+ printf(" -h, --help Show help options\n");
++ printf(" -l, --log-stdout Write log messages also to stdout\n");
+ printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
+ printf(" -s, --signcontent Enables cryptographic signing of update content\n");
+ printf("\n");
+@@ -72,6 +74,9 @@ static bool parse_options(int argc, char **argv)
+ case 'h':
+ usage(argv[0]);
+ return false;
++ case 'l':
++ init_log_stdout();
++ break;
+ case 'S':
+ if (!optarg || !set_state_dir(optarg)) {
+ printf("Invalid --statedir argument ''%s'\n\n", optarg);
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch
new file mode 100644
index 0000000..440cca8
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch
@@ -0,0 +1,42 @@
+From c9e9fb971ab0494047b4d8c0e656e0a06ad9b236 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 8 Nov 2016 18:39:49 +0100
+Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs
+
+The fullfile .tar is needed for a pack as fallback when linking from
+the full rootfs isn't possible. In practice this shouldn't happen.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/pack.c | 13 +++++++++++++
+ 1 file changed, 13 insertions(+)
+
+diff --git a/src/pack.c b/src/pack.c
+index ccb28bd..e331da2 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -187,6 +187,19 @@ static int stage_entry(struct file *file,
+
+ if (ret) {
+ ret = link(tarfrom, tarto);
++ if (ret && errno == ENOENT && content_url) {
++ LOG(NULL, "Download fallback for pack", "%s: %s to %s", packname, tarfrom, tarto);
++
++ /* Must be "tarfrom" that is missing. Download directly into target location.*/
++ char *cmd;
++ string_or_die(&cmd, "curl -s -o '%s' %s/%d/files/%s.tar",
++ tarto, content_url, file->last_change, file->hash);
++ if (system(cmd)) {
++ LOG(file, "Downloading failed", "%s", cmd);
++ } else {
++ ret = 0;
++ }
++ }
+ if (ret && errno != EEXIST) {
+ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno);
+ }
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch
new file mode 100644
index 0000000..4cc3af4
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch
@@ -0,0 +1,38 @@
+From 7b9e1bc78b704a2fb9610dddc6f0925bb6412f55 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 8 Nov 2016 18:34:44 +0100
+Subject: [PATCH 2/3] pack.c: do not clean packstage
+
+This works around a bug in pseudo
+(https://bugzilla.yoctoproject.org/show_bug.cgi?id=10623) where
+unlinking the linked entries in the staging area also removes the
+xattrs of the original files in the full directory tree.
+
+This works as long as the swupd_create_pack command does not get
+called again without cleaning the packstage.
+
+Upstream-Status: Inappropriate [workaround]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/pack.c | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+
+diff --git a/src/pack.c b/src/pack.c
+index 46c7c68..984c2d6 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -46,7 +46,9 @@ static void empty_pack_stage(int full, int from_version, int to_version, char *m
+ // clean any stale data (eg: re-run after a failure)
+ string_or_die(&param, "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version);
+ char *const rmcmd[] = { "rm", "-fr", param, NULL };
+- if (system_argv(rmcmd) != 0) {
++ if (true) {
++ LOG(NULL, "Skipping removal of packstage", "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version);
++ } else if (system_argv(rmcmd) != 0) {
+ fprintf(stderr, "Failed to clean %s/%s/%i_to_%i\n",
+ packstage_dir, module, from_version, to_version);
+ free(param);
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch
new file mode 100644
index 0000000..7e981e3
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch
@@ -0,0 +1,60 @@
+From 88fd362785ae8871537573fd7073e499542b0a0d Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Thu, 17 Nov 2016 10:09:27 +0100
+Subject: [PATCH 3/3] create_pack: abort delta handling early when impossible
+ due to IMA
+
+Currently, deltas cannot be computed for systems using IMA because IMA
+relies on a security.ima xattr which changes each time the file
+content changes and swupd doesn't support deltas for files with
+different xattrs.
+
+It would be worthwhile to add such a support, but that's a bit
+complicated (needs to be done in client and server and implies a
+format change), so for now just abort early when a security.xattr is
+found.
+
+This speeds up delta computation considerably in Ostro OS because it
+skips the slow downloading of the old file.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/delta.c | 13 +++++++++++++
+ 1 file changed, 13 insertions(+)
+
+diff --git a/src/delta.c b/src/delta.c
+index 8fff4c9..d3c4b35 100644
+--- a/src/delta.c
++++ b/src/delta.c
+@@ -30,6 +30,7 @@
+ #include <string.h>
+ #include <sys/stat.h>
+ #include <sys/types.h>
++#include <attr/xattr.h>
+ #include <unistd.h>
+
+ #include "swupd.h"
+@@ -64,6 +65,18 @@ void __create_delta(struct file *file, int from_version, int to_version)
+
+ string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
+
++ if (lgetxattr(newfile, "security.ima", NULL, 0) > 0) {
++ /* There is a non-empty security.ima xattr on the new file.
++ * That xattr contains a hash of the file content. We know that
++ * the file content has changed, so the xattr will be different
++ * from the one on the old file and we can bail out early without
++ * even bothering with retrieving the original file. A better
++ * solution for systems with IMA would be to support deltas even
++ * when xattrs are different.
++ */
++ goto out;
++ }
++
+ if (access(original, F_OK) &&
+ content_url) {
+ /* File does not exist. Try to get it from the online update repo instead.
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch
new file mode 100644
index 0000000..3a71d9c
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch
@@ -0,0 +1,246 @@
+From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 8 Nov 2016 18:39:49 +0100
+Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for
+ diffing
+
+The new, optional mode gets enabled via the --content-url parameter.
+When specified, swupd_create_pack will download the <original
+version>/files/<hash>.tar file from the update server if the required
+file is missing.
+
+This mode is meant for use with meta-swupd where new CI builds start
+with an empty "image" directory. NFS access to the full, expanded
+"image" directories of the previous build is not possible in such a
+setup (would require root for special file attributes).
+
+Downloading and unpacking image directories of previous builds would
+be possible, but is expected to be slower (when downloading all files,
+not just those needed for diffing) or more complex (when deciding about
+required files outside of swupd_create_pack).
+
+The downside of this new approach is that it relies on not changing
+the content or naming of the tar files. For example, switching from
+GNU tar to bsdtar would not work. But that's not an issue for the
+intended usage in Ostro OS, which already made the switch to
+bsdtar.
+
+If for some reason a format change is necessary, delta computation
+becomes impossible and falls back to staging the full files.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ include/swupd.h | 2 ++
+ src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--
+ src/globals.c | 13 ++++++++
+ src/make_packs.c | 8 +++++
+ 4 files changed, 111 insertions(+), 2 deletions(-)
+
+diff --git a/include/swupd.h b/include/swupd.h
+index 31c7191..c1c0e96 100644
+--- a/include/swupd.h
++++ b/include/swupd.h
+@@ -147,12 +147,14 @@ extern char *state_dir;
+ extern char *packstage_dir;
+ extern char *image_dir;
+ extern char *staging_dir;
++extern char *content_url;
+
+ extern bool init_globals(void);
+ extern void free_globals(void);
+ extern bool set_format(char *);
+ extern void check_root(void);
+ extern bool set_state_dir(char *);
++extern bool set_content_url(const char *);
+ extern bool init_state_globals(void);
+ extern void free_state_globals(void);
+
+diff --git a/src/delta.c b/src/delta.c
+index 67e7df7..7e978b1 100644
+--- a/src/delta.c
++++ b/src/delta.c
+@@ -37,8 +37,14 @@
+
+ void __create_delta(struct file *file, int from_version)
+ {
+- char *original, *newfile, *outfile, *dotfile, *testnewfile;
+- char *conf, *param1, *param2;
++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL;
++ char *tmpdir = NULL;
++ char *url = NULL;
++ char *cmd = NULL;
++ char *conf = NULL, *param1 = NULL, *param2 = NULL;
++ bool delete_original = false;
++ struct manifest *manifest = NULL;
++ GError *gerror = NULL;
+ int ret;
+
+ if (file->is_link) {
+@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version)
+
+ string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
+
++ if (access(original, F_OK) &&
++ content_url) {
++ /* File does not exist. Try to get it from the online update repo instead.
++ * This fallback is meant to be used for CI builds which start with no local
++ * state and only HTTP(S) access to the published www directory.
++ * Not being able to retrieve the file is not an error and will merely
++ * prevent computing the delta.
++ */
++ string_or_die(&tmpdir, "%s/make-pack-tmpdir-XXXXXX", state_dir);
++ tmpdir = g_dir_make_tmp("make-pack-XXXXXX", &gerror);
++ if (!tmpdir) {
++ LOG(NULL, "Failed to create temporary directory for untarring original file", "%s",
++ gerror->message);
++ assert(0);
++ }
++ /* Determine hash of original file in the corresponding Manifest. */
++ manifest = manifest_from_file(from_version, "full");
++ if (!manifest) {
++ LOG(NULL, "Failed to read full Manifest", "version %d, cannot retrieve original file",
++ from_version);
++ goto out;
++ }
++ const char *last_hash = NULL;
++ GList *list = g_list_first(manifest->files);
++ while (list) {
++ struct file *original_file = list->data;
++ if (!strcmp(file->filename, original_file->filename)) {
++ last_hash = original_file->hash;
++ break;
++ }
++ list = g_list_next(list);
++ }
++ if (!last_hash) {
++ LOG(NULL, "Original file not found", "%s in full manifest for %d - inconsistent update data?!",
++ file->filename, from_version);
++ goto out;
++ }
++
++ /* We use a temporary copy because we don't want to
++ * tamper with the original "full" folder which
++ * probably does not even exist. Using a temporary file
++ * file implies re-downloading in the future, but that's
++ * consistent with the intended usage in a CI environment
++ * which always starts from scratch.
++ */
++ free(original);
++ string_or_die(&original, "%s/%s", tmpdir, last_hash);
++ delete_original = true;
++
++ /*
++ * This is a proof-of-concept. A real implementation should use
++ * a combination of libcurl + libarchive calls to unpack the files.
++ * For current Ostro OS, deltas despite xattr differences would
++ * be needed, otherwise this code here is of little use (all
++ * modified files fail the xattr sameness check, because security.ima
++ * changes when file content changes).
++ */
++ string_or_die(&url, "%s/%d/files/%s.tar", content_url, from_version, last_hash);
++ LOG(file, "Downloading original file", "%s to %s", url, original);
++
++ /* bsdtar can detect compression when reading from stdin, GNU tar can't. */
++ string_or_die(&cmd, "curl -s %s | bsdtar -C %s -xf -", url, tmpdir);
++ if (system(cmd)) {
++ LOG(file, "Downloading/unpacking failed, skipping delta", "%s", url);
++ goto out;
++ }
++ }
++
+ free(conf);
+
+ conf = config_output_dir();
+@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version)
+ LOG(NULL, "Failed to rename", "");
+ }
+ out:
++ if (delete_original) {
++ unlink(original);
++ }
++ if (tmpdir) {
++ rmdir(tmpdir);
++ g_free(tmpdir);
++ }
++ if (manifest) {
++ free_manifest(manifest);
++ }
++ g_clear_error(&gerror);
++ free(cmd);
+ free(testnewfile);
+ free(conf);
+ free(newfile);
+diff --git a/src/globals.c b/src/globals.c
+index 74758ce..0e047e2 100644
+--- a/src/globals.c
++++ b/src/globals.c
+@@ -40,6 +40,7 @@ char *state_dir = NULL;
+ char *packstage_dir = NULL;
+ char *image_dir = NULL;
+ char *staging_dir = NULL;
++char *content_url = NULL;
+
+ bool set_format(char *userinput)
+ {
+@@ -76,6 +77,17 @@ bool set_state_dir(char *dir)
+ return true;
+ }
+
++bool set_content_url(const char *url)
++{
++ if (content_url) {
++ free(content_url);
++ }
++ string_or_die(&content_url, "%s", url);
++
++ return true;
++}
++
++
+ bool init_globals(void)
+ {
+ if (format == 0) {
+@@ -123,4 +135,5 @@ void free_state_globals(void)
+ free(packstage_dir);
+ free(image_dir);
+ free(staging_dir);
++ free(content_url);
+ }
+diff --git a/src/make_packs.c b/src/make_packs.c
+index 2d3e25e..827be56 100644
+--- a/src/make_packs.c
++++ b/src/make_packs.c
+@@ -49,6 +49,7 @@ static const struct option prog_opts[] = {
+ { "log-stdout", no_argument, 0, 'l' },
+ { "statedir", required_argument, 0, 'S' },
+ { "signcontent", no_argument, 0, 's' },
++ { "content-url", required_argument, 0, 'u' },
+ { 0, 0, 0, 0 }
+ };
+
+@@ -61,6 +62,7 @@ static void usage(const char *name)
+ printf(" -l, --log-stdout Write log messages also to stdout\n");
+ printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
+ printf(" -s, --signcontent Enables cryptographic signing of update content\n");
++ printf(" -u, --content-url Base URL of the update repo (optional, used to retrieve missing files on demand");
+ printf("\n");
+ }
+
+@@ -86,6 +88,12 @@ static bool parse_options(int argc, char **argv)
+ case 's':
+ enable_signing = true;
+ break;
++ case 'u':
++ if (!optarg || !set_content_url(optarg)) {
++ printf("Invalid --content-url argument ''%s''\n\n", optarg);
++ return false;
++ }
++ break;
+ }
+ }
+
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch
new file mode 100644
index 0000000..43ae6ff
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch
@@ -0,0 +1,56 @@
+From d253e81e001ccbff9d7a8ee1768006c5ac36b259 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 27 Sep 2016 16:43:57 +0200
+Subject: [PATCH 25/29] swupd_make_pack: fix extracting files with bsdtar
+
+TAR_XATTR_ARGS is no longer used as part of a plain string. Embedding
+the empty "" value for bsdtar inside an argv argument list passes an
+empty parameter to bsdtar, leading to:
+ bsdtar: Must specify one of -c, -r, -t, -u, -x
+
+To allow the the "no parameter" case, it has to be argument list: that
+can be empty. If not empty, it has to end with a comma.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/41]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ include/swupd.h | 4 ++--
+ src/pack.c | 2 +-
+ 2 files changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/include/swupd.h b/include/swupd.h
+index 43a5002..0aa6399 100644
+--- a/include/swupd.h
++++ b/include/swupd.h
+@@ -20,12 +20,12 @@
+ #define TAR_COMMAND "bsdtar"
+ #define TAR_XATTR_ARGS ""
+ #define TAR_XATTR_ARGS_STRLIST
+-#define TAR_WARN_ARGS ""
++#define TAR_WARN_ARGS_STRLIST
+ #else
+ #define TAR_COMMAND "tar"
+ #define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'"
+ #define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'",
+-#define TAR_WARN_ARGS "--warning=no-timestamp"
++#define TAR_WARN_ARGS_STRLIST "--warning=no-timestamp",
+ #endif
+
+ #if SWUPD_WITH_SELINUX
+diff --git a/src/pack.c b/src/pack.c
+index a1fbc51..12d7443 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -116,7 +116,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module)
+ * time on the client...
+ */
+ string_or_die(&param, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version);
+- char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
++ char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS_STRLIST TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL };
+ if (system_argv(tarcmd) == 0) {
+ unlink(path);
+ }
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch
new file mode 100644
index 0000000..abc3082
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch
@@ -0,0 +1,30 @@
+From a8b77a7d4479df7662faf3a15f5aca6828d1c67a Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 27 Sep 2016 08:12:49 +0200
+Subject: [PATCH 26/29] fullfiles.c: fix invalid LOG() call
+
+LOG() takes an additional fixed string before the format string.
+
+Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/7e38f013eff01b84a8df88557f5312dc070ef0b3]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/fullfiles.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/fullfiles.c b/src/fullfiles.c
+index 31b9ab5..3be43d1 100644
+--- a/src/fullfiles.c
++++ b/src/fullfiles.c
+@@ -72,7 +72,7 @@ static void create_fullfile(struct file *file)
+ string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
+ if (lstat(origin, &sbuf) < 0) {
+ /* no input file: means earlier phase of update creation failed */
+- LOG(NULL, "Failed to stat %s\n", origin);
++ LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno));
+ assert(0);
+ }
+
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch
new file mode 100644
index 0000000..0a68667
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch
@@ -0,0 +1,89 @@
+From 8686189b3b080446fae732a85b72528b7fe68ba6 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 27 Sep 2016 08:25:40 +0200
+Subject: [PATCH 27/29] update control over parallelism
+
+The SWUPD_NUM_THREADS env variable is now understood by all three
+commands and overrides the default number of threads. Setting it to 1
+is useful while debugging the code that runs inside threads (only one
+thread hits breakpoints there).
+
+The hard-coded parallelism of 12 threads when analysing the file system
+gets replaced with n, where n is the number of available CPUs. The default
+is the same as before elsewhere (n for packing, 3 * n for fullfiles).
+
+Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/4e0fdd4193a8ce9dcf3cfc5e488dfd4b23b7e7d9]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/analyze_fs.c | 5 ++++-
+ src/fullfiles.c | 7 +++++--
+ src/pack.c | 7 +++++--
+ 3 files changed, 14 insertions(+), 5 deletions(-)
+
+diff --git a/src/analyze_fs.c b/src/analyze_fs.c
+index 3bfb288..0f16343 100644
+--- a/src/analyze_fs.c
++++ b/src/analyze_fs.c
+@@ -387,6 +387,9 @@ struct manifest *full_manifest_from_directory(int version)
+ {
+ struct manifest *manifest;
+ char *dir;
++ int numthreads = getenv("SWUPD_NUM_THREADS") ?
++ atoi(getenv("SWUPD_NUM_THREADS")) :
++ sysconf(_SC_NPROCESSORS_ONLN);
+
+ LOG(NULL, "Computing hashes", "for %i/full", version);
+
+@@ -394,7 +397,7 @@ struct manifest *full_manifest_from_directory(int version)
+
+ string_or_die(&dir, "%s/%i/full", image_dir, version);
+
+- threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL);
++ threadpool = g_thread_pool_new(get_hash, dir, numthreads, FALSE, NULL);
+
+ iterate_directory(manifest, dir, "", true);
+
+diff --git a/src/fullfiles.c b/src/fullfiles.c
+index 3be43d1..216a1d7 100644
+--- a/src/fullfiles.c
++++ b/src/fullfiles.c
+@@ -291,10 +291,13 @@ static void submit_fullfile_tasks(GList *files)
+ int ret;
+ int count = 0;
+ GError *err = NULL;
++ int numthreads = getenv("SWUPD_NUM_THREADS") ?
++ atoi(getenv("SWUPD_NUM_THREADS")) :
++ sysconf(_SC_NPROCESSORS_ONLN) * 3;
+
+- LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3);
++ LOG(NULL, "fullfile threadpool", "%d threads", numthreads);
+ threadpool = g_thread_pool_new(create_fullfile_task, NULL,
+- sysconf(_SC_NPROCESSORS_ONLN) * 3,
++ numthreads,
+ TRUE, NULL);
+
+ printf("Starting downloadable fullfiles data creation\n");
+diff --git a/src/pack.c b/src/pack.c
+index 12d7443..3ffb88a 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -285,10 +285,13 @@ static void make_pack_deltas(GList *files)
+ struct file *file;
+ int ret;
+ GError *err = NULL;
++ int numthreads = getenv("SWUPD_NUM_THREADS") ?
++ atoi(getenv("SWUPD_NUM_THREADS")) :
++ sysconf(_SC_NPROCESSORS_ONLN);
+
+- LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN));
++ LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
+ threadpool = g_thread_pool_new(create_delta, NULL,
+- sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL);
++ numthreads, FALSE, NULL);
+
+ item = g_list_first(files);
+ while (item) {
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch
new file mode 100644
index 0000000..0322735
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch
@@ -0,0 +1,100 @@
+From f74807f9aebbb7b8feb1f50107e268bd869f2691 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Wed, 28 Sep 2016 16:55:22 +0200
+Subject: [PATCH 1/3] enable locales in all programs
+
+This is a pre-condition for using libarchive directly: libarchive
+needs to know what the encoding of filenames is, and it uses the
+current locale for that. Without setlocale(), the locale is "C", which
+only supports ASCII filenames, leading to warnings about "Can't
+encode..." from libarchive when it is forced to fall back to copying
+strings verbatim when writing archives that require UTF-8 encoding.
+
+As a side effect, error messages from libc will get translated
+according to the user's environment.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/44]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+
+---
+ src/create_update.c | 6 ++++++
+ src/make_fullfiles.c | 6 ++++++
+ src/make_packs.c | 6 ++++++
+ 3 files changed, 18 insertions(+)
+
+diff --git a/src/create_update.c b/src/create_update.c
+index 97045e5..766609b 100644
+--- a/src/create_update.c
++++ b/src/create_update.c
+@@ -29,6 +29,7 @@
+ #include <errno.h>
+ #include <getopt.h>
+ #include <glib.h>
++#include <locale.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+@@ -240,6 +241,11 @@ int main(int argc, char **argv)
+ /* keep valgrind working well */
+ setenv("G_SLICE", "always-malloc", 0);
+
++ if (!setlocale(LC_ALL, "")) {
++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
++ return EXIT_FAILURE;
++ }
++
+ if (!parse_options(argc, argv)) {
+ free_globals();
+ return EXIT_FAILURE;
+diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c
+index 4ea2f01..2a1e2e9 100644
+--- a/src/make_fullfiles.c
++++ b/src/make_fullfiles.c
+@@ -23,6 +23,7 @@
+ #define _GNU_SOURCE
+ #include <assert.h>
+ #include <getopt.h>
++#include <locale.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+@@ -88,6 +89,11 @@ int main(int argc, char **argv)
+ /* keep valgrind working well */
+ setenv("G_SLICE", "always-malloc", 0);
+
++ if (!setlocale(LC_ALL, "")) {
++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
++ return EXIT_FAILURE;
++ }
++
+ if (!parse_options(argc, argv)) {
+ free_state_globals();
+ return EXIT_FAILURE;
+diff --git a/src/make_packs.c b/src/make_packs.c
+index 4002cd9..8560b3f 100644
+--- a/src/make_packs.c
++++ b/src/make_packs.c
+@@ -27,6 +27,7 @@
+ #include <getopt.h>
+ #include <getopt.h>
+ #include <glib.h>
++#include <locale.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+@@ -101,6 +102,11 @@ int main(int argc, char **argv)
+ int exit_status = EXIT_FAILURE;
+ char *file_path = NULL;
+
++ if (!setlocale(LC_ALL, "")) {
++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]);
++ return EXIT_FAILURE;
++ }
++
+ if (!parse_options(argc, argv)) {
+ free_state_globals();
+ return EXIT_FAILURE;
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch
new file mode 100644
index 0000000..d599261
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch
@@ -0,0 +1,626 @@
+From 4ebc886b910fa4cb83864658069a30dd133caca8 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Wed, 30 Mar 2016 13:14:42 +0200
+Subject: [PATCH 3/6] fullfiles: use libarchive directly
+
+Calling an external tar command makes the code fairly complicated,
+because it is necessary to set up a suitable temporary directory with
+the desired content. By calling libarchive directly, no temporary copy
+is needed and directories and files can be treated (almost) the same
+way. The only difference is that for files, data has to be added to
+the archive.
+
+More important, performance under bitbake took a big hit because of
+the external commands. Launching them when running under pseudo is a
+lot slower compared to running natively as root. For example, the
+unmodified swupd-make-fullfiles took over 20min for
+ostro-image-swupd. With this change, it completes in 3:30min. When
+running natively as root, there is also some improvement because less
+work needs to be done, but total runtime only decreases from 2:28min
+to 2:14min.
+
+By calling libarchive directly, swupd also gets better control over
+error handling. bsdtar emits a warning for invalid filename or
+linkname encoding, which was silently ignored when using the external
+command. Now it is treated as an error.
+
+Archives always get created using the restricted pax interchange
+format (see
+https://github.com/libarchive/libarchive/wiki/ManPageLibarchiveFormats5),
+same as with using bsdtar as external command. GNU tar should be able
+to decode them properly as long as no extensions are needed (for
+example, for xattrs).
+
+Only the smallest archive really gets written to disk. Until then, the
+compressed archive is kept in memory. As an additional optimization,
+creating an archive gets aborted once it is already larger than the
+currently best one. In practice, that particular optimization did not
+have any significant impact on performance.
+
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/48]
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+
+---
+ Makefile.am | 6 +-
+ configure.ac | 1 +
+ include/libarchive_helper.h | 42 ++++++
+ src/fullfiles.c | 324 +++++++++++++++++++++++++-------------------
+ src/in_memory_archive.c | 67 +++++++++
+ 5 files changed, 300 insertions(+), 140 deletions(-)
+ create mode 100644 include/libarchive_helper.h
+ create mode 100644 src/in_memory_archive.c
+
+diff --git a/Makefile.am b/Makefile.am
+index 528f02f..b14feb0 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -23,6 +23,7 @@ swupd_create_update_SOURCES = \
+ src/helpers.c \
+ src/heuristics.c \
+ src/log.c \
++ src/in_memory_archive.c \
+ src/manifest.c \
+ src/pack.c \
+ src/rename.c \
+@@ -54,6 +55,7 @@ swupd_make_fullfiles_SOURCES = \
+ src/fullfiles.c \
+ src/globals.c \
+ src/helpers.c \
++ src/in_memory_archive.c \
+ src/log.c \
+ src/make_fullfiles.c \
+ src/manifest.c \
+@@ -63,12 +65,13 @@ swupd_make_fullfiles_SOURCES = \
+ src/stats.c \
+ src/xattrs.c
+
+-AM_CPPFLAGS = $(glib_CFLAGS) -I$(top_srcdir)/include
++AM_CPPFLAGS = $(glib_CFLAGS) $(libarchive_CFLAGS) -I$(top_srcdir)/include
+
+ swupd_create_update_LDADD = \
+ $(glib_LIBS) \
+ $(zlib_LIBS) \
+ $(openssl_LIBS) \
++ $(libarchive_LIBS) \
+ $(bsdiff_LIBS)
+
+ swupd_make_pack_LDADD = \
+@@ -81,6 +84,7 @@ swupd_make_fullfiles_LDADD = \
+ $(glib_LIBS) \
+ $(zlib_LIBS) \
+ $(openssl_LIBS) \
++ $(libarchive_LIBS) \
+ $(bsdiff_LIBS)
+
+ if ENABLE_LZMA
+diff --git a/configure.ac b/configure.ac
+index 8819206..3e2d933 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -36,6 +36,7 @@ AC_ARG_ENABLE(
+ AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command])),
+ AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command])
+ )
++PKG_CHECK_MODULES([libarchive], [libarchive])
+
+ AC_ARG_ENABLE(
+ [tests],
+diff --git a/include/libarchive_helper.h b/include/libarchive_helper.h
+new file mode 100644
+index 0000000..ad28def
+--- /dev/null
++++ b/include/libarchive_helper.h
+@@ -0,0 +1,42 @@
++/*
++ * Software Updater - server side
++ *
++ * Copyright © 2016 Intel Corporation.
++ *
++ * This program is free software: you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation, version 2 or later of the License.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
++ * GNU General Public License for more details.
++ *
++ * You should have received a copy of the GNU General Public License
++ * along with this program. If not, see <http://www.gnu.org/licenses/>.
++ *
++ * Authors:
++ * Patrick Ohly <patrick.ohly@intel.com>
++ *
++ */
++
++#ifndef __INCLUDE_GUARD_LIBARCHIVE_HELPER_H
++#define __INCLUDE_GUARD_LIBARCHIVE_HELPER_H
++
++#include <archive.h>
++#include <stdint.h>
++
++/*
++ * Used by archive_write_open() callbacks to store the resulting archive in memory.
++ */
++struct in_memory_archive {
++ uint8_t *buffer;
++ size_t allocated;
++ size_t used;
++ /* If not 0, aborts writing when the used data would become larger than this. */
++ size_t maxsize;
++};
++
++ssize_t in_memory_write(struct archive *, void *client_data, const void *buffer, size_t length);
++
++#endif /* __INCLUDE_GUARD_LIBARCHIVE_HELPER_H */
+diff --git a/src/fullfiles.c b/src/fullfiles.c
+index 216a1d7..9fdfc2f 100644
+--- a/src/fullfiles.c
++++ b/src/fullfiles.c
+@@ -22,6 +22,8 @@
+ */
+
+ #define _GNU_SOURCE
++#include <archive.h>
++#include <archive_entry.h>
+ #include <assert.h>
+ #include <errno.h>
+ #include <fcntl.h>
+@@ -33,24 +35,27 @@
+ #include <string.h>
+ #include <sys/stat.h>
+ #include <sys/types.h>
++#include <sys/xattr.h>
+ #include <unistd.h>
+
+ #include "swupd.h"
++#include "libarchive_helper.h"
+
+ /* output must be a file, which is a (compressed) tar file, of the file denoted by "file", without any of its
+ directory paths etc etc */
+ static void create_fullfile(struct file *file)
+ {
+- char *origin;
++ char *origin = NULL;
+ char *tarname = NULL;
+- char *rename_source = NULL;
+- char *rename_target = NULL;
+- char *rename_tmpdir = NULL;
+- int ret;
+ struct stat sbuf;
+ char *empty, *indir, *outdir;
+- char *param1, *param2;
+- int stderrfd;
++ struct archive_entry *entry = NULL;
++ struct archive *from = NULL, *to = NULL;
++ struct in_memory_archive best = { .buffer = NULL };
++ struct in_memory_archive current = { .buffer = NULL };
++ uint8_t *file_content = NULL;
++ size_t file_size;
++ int fd = -1;
+
+ if (file->is_deleted) {
+ return; /* file got deleted -> by definition we cannot tar it up */
+@@ -59,15 +64,17 @@ static void create_fullfile(struct file *file)
+ empty = config_empty_dir();
+ indir = config_image_base();
+ outdir = config_output_dir();
++ entry = archive_entry_new();
++ assert(entry);
++ from = archive_read_disk_new();
++ assert(from);
+
+ string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
+ if (access(tarname, R_OK) == 0) {
+ /* output file already exists...done */
+- free(tarname);
++ goto done;
+ return;
+ }
+- free(tarname);
+- //printf("%s was missing\n", file->hash);
+
+ string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
+ if (lstat(origin, &sbuf) < 0) {
+@@ -76,156 +83,195 @@ static void create_fullfile(struct file *file)
+ assert(0);
+ }
+
+- if (file->is_dir) { /* directories are easy */
+- char *tmp1, *tmp2, *dir, *base;
+-
+- tmp1 = strdup(origin);
+- assert(tmp1);
+- base = basename(tmp1);
+-
+- tmp2 = strdup(origin);
+- assert(tmp2);
+- dir = dirname(tmp2);
+-
+- string_or_die(&rename_tmpdir, "%s/XXXXXX", outdir);
+- if (!mkdtemp(rename_tmpdir)) {
+- LOG(NULL, "Failed to create temporary directory for %s move", origin);
+- assert(0);
+- }
+-
+- string_or_die(&param1, "--exclude=%s/?*", base);
+- string_or_die(&param2, "./%s", base);
+- char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL };
+- char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL };
+-
+- stderrfd = open("/dev/null", O_WRONLY);
+- if (stderrfd == -1) {
+- LOG(NULL, "Failed to open /dev/null", "");
+- assert(0);
+- }
+- if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) {
+- assert(0);
+- }
+- free(param1);
+- free(param2);
+- close(stderrfd);
+-
+- string_or_die(&rename_source, "%s/%s", rename_tmpdir, base);
+- string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash);
+- if (rename(rename_source, rename_target)) {
+- LOG(NULL, "rename failed for %s to %s", rename_source, rename_target);
++ /* step 1: tar it with each compression type */
++ typedef int (*filter_t)(struct archive *);
++ static const filter_t compression_filters[] = {
++ /*
++ * Start with the compression method that is most likely (*) to produce
++ * the best result. That will allow aborting creation of archives earlier
++ * when they become larger than the currently smallest archive.
++ *
++ * (*) statistics for ostro-image-swupd:
++ * 43682 LZMA
++ * 13398 gzip
++ * 844 bzip2
++ */
++ archive_write_add_filter_lzma,
++ archive_write_add_filter_gzip,
++ archive_write_add_filter_bzip2,
++ /*
++ * TODO (?): can archive_write_add_filter_none ever be better than compressing?
++ */
++ NULL
++ };
++ file_size = S_ISREG(sbuf.st_mode) ? sbuf.st_size : 0;
++
++ archive_entry_copy_sourcepath(entry, origin);
++ if (archive_read_disk_entry_from_file(from, entry, -1, &sbuf)) {
++ LOG(NULL, "Getting directory attributes failed", "%s: %s",
++ origin, archive_error_string(from));
++ assert(0);
++ }
++ archive_entry_copy_pathname(entry, file->hash);
++ if (file_size) {
++ file_content = malloc(file_size);
++ if (!file_content) {
++ LOG(NULL, "out of memory", "");
+ assert(0);
+ }
+- free(rename_source);
+-
+- /* for a directory file, tar up simply with gzip */
+- string_or_die(&param1, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
+- char *const tarcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param1, file->hash, NULL };
+-
+- if (system_argv(tarcmd) != 0) {
++ fd = open(origin, O_RDONLY);
++ if (fd == -1) {
++ LOG(NULL, "Failed to open file", "%s: %s",
++ origin, strerror(errno));
+ assert(0);
+ }
+- free(param1);
+-
+- if (rmdir(rename_target)) {
+- LOG(NULL, "rmdir failed for %s", rename_target);
+- }
+- free(rename_target);
+- if (rmdir(rename_tmpdir)) {
+- LOG(NULL, "rmdir failed for %s", rename_tmpdir);
+- }
+- free(rename_tmpdir);
+-
+- free(tmp1);
+- free(tmp2);
+- } else { /* files are more complex */
+- char *gzfile = NULL, *bzfile = NULL, *xzfile = NULL;
+- char *tempfile;
+- uint64_t gz_size = LONG_MAX, bz_size = LONG_MAX, xz_size = LONG_MAX;
+-
+- /* step 1: hardlink the guy to an empty directory with the hash as the filename */
+- string_or_die(&tempfile, "%s/%s", empty, file->hash);
+- if (link(origin, tempfile) < 0) {
+- LOG(NULL, "hardlink failed", "%s due to %s (%s -> %s)", file->filename, strerror(errno), origin, tempfile);
+- char *const argv[] = { "cp", "-a", origin, tempfile, NULL };
+- if (system_argv(argv) != 0) {
++ size_t done = 0;
++ while (done < file_size) {
++ ssize_t curr;
++ curr = read(fd, file_content + done, file_size - done);
++ if (curr == -1) {
++ LOG(NULL, "Failed to read from file", "%s: %s",
++ origin, strerror(errno));
+ assert(0);
+ }
++ done += curr;
+ }
++ }
+
+- /* step 2a: tar it with each compression type */
+- // lzma
+- string_or_die(&param1, "--directory=%s", empty);
+- string_or_die(&param2, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash);
+- char *const tarlzmacmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf", param2, file->hash, NULL };
+-
+- if (system_argv(tarlzmacmd) != 0) {
++ for (int i = 0; compression_filters[i]; i++) {
++ /* Need to re-initialize the archive handle, it cannot be re-used. */
++ if (to) {
++ archive_write_free(to);
++ }
++ /*
++ * Use the recommended restricted pax interchange
++ * format. Numeric uid/gid values are stored in the archive
++ * (no uid/gid lookup enabled) because symbolic names can lead
++ * to a hash mismatch during unpacking when /etc/passwd or
++ * /etc/group change during an update (see
++ * https://github.com/clearlinux/swupd-client/issues/101).
++ *
++ * Filenames read from the file system are expected to be
++ * valid according to the current locale. archive_write_header()
++ * will warn about filenames that it cannot properly decode
++ * and proceeds by writing the raw bytes, but we treat this an
++ * error by not distinguishing between ARCHIVE_FATAL
++ * and ARCHIVE_WARN.
++ *
++ * When we fail with "Can't translate" errors, make sure that
++ * LANG and/or LC_ env variables are set.
++ */
++ to = archive_write_new();
++ assert(to);
++ if (archive_write_set_format_pax_restricted(to)) {
++ LOG(NULL, "PAX format", "%s", archive_error_string(to));
+ assert(0);
+ }
+- free(param1);
+- free(param2);
+-
+- // gzip
+- string_or_die(&param1, "--directory=%s", empty);
+- string_or_die(&param2, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash);
+- char *const targzipcmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param2, file->hash, NULL };
+-
+- if (system_argv(targzipcmd) != 0) {
++ do {
++ /* Try compression methods until we find one which is supported. */
++ if (!compression_filters[i](to)) {
++ break;
++ }
++ } while(compression_filters[++i]);
++ /*
++ * Regardless of the block size below, never pad the
++ * last block, it just makes the archive larger.
++ */
++ if (archive_write_set_bytes_in_last_block(to, 1)) {
++ LOG(NULL, "Removing padding failed", "");
+ assert(0);
+ }
+- free(param1);
+- free(param2);
+-
+-#ifdef SWUPD_WITH_BZIP2
+- string_or_die(&param1, "--directory=%s", empty);
+- string_or_die(&param2, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash);
+- char *const tarbzip2cmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-jcf", param2, file->hash, NULL };
+-
+- if (system_argv(tarbzip2cmd) != 0) {
++ /*
++ * Invoke in_memory_write() as often as possible and check each
++ * time whether we are already larger than the currently best
++ * algorithm.
++ */
++ current.maxsize = best.used;
++ if (archive_write_set_bytes_per_block(to, 0)) {
++ LOG(NULL, "Removing blocking failed", "");
+ assert(0);
+ }
+- free(param1);
+- free(param2);
+-
+-#endif
+-
+- /* step 2b: pick the smallest of the three compression formats */
+- string_or_die(&gzfile, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash);
+- if (stat(gzfile, &sbuf) == 0) {
+- gz_size = sbuf.st_size;
++ /*
++ * We can make an educated guess how large the resulting archive will be.
++ * Avoids realloc() calls when the file is big.
++ */
++ if (!current.allocated) {
++ current.allocated = file_size + 4096;
++ current.buffer = malloc(current.allocated);
+ }
+- string_or_die(&bzfile, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash);
+- if (stat(bzfile, &sbuf) == 0) {
+- bz_size = sbuf.st_size;
++ if (!current.buffer) {
++ LOG(NULL, "out of memory", "");
++ assert(0);
+ }
+- string_or_die(&xzfile, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash);
+- if (stat(xzfile, &sbuf) == 0) {
+- xz_size = sbuf.st_size;
++ if (archive_write_open(to, &current, NULL, in_memory_write, NULL)) {
++ LOG(NULL, "Failed to create archive", "%s",
++ archive_error_string(to));
++ assert(0);
+ }
+- string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash);
+- if (gz_size <= xz_size && gz_size <= bz_size) {
+- ret = rename(gzfile, tarname);
+- } else if (xz_size <= bz_size) {
+- ret = rename(xzfile, tarname);
+- } else {
+- ret = rename(bzfile, tarname);
++ if (archive_write_header(to, entry) ||
++ file_content && archive_write_data(to, file_content, file_size) != (ssize_t)file_size ||
++ archive_write_close(to)) {
++ if (current.maxsize && current.used >= current.maxsize) {
++ archive_write_free(to);
++ to = NULL;
++ continue;
++ }
++ LOG(NULL, "Failed to store file in archive", "%s: %s",
++ origin, archive_error_string(to));
++ assert(0);
+ }
+- if (ret != 0) {
+- LOG(file, "post-tar rename failed", "ret=%d", ret);
++ if (!best.used || current.used < best.used) {
++ free(best.buffer);
++ best = current;
++ memset(&current, 0, sizeof(current));
++ } else {
++ /* Simply re-use the buffer for the next iteration. */
++ current.used = 0;
+ }
+- unlink(bzfile);
+- unlink(xzfile);
+- unlink(gzfile);
+- free(bzfile);
+- free(xzfile);
+- free(gzfile);
+- free(tarname);
+-
+- /* step 3: remove the hardlink */
+- unlink(tempfile);
+- free(tempfile);
++ }
++ if (!best.used) {
++ LOG(NULL, "creating archive failed with all compression methods", "");
++ assert(0);
+ }
+
++ /* step 2: write out to disk. Archives are immutable and thus read-only. */
++ fd = open(tarname, O_CREAT|O_WRONLY, S_IRUSR|S_IRGRP|S_IROTH);
++ if (fd <= 0) {
++ LOG(NULL, "Failed to create archive", "%s: %s",
++ tarname, strerror(errno));
++ assert(0);
++ }
++ size_t done = 0;
++ while (done < best.used) {
++ ssize_t curr;
++ curr = write(fd, best.buffer + done, best.used - done);
++ if (curr == -1) {
++ LOG(NULL, "Failed to write archive", "%s: %s",
++ tarname, strerror(errno));
++ assert(0);
++ }
++ done += curr;
++ }
++ if (close(fd)) {
++ LOG(NULL, "Failed to complete writing archive", "%s: %s",
++ tarname, strerror(errno));
++ assert(0);
++ }
++ fd = -1;
++ free(best.buffer);
++ free(current.buffer);
++ free(file_content);
++
++ done:
++ if (fd >= 0) {
++ close(fd);
++ }
++ archive_read_free(from);
++ if (to) {
++ archive_write_free(to);
++ }
++ archive_entry_free(entry);
++ free(tarname);
+ free(indir);
+ free(outdir);
+ free(empty);
+diff --git a/src/in_memory_archive.c b/src/in_memory_archive.c
+new file mode 100644
+index 0000000..abd7e54
+--- /dev/null
++++ b/src/in_memory_archive.c
+@@ -0,0 +1,67 @@
++/*
++ * Software Updater - server side
++ *
++ * Copyright © 2016 Intel Corporation.
++ *
++ * This program is free software: you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation, version 2 or later of the License.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
++ * GNU General Public License for more details.
++ *
++ * You should have received a copy of the GNU General Public License
++ * along with this program. If not, see <http://www.gnu.org/licenses/>.
++ *
++ * Authors:
++ * Patrick Ohly <patrick.ohly@intel.com>
++ *
++ */
++
++#include <errno.h>
++#include <stdlib.h>
++
++#include "libarchive_helper.h"
++
++ssize_t in_memory_write(struct archive *archive, void *client_data, const void *buffer, size_t length)
++{
++ struct in_memory_archive *in_memory = client_data;
++ void *newbuff;
++
++ if (in_memory->maxsize && in_memory->used + length >= in_memory->maxsize) {
++ archive_set_error(archive, EFBIG, "resulting archive would become larger than %lu",
++ (unsigned long)in_memory->maxsize);
++ archive_write_fail(archive);
++ /*
++ * Despite the error and archive_write_fail(), libarchive internally calls us
++ * again and when we fail again, overwrites our error with something about
++ * "Failed to clean up compressor". Therefore our caller needs to check for "used == maxsize"
++ * to detect that we caused the failure.
++ */
++ in_memory->used = in_memory->maxsize;
++ return -1;
++ }
++
++ if (in_memory->used + length > in_memory->allocated) {
++ /* Start with a small chunk, double in size to avoid too many reallocs. */
++ size_t new_size = in_memory->allocated ?
++ in_memory->allocated * 2 :
++ 4096;
++ while (new_size < in_memory->used + length) {
++ new_size *= 2;
++ }
++ newbuff = realloc(in_memory->buffer, new_size);
++ if (!newbuff) {
++ archive_set_error(archive, ENOMEM, "failed to enlarge buffer");
++ return -1;
++ }
++ in_memory->buffer = newbuff;
++ in_memory->allocated = new_size;
++ }
++
++ memcpy(in_memory->buffer + in_memory->used, buffer, length);
++ in_memory->used += length;
++ return length;
++}
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch b/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch
index 4f5ba19..4f5ba19 100644
--- a/recipes-core/swupd-server/swupd-server/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch
+++ b/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch
diff --git a/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch
index a3a72ad..f27b111 100644
--- a/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch
+++ b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch
@@ -1,7 +1,7 @@
-From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001
+From 4be3d38364cc1bead38c32465cb7e2f3a0be4c88 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 16 Nov 2016 14:26:30 +0100
-Subject: [PATCH 1/3] create_pack: rely less on previous builds
+Subject: [PATCH 11/13] create_pack: rely less on previous builds
When a file has not been modified in the current build, then by
definition the current copy of the file is the same as in the build
@@ -28,7 +28,10 @@ make_pack_full_files() and make_final_pack() used the exact same code
for populating the "staged" directory. Now that common code is in
stage_entry().
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+
---
include/swupd.h | 2 +-
src/delta.c | 4 +--
@@ -36,32 +39,32 @@ Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
3 files changed, 62 insertions(+), 49 deletions(-)
diff --git a/include/swupd.h b/include/swupd.h
-index c1c0e96..cf384e3 100644
+index 2263018..1b3a046 100644
--- a/include/swupd.h
+++ b/include/swupd.h
-@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest);
+@@ -247,7 +247,7 @@ extern void type_change_detection(struct manifest *manifest);
extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
extern void link_renames(GList *newfiles, struct manifest *from_manifest);
--extern void __create_delta(struct file *file, int from_version);
-+extern void __create_delta(struct file *file, int from_version, int to_version);
+-extern void __create_delta(struct file *file, int from_version, char *from_hash);
++extern void __create_delta(struct file *file, int from_version, int to_version, char *from_hash);
extern void account_delta_hit(void);
extern void account_delta_miss(void);
diff --git a/src/delta.c b/src/delta.c
-index 7e978b1..8fff4c9 100644
+index 563abb9..8c1bc64 100644
--- a/src/delta.c
+++ b/src/delta.c
@@ -35,7 +35,7 @@
#include "swupd.h"
#include "xattrs.h"
--void __create_delta(struct file *file, int from_version)
-+void __create_delta(struct file *file, int from_version, int to_version)
+-void __create_delta(struct file *file, int from_version, char *from_hash)
++void __create_delta(struct file *file, int from_version, int to_version, char *from_hash)
{
- char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL;
+ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL, *conf = NULL;
char *tmpdir = NULL;
-@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version)
+@@ -59,7 +59,7 @@ void __create_delta(struct file *file, int from_version, char *from_hash)
}
conf = config_image_base();
@@ -71,7 +74,7 @@ index 7e978b1..8fff4c9 100644
string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
diff --git a/src/pack.c b/src/pack.c
-index 984c2d6..ccb28bd 100644
+index a7094b8..f7770b2 100644
--- a/src/pack.c
+++ b/src/pack.c
@@ -37,6 +37,7 @@
@@ -184,8 +187,8 @@ index 984c2d6..ccb28bd 100644
/* if the file was not found in the from version, skip delta creation */
if (file->peer) {
-- __create_delta(file, file->peer->last_change);
-+ __create_delta(file, file->peer->last_change, *to_version);
+- __create_delta(file, file->peer->last_change, file->peer->hash);
++ __create_delta(file, file->peer->last_change, *to_version, file->peer->hash);
}
}
@@ -194,8 +197,8 @@ index 984c2d6..ccb28bd 100644
{
GThreadPool *threadpool;
GList *item;
-@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files)
- sysconf(_SC_NPROCESSORS_ONLN);
+@@ -290,7 +323,7 @@ static void make_pack_deltas(GList *files)
+ int numthreads = num_threads(1.0);
LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
- threadpool = g_thread_pool_new(create_delta, NULL,
@@ -203,7 +206,7 @@ index 984c2d6..ccb28bd 100644
numthreads, FALSE, NULL);
item = g_list_first(files);
-@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack)
+@@ -365,7 +398,7 @@ static int make_final_pack(struct packdata *pack)
file->last_change, file->hash);
string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
pack->module, pack->from, pack->to, file->hash);
@@ -212,7 +215,7 @@ index 984c2d6..ccb28bd 100644
string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
pack->module, pack->from, pack->to, file->hash);
-@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack)
+@@ -399,27 +432,7 @@ static int make_final_pack(struct packdata *pack)
}
}
} else {
@@ -241,7 +244,7 @@ index 984c2d6..ccb28bd 100644
if (ret == 0) {
pack->fullcount++;
}
-@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack)
+@@ -537,7 +550,7 @@ int make_pack(struct packdata *pack)
/* step 2: consolidate delta list & create all delta files*/
delta_list = consolidate_packs_delta_files(delta_list, pack);
diff --git a/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch b/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch
index 7151c5c..db25851 100644
--- a/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch
+++ b/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch
@@ -1,7 +1,7 @@
-From e1f0d54a940eb7d04e2fbd59bd995a819331425f Mon Sep 17 00:00:00 2001
+From f35a8484f100ac0da0fbe173fcd5e8843321b000 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Fri, 30 Sep 2016 08:42:08 +0200
-Subject: [PATCH 1/2] swupd-create-update: alternative input layout
+Subject: [PATCH 07/13] swupd-create-update: alternative input layout
In Ostro OS, we already have a "full" directory with all files.
Splitting it up into bundles just so that swupd-create-update can
@@ -24,16 +24,18 @@ That way it is even possible to mix the two modes, i.e. replacing only
some bundles with a content list, although that's probably not all
that useful.
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/55]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+
---
- src/analyze_fs.c | 171 ++++++++++++++++++++++++++++++++++++++--------------
- src/chroot.c | 24 +++++---
+ src/analyze_fs.c | 176 +++++++++++++++++++++++++++++++++++++++-------------
+ src/chroot.c | 24 ++++---
src/create_update.c | 6 +-
- src/fullfiles.c | 2 +
- 4 files changed, 150 insertions(+), 53 deletions(-)
+ 3 files changed, 153 insertions(+), 53 deletions(-)
diff --git a/src/analyze_fs.c b/src/analyze_fs.c
-index 0f16343..d534587 100644
+index ce30393..ac3731c 100644
--- a/src/analyze_fs.c
+++ b/src/analyze_fs.c
@@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data)
@@ -45,15 +47,15 @@ index 0f16343..d534587 100644
{
char c;
int i;
-@@ -301,25 +301,145 @@ static bool illegal_characters(char *filename)
+@@ -301,27 +301,151 @@ static bool illegal_characters(char *filename)
return false;
}
-+static void add_file(struct manifest *manifest,
-+ const char *entry_name,
-+ char *sub_filename,
-+ char *fullname,
-+ bool do_hash)
++static struct file *add_file(struct manifest *manifest,
++ const char *entry_name,
++ char *sub_filename,
++ char *fullname,
++ bool do_hash)
+{
+ GError *err = NULL;
+ struct file *file;
@@ -62,7 +64,7 @@ index 0f16343..d534587 100644
+ printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+ free(sub_filename);
+ free(fullname);
-+ return;
++ return NULL;
+ }
+
+ file = calloc(1, sizeof(struct file));
@@ -104,6 +106,7 @@ index 0f16343..d534587 100644
+ }
+ manifest->files = g_list_prepend(manifest->files, file);
+ manifest->count++;
++ return file;
+}
+
+
@@ -120,10 +123,11 @@ index 0f16343..d534587 100644
dir = opendir(fullpath);
if (!dir) {
++ bool fatal_error = errno != ENOENT;
+ FILE *content;
-+ int len;
-+ free(fullpath);
-+ if (errno != ENOENT) {
++
+ free(fullpath);
++ if (fatal_error) {
+ return;
+ }
+ /*
@@ -134,14 +138,15 @@ index 0f16343..d534587 100644
+ * corresponding file system entry is then
+ * expected to be in a pre-populated "full"
+ * directory.
++ *
++ * Only supported at top level (i.e. empty
++ * subpath) to keep the code and testing
++ * simpler.
+ */
-+ if (subpath[0]) {
-+ string_or_die(&fullpath, "%s/%s.content.txt", pathprefix, len, subpath);
-+ } else {
-+ string_or_die(&fullpath, "%s.content.txt", pathprefix);
-+ }
++ assert(!subpath[0]);
++ string_or_die(&fullpath, "%s.content.txt", pathprefix);
+ content = fopen(fullpath, "r");
- free(fullpath);
++ free(fullpath);
+ fullpath = NULL;
+ if (content) {
+ char *line = NULL;
@@ -193,8 +198,11 @@ index 0f16343..d534587 100644
- struct file *file;
char *sub_filename;
char *fullname;
++ struct file *file;
-@@ -334,50 +454,13 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
+ entry = readdir(dir);
+ if (!entry) {
+@@ -334,50 +458,14 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
}
string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
@@ -216,13 +224,12 @@ index 0f16343..d534587 100644
string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
- populate_file_struct(file, fullname);
- free(fullname);
--
- if (entry->d_type == DT_DIR) {
+
+- if (file->is_dir) {
- iterate_directory(manifest, pathprefix, file->filename, do_hash);
-+ iterate_directory(manifest, pathprefix, sub_filename, do_hash);
- }
-+ /* takes ownership of the strings */
-+ add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
+- }
++ /* takes ownership of the strings, so we don't need to free it */
++ file = add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
- /* if for some reason there is a file in the official build
- * which should not be included in the Manifest, then open a bug
@@ -242,7 +249,9 @@ index 0f16343..d534587 100644
- closedir(dir);
- return;
- }
-- }
++ if (file && file->is_dir) {
++ iterate_directory(manifest, pathprefix, file->filename, do_hash);
+ }
- manifest->files = g_list_prepend(manifest->files, file);
- manifest->count++;
}
@@ -297,7 +306,7 @@ index 32ed997..f3832e1 100644
free(param);
}
diff --git a/src/create_update.c b/src/create_update.c
-index 766609b..74d5376 100644
+index 4a8156b..4a00a25 100644
--- a/src/create_update.c
+++ b/src/create_update.c
@@ -141,6 +141,7 @@ static bool parse_options(int argc, char **argv)
@@ -329,19 +338,6 @@ index 766609b..74d5376 100644
}
static int check_build_env(void)
-diff --git a/src/fullfiles.c b/src/fullfiles.c
-index 9fdfc2f..214b7b4 100644
---- a/src/fullfiles.c
-+++ b/src/fullfiles.c
-@@ -136,6 +136,8 @@ static void create_fullfile(struct file *file)
- }
- done += curr;
- }
-+ close(fd);
-+ fd = -1;
- }
-
- for (int i = 0; compression_filters[i]; i++) {
--
2.1.4
diff --git a/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch b/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch
index 1762d66..56ea17d 100644
--- a/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch
+++ b/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch
@@ -15,6 +15,8 @@ The implementation recycles the existing init_log_stdout() (not used
before) and gives it the slightly different meaning of "also log to
stdout".
+Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/72dd27a886ad9b2c66bb7cdb5c4cadb24f783654]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
src/create_update.c | 5 +++++
diff --git a/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch
index 4a2eac8..440cca8 100644
--- a/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch
+++ b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch
@@ -6,6 +6,8 @@ Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs
The fullfile .tar is needed for a pack as fallback when linking from
the full rootfs isn't possible. In practice this shouldn't happen.
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
src/pack.c | 13 +++++++++++++
diff --git a/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch
index 6cb2c57..7e981e3 100644
--- a/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch
+++ b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch
@@ -17,6 +17,8 @@ found.
This speeds up delta computation considerably in Ostro OS because it
skips the slow downloading of the old file.
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
src/delta.c | 13 +++++++++++++
diff --git a/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch
index 063c701..6e221b9 100644
--- a/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch
+++ b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch
@@ -1,8 +1,8 @@
-From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001
+From d05c819ff83f04145dac355e183b426b5802d6b4 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Tue, 8 Nov 2016 18:39:49 +0100
-Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for
- diffing
+Subject: [PATCH 09/12] swupd_create_pack: download original files on demand
+ for diffing
The new, optional mode gets enabled via the --content-url parameter.
When specified, swupd_create_pack will download the <original
@@ -28,19 +28,22 @@ bsdtar.
If for some reason a format change is necessary, delta computation
becomes impossible and falls back to staging the full files.
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+
---
include/swupd.h | 2 ++
- src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--
- src/globals.c | 13 ++++++++
- src/make_packs.c | 8 +++++
- 4 files changed, 111 insertions(+), 2 deletions(-)
+ src/delta.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-
+ src/globals.c | 13 +++++++++
+ src/make_packs.c | 8 ++++++
+ 4 files changed, 110 insertions(+), 1 deletion(-)
diff --git a/include/swupd.h b/include/swupd.h
-index 31c7191..c1c0e96 100644
+index f9c0583..e1fa6ff 100644
--- a/include/swupd.h
+++ b/include/swupd.h
-@@ -147,12 +147,14 @@ extern char *state_dir;
+@@ -149,12 +149,14 @@ extern char *state_dir;
extern char *packstage_dir;
extern char *image_dir;
extern char *staging_dir;
@@ -56,27 +59,25 @@ index 31c7191..c1c0e96 100644
extern void free_state_globals(void);
diff --git a/src/delta.c b/src/delta.c
-index 67e7df7..7e978b1 100644
+index ad40820..563abb9 100644
--- a/src/delta.c
+++ b/src/delta.c
-@@ -37,8 +37,14 @@
+@@ -37,7 +37,13 @@
- void __create_delta(struct file *file, int from_version)
+ void __create_delta(struct file *file, int from_version, char *from_hash)
{
-- char *original, *newfile, *outfile, *dotfile, *testnewfile;
-- char *conf, *param1, *param2;
-+ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL;
+- char *original, *newfile, *outfile, *dotfile, *testnewfile, *conf;
++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL, *conf = NULL;
+ char *tmpdir = NULL;
+ char *url = NULL;
+ char *cmd = NULL;
-+ char *conf = NULL, *param1 = NULL, *param2 = NULL;
+ bool delete_original = false;
+ struct manifest *manifest = NULL;
+ GError *gerror = NULL;
int ret;
if (file->is_link) {
-@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version)
+@@ -57,6 +63,74 @@ void __create_delta(struct file *file, int from_version, char *from_hash)
string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
@@ -151,7 +152,7 @@ index 67e7df7..7e978b1 100644
free(conf);
conf = config_output_dir();
-@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version)
+@@ -136,6 +210,18 @@ void __create_delta(struct file *file, int from_version, char *from_hash)
LOG(NULL, "Failed to rename", "");
}
out:
diff --git a/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch b/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch
index 707e1f0..43ae6ff 100644
--- a/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch
+++ b/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch
@@ -11,6 +11,8 @@ empty parameter to bsdtar, leading to:
To allow the the "no parameter" case, it has to be argument list: that
can be empty. If not empty, it has to end with a comma.
+Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/41]
+
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
---
include/swupd.h | 4 ++--
diff --git a/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch b/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch
index 0a68667..945e067 100644
--- a/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch
+++ b/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch
@@ -1,12 +1,14 @@
-From 8686189b3b080446fae732a85b72528b7fe68ba6 Mon Sep 17 00:00:00 2001
+From 5926a6ea8ca389ba11b467b231038b5bc5320a8b Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Tue, 27 Sep 2016 08:25:40 +0200
-Subject: [PATCH 27/29] update control over parallelism
+Subject: [PATCH 04/13] update control over parallelism
The SWUPD_NUM_THREADS env variable is now understood by all three
commands and overrides the default number of threads. Setting it to 1
is useful while debugging the code that runs inside threads (only one
-thread hits breakpoints there).
+thread hits breakpoints there). If SWUPD_NUM_THREADS is invalid, a
+warning is printed and the variable gets ignored, i.e. the default
+parallelism is used.
The hard-coded parallelism of 12 threads when analysing the file system
gets replaced with n, where n is the number of available CPUs. The default
@@ -15,27 +17,40 @@ is the same as before elsewhere (n for packing, 3 * n for fullfiles).
Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/4e0fdd4193a8ce9dcf3cfc5e488dfd4b23b7e7d9]
Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+
---
- src/analyze_fs.c | 5 ++++-
- src/fullfiles.c | 7 +++++--
- src/pack.c | 7 +++++--
- 3 files changed, 14 insertions(+), 5 deletions(-)
+ include/swupd.h | 1 +
+ src/analyze_fs.c | 3 ++-
+ src/fullfiles.c | 5 +++--
+ src/helpers.c | 26 ++++++++++++++++++++++++++
+ src/pack.c | 5 +++--
+ 5 files changed, 35 insertions(+), 5 deletions(-)
+diff --git a/include/swupd.h b/include/swupd.h
+index f9c0583..f334bdb 100644
+--- a/include/swupd.h
++++ b/include/swupd.h
+@@ -258,6 +258,7 @@ extern int system_argv(char *const argv[]);
+ extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr);
+ extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1,
+ char *const argvp2[], int stdoutp2, int stderrp2);
++extern int num_threads(float scaling);
+
+ extern bool signature_initialize(void);
+ extern void signature_terminate(void);
diff --git a/src/analyze_fs.c b/src/analyze_fs.c
-index 3bfb288..0f16343 100644
+index a220d79..ce30393 100644
--- a/src/analyze_fs.c
+++ b/src/analyze_fs.c
-@@ -387,6 +387,9 @@ struct manifest *full_manifest_from_directory(int version)
+@@ -387,6 +387,7 @@ struct manifest *full_manifest_from_directory(int version)
{
struct manifest *manifest;
char *dir;
-+ int numthreads = getenv("SWUPD_NUM_THREADS") ?
-+ atoi(getenv("SWUPD_NUM_THREADS")) :
-+ sysconf(_SC_NPROCESSORS_ONLN);
++ int numthreads = num_threads(1.0);
LOG(NULL, "Computing hashes", "for %i/full", version);
-@@ -394,7 +397,7 @@ struct manifest *full_manifest_from_directory(int version)
+@@ -394,7 +395,7 @@ struct manifest *full_manifest_from_directory(int version)
string_or_die(&dir, "%s/%i/full", image_dir, version);
@@ -45,16 +60,14 @@ index 3bfb288..0f16343 100644
iterate_directory(manifest, dir, "", true);
diff --git a/src/fullfiles.c b/src/fullfiles.c
-index 3be43d1..216a1d7 100644
+index ca8feda..882e83b 100644
--- a/src/fullfiles.c
+++ b/src/fullfiles.c
-@@ -291,10 +291,13 @@ static void submit_fullfile_tasks(GList *files)
+@@ -291,10 +291,11 @@ static void submit_fullfile_tasks(GList *files)
int ret;
int count = 0;
GError *err = NULL;
-+ int numthreads = getenv("SWUPD_NUM_THREADS") ?
-+ atoi(getenv("SWUPD_NUM_THREADS")) :
-+ sysconf(_SC_NPROCESSORS_ONLN) * 3;
++ int numthreads = num_threads(3.0);
- LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3);
+ LOG(NULL, "fullfile threadpool", "%d threads", numthreads);
@@ -64,17 +77,49 @@ index 3be43d1..216a1d7 100644
TRUE, NULL);
printf("Starting downloadable fullfiles data creation\n");
+diff --git a/src/helpers.c b/src/helpers.c
+index 95d000c..c2ae10e 100644
+--- a/src/helpers.c
++++ b/src/helpers.c
+@@ -284,3 +284,29 @@ void check_root(void)
+ exit(EXIT_FAILURE);
+ }
+ }
++
++int num_threads(float scaling)
++{
++ const char *var = getenv("SWUPD_NUM_THREADS");
++ int result = sysconf(_SC_NPROCESSORS_ONLN) * scaling;
++ float scaled_result;
++
++ if (var && *var) {
++ char *endptr;
++ long int value;
++
++ errno = 0;
++ value = strtol(var, &endptr, 0);
++
++ if ((errno != 0 && value == 0) || *endptr) {
++ LOG(NULL, "SWUPD_NUM_THREADS must be an integer", "%s", var);
++ } else if ((errno == ERANGE && (value == LONG_MAX || value == LONG_MIN)) ||
++ value < 1 || value > INT_MAX) {
++ LOG(NULL, "SWUPD_NUM_THREADS out of range", "%s", var);
++ } else {
++ result = (int)value;
++ }
++ }
++
++ return result;
++}
diff --git a/src/pack.c b/src/pack.c
-index 12d7443..3ffb88a 100644
+index 12d7443..2753e1f 100644
--- a/src/pack.c
+++ b/src/pack.c
-@@ -285,10 +285,13 @@ static void make_pack_deltas(GList *files)
+@@ -285,10 +285,11 @@ static void make_pack_deltas(GList *files)
struct file *file;
int ret;
GError *err = NULL;
-+ int numthreads = getenv("SWUPD_NUM_THREADS") ?
-+ atoi(getenv("SWUPD_NUM_THREADS")) :
-+ sysconf(_SC_NPROCESSORS_ONLN);
++ int numthreads = num_threads(1.0);
- LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN));
+ LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
diff --git a/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch b/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch
index d599261..d62cf7c 100644
--- a/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch
+++ b/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch
@@ -1,7 +1,7 @@
-From 4ebc886b910fa4cb83864658069a30dd133caca8 Mon Sep 17 00:00:00 2001
+From b02a3ba1a61868db334d516f2577aa7aa7f07d6f Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 30 Mar 2016 13:14:42 +0200
-Subject: [PATCH 3/6] fullfiles: use libarchive directly
+Subject: [PATCH 06/13] fullfiles: use libarchive directly
Calling an external tar command makes the code fairly complicated,
because it is necessary to set up a suitable temporary directory with
@@ -45,14 +45,14 @@ Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
Makefile.am | 6 +-
configure.ac | 1 +
include/libarchive_helper.h | 42 ++++++
- src/fullfiles.c | 324 +++++++++++++++++++++++++-------------------
+ src/fullfiles.c | 326 +++++++++++++++++++++++++-------------------
src/in_memory_archive.c | 67 +++++++++
- 5 files changed, 300 insertions(+), 140 deletions(-)
+ 5 files changed, 302 insertions(+), 140 deletions(-)
create mode 100644 include/libarchive_helper.h
create mode 100644 src/in_memory_archive.c
diff --git a/Makefile.am b/Makefile.am
-index 528f02f..b14feb0 100644
+index 0ae2643..c1c8e07 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -23,6 +23,7 @@ swupd_create_update_SOURCES = \
@@ -63,15 +63,15 @@ index 528f02f..b14feb0 100644
src/manifest.c \
src/pack.c \
src/rename.c \
-@@ -54,6 +55,7 @@ swupd_make_fullfiles_SOURCES = \
- src/fullfiles.c \
+@@ -56,6 +57,7 @@ swupd_make_fullfiles_SOURCES = \
src/globals.c \
+ src/groups.c \
src/helpers.c \
+ src/in_memory_archive.c \
src/log.c \
src/make_fullfiles.c \
src/manifest.c \
-@@ -63,12 +65,13 @@ swupd_make_fullfiles_SOURCES = \
+@@ -65,12 +67,13 @@ swupd_make_fullfiles_SOURCES = \
src/stats.c \
src/xattrs.c
@@ -86,7 +86,7 @@ index 528f02f..b14feb0 100644
$(bsdiff_LIBS)
swupd_make_pack_LDADD = \
-@@ -81,6 +84,7 @@ swupd_make_fullfiles_LDADD = \
+@@ -83,6 +86,7 @@ swupd_make_fullfiles_LDADD = \
$(glib_LIBS) \
$(zlib_LIBS) \
$(openssl_LIBS) \
@@ -95,7 +95,7 @@ index 528f02f..b14feb0 100644
if ENABLE_LZMA
diff --git a/configure.ac b/configure.ac
-index 8819206..3e2d933 100644
+index 277146b..b4943b1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -36,6 +36,7 @@ AC_ARG_ENABLE(
@@ -155,7 +155,7 @@ index 0000000..ad28def
+
+#endif /* __INCLUDE_GUARD_LIBARCHIVE_HELPER_H */
diff --git a/src/fullfiles.c b/src/fullfiles.c
-index 216a1d7..9fdfc2f 100644
+index 882e83b..e27d9ce 100644
--- a/src/fullfiles.c
+++ b/src/fullfiles.c
@@ -22,6 +22,8 @@
@@ -223,7 +223,7 @@ index 216a1d7..9fdfc2f 100644
string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename);
if (lstat(origin, &sbuf) < 0) {
-@@ -76,156 +83,195 @@ static void create_fullfile(struct file *file)
+@@ -76,156 +83,197 @@ static void create_fullfile(struct file *file)
assert(0);
}
@@ -349,6 +349,8 @@ index 216a1d7..9fdfc2f 100644
}
+ done += curr;
}
++ close(fd);
++ fd = -1;
+ }
- /* step 2a: tar it with each compression type */
diff --git a/recipes-core/swupd-server/swupd-server_3.2.5.bb b/recipes-core/swupd-server/swupd-server_3.2.5.bb
new file mode 100644
index 0000000..632a11a
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server_3.2.5.bb
@@ -0,0 +1,60 @@
+SUMMARY = "swupd sofware update from Clear Linux - server component"
+HOMEPAGE = "https://github.com/clearlinux/swupd-server"
+LICENSE = "GPL-2.0"
+LIC_FILES_CHKSUM = "file://COPYING;md5=f8d90fb802930e30e49c39c8126a959e"
+
+DEPENDS = "file glib-2.0 rsync openssl libarchive bsdiff bzip2"
+# Need the special "-replacement" variant because bzip2 and file
+# are assumed to be provided and would not get built.
+DEPENDS_append_class-native = " file-replacement-native bzip2-replacement-native"
+
+# This matches the SWUPD_TOOLS_FORMAT in swupd-image.bbclass.
+# When updating to a new release which changes the format of
+# the output, copy the recipe first to ensure that the old
+# release is still available if needed by swupd-image.bbclass,
+# then bump this number.
+#
+# The rest of the recipe ensures that different swupd-server
+# versions can be build and installed in parallel (format
+# number embedded in PN and the resulting files).
+SWUPD_SERVER_FORMAT = "3"
+PN = "swupd-server-format${SWUPD_SERVER_FORMAT}"
+FILESEXTRAPATHS_prepend = "${THISDIR}/swupd-server-3.2.5:"
+PV = "3.2.5+git${SRCPV}"
+SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \
+ file://0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch \
+ file://0026-fullfiles.c-fix-invalid-LOG-call.patch \
+ file://0027-update-control-over-parallelism.patch \
+ file://0028-enable-locales-in-all-programs.patch \
+ file://0029-fullfiles-use-libarchive-directly.patch \
+ file://0001-swupd-create-update-alternative-input-layout.patch \
+ file://0002-add-logging-to-stdout.patch \
+ file://swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch \
+ file://0001-delta.c-fix-xattr-test-after-patching.patch \
+ file://0002-pack.c-do-not-clean-packstage.patch \
+ file://0003-swupd_create_pack-download-original-files-on-demand-.patch \
+ file://0001-create_pack-rely-less-on-previous-builds.patch \
+ file://0002-create_pack-download-fullfile-on-demand-for-packs.patch \
+ file://0003-create_pack-abort-delta-handling-early-when-impossib.patch \
+ "
+SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55"
+
+S = "${WORKDIR}/git"
+
+inherit pkgconfig autotools
+
+EXTRA_OECONF = "--enable-bzip2 --enable-lzma --disable-stateless --disable-tests --enable-bsdtar"
+
+# safer-calls-to-system-utilities.patch uses for loop initial declaration
+CFLAGS_append = " -std=c99"
+
+RDEPENDS_${PN} = "rsync"
+RDEPENDS_${PN}_class-target = " bsdtar"
+
+BBCLASSEXTEND = "native"
+
+do_install_append () {
+ for i in ${D}${bindir}/swupd_*; do
+ mv $i ${i}_${SWUPD_SERVER_FORMAT}
+ done
+}
diff --git a/recipes-core/swupd-server/swupd-server_git.bb b/recipes-core/swupd-server/swupd-server_git.bb
index 5f93034..dbe7b19 100644
--- a/recipes-core/swupd-server/swupd-server_git.bb
+++ b/recipes-core/swupd-server/swupd-server_git.bb
@@ -8,7 +8,19 @@ DEPENDS = "file glib-2.0 rsync openssl libarchive bsdiff bzip2"
# are assumed to be provided and would not get built.
DEPENDS_append_class-native = " file-replacement-native bzip2-replacement-native"
-PV = "3.2.5+git${SRCPV}"
+# This matches the SWUPD_TOOLS_FORMAT in swupd-image.bbclass.
+# When updating to a new release which changes the format of
+# the output, copy the recipe first to ensure that the old
+# release is still available if needed by swupd-image.bbclass,
+# then bump this number.
+#
+# The rest of the recipe ensures that different swupd-server
+# versions can be build and installed in parallel (format
+# number embedded in PN and the resulting files).
+SWUPD_SERVER_FORMAT = "4"
+PN = "swupd-server-format${SWUPD_SERVER_FORMAT}"
+FILESEXTRAPATHS_prepend = "${THISDIR}/swupd-server:"
+PV = "3.3.0+git${SRCPV}"
SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \
file://0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch \
file://0026-fullfiles.c-fix-invalid-LOG-call.patch \
@@ -17,15 +29,13 @@ SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \
file://0029-fullfiles-use-libarchive-directly.patch \
file://0001-swupd-create-update-alternative-input-layout.patch \
file://0002-add-logging-to-stdout.patch \
- file://swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch \
- file://0001-delta.c-fix-xattr-test-after-patching.patch \
file://0002-pack.c-do-not-clean-packstage.patch \
file://0003-swupd_create_pack-download-original-files-on-demand-.patch \
file://0001-create_pack-rely-less-on-previous-builds.patch \
file://0002-create_pack-download-fullfile-on-demand-for-packs.patch \
file://0003-create_pack-abort-delta-handling-early-when-impossib.patch \
"
-SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55"
+SRCREV = "9148ea8a8ecd73720d450d30acc4bceec310ab3b"
S = "${WORKDIR}/git"
@@ -40,3 +50,9 @@ RDEPENDS_${PN} = "rsync"
RDEPENDS_${PN}_class-target = " bsdtar"
BBCLASSEXTEND = "native"
+
+do_install_append () {
+ for i in ${D}${bindir}/swupd_*; do
+ mv $i ${i}_${SWUPD_SERVER_FORMAT}
+ done
+}