diff options
33 files changed, 2609 insertions, 839 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass index 0cac12d..5ba9cfb 100644 --- a/classes/swupd-image.bbclass +++ b/classes/swupd-image.bbclass @@ -32,11 +32,47 @@ SWUPD_IMAGE_PN = "${@ d.getVar('PN_BASE', True) or d.getVar('PN', True)}" # to be published will be in the "www" sub-directory. DEPLOY_DIR_SWUPD = "${DEPLOY_DIR}/swupd/${MACHINE}/${SWUPD_IMAGE_PN}" -# The current format has to match the the source code of the -# swupd-client that is in the image. This recipe picks a suitable -# swupd-client via the client's RPROVIDES. -SWUPD_FORMAT ??= "3" -IMAGE_INSTALL_append = " swupd-client-format${SWUPD_FORMAT}" +# The "format" needs to be bumped for different reasons: +# - the output of the swupd-server changes in a way that +# a swupd-client currently installed on devices will not +# understand it (example: changing file names or using +# a new compression method for archives) +# - the content of the distro changes such that a device +# cannot update directly to the latest build (example: +# the distro changes the boot loader and some swupd postinst +# helper which knows about that change must be installed on +# the device first before actually switching) +# +# meta-swupd handles the first case with SWUPD_TOOLS_FORMAT. +# The default value matches the default versions of the swupd-server +# and swupd-client. Distros can override this if they need to pick +# non-default versions of the tools, but that is not tested. +# +# Distros need to handle the second case by preparing and releasing +# a build that devices can update to (i.e. the version URL the devices +# check must have that update), then make the incompatible change and +# in the next build bump the SWUPD_DISTRO_FORMAT. +# +# In both cases, SWUPD_FORMAT gets bumped. meta-swupd notices that +# and then prepares a special transitional update: +# - the rootfs is configured to use the new SWUUPD_FORMAT and +# OS_VERSION +# - a fake OS_VERSION-1 release is built using a swupd-server that is +# compatible with the swupd-client before the bump +# - the OS_VERSION release then is the first one using the new format +# +# This way, devices are forced to update to OS_VERSION-1 because that +# will forever be the "latest" version for their current format. +# Once they have updated, the device really is on OS_VERSION, configured +# to use the new format, and the next update check will see future +# releases again. +# +# For this to work, "swupd-client" should always be invoked without +# explicit format parameter. +SWUPD_TOOLS_FORMAT ?= "4" +SWUPD_DISTRO_FORMAT ?= "0" +SWUPD_FORMAT = "${@ str(int('${SWUPD_TOOLS_FORMAT}') + int('${SWUPD_DISTRO_FORMAT}')) }" +IMAGE_INSTALL_append = " swupd-client-format${SWUPD_TOOLS_FORMAT}" # The information about where to find version information and actual # content is needed in several places: @@ -70,6 +106,11 @@ SWUPD_LOG_FN ??= "bbdebug 1" # a non-negative integer that fits in an int. OS_VERSION ??= "${DISTRO_VERSION}" +# When doing format changes, this version number is used for the intermediate +# release. Default is OS_VERSION - 1. There's a separate sanity check for +# OS_VERSION below, so this code should always work. +OS_VERSION_INTERIM ?= "${@ ${OS_VERSION} - 1 }" + # We need to preserve xattrs, which works with bsdtar out of the box. # It also has saner file handling (less syscalls per file) than GNU tar. # Last but not least, GNU tar 1.27.1 had weird problems extracting @@ -325,6 +366,65 @@ do_swupd_update () { fi fi + swupd_format_of_version () { + if [ ! -f ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM ]; then + bbfatal "Cannot determine swupd format of $1, ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM not found." + exit 1 + fi + format=`head -1 ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM | perl -n -e '/^MANIFEST\s(\d+)$/ && print $1'` + if [ ! "$format" ]; then + bbfatal "Cannot determine swupd format of $1, ${DEPLOY_DIR_SWUPD}/www/$1/Manifest.MoM does not have MANIFEST with format number in first line." + exit 1 + fi + echo $format + } + + # do_fetch_swupd_inputs() creates this file when a content + # URL was set, so creating an empty file shouldn't be necessary + # in most cases. Also determine whether we are switching + # formats. + # + # When the new format is different compared to what was used by + # latest.version, then swupd-server will automatically ignore + # the old content. That includes the case where tool format + # hasn't changed and only the distro format was bumped. In that + # case, reusing old content would be possible, but swupd-server + # would have to be improved to know that. + if [ -e ${DEPLOY_DIR_SWUPD}/image/latest.version ]; then + PREVREL=`cat ${DEPLOY_DIR_SWUPD}/image/latest.version` + if [ ! -e ${DEPLOY_DIR_SWUPD}/www/$PREVREL/Manifest.MoM ]; then + bbfatal "${DEPLOY_DIR_SWUPD}/image/latest.version specifies $PREVREL as last version, but there is no corresponding ${DEPLOY_DIR_SWUPD}/www/$PREVREL/Manifest.MoM." + exit 1 + fi + PREVFORMAT=`swupd_format_of_version $PREVREL` + if [ ! "$PREVFORMAT" ]; then + bbfatal "Format number not found in first line of ${DEPLOY_DIR_SWUPD}/www/$PREVREL/Manifest.MoM" + exit 1 + fi + # For now assume that SWUPD_DISTRO_FORMAT is always 0 and that thus + # $PREVFORMAT also is the format of the previous tools. + PREVTOOLSFORMAT=$PREVFORMAT + + if [ $PREVFORMAT -ne ${SWUPD_FORMAT} ] && [ $PREVREL -ge ${OS_VERSION_INTERIM} ]; then + bbfatal "Building two releases because of a format change, so OS_VERSION - 1 = ${OS_VERSION_INTERIM} must be higher than last version $PREVREL." + elif [ $PREVREL -ge ${OS_VERSION} ]; then + bbfatal "OS_VERSION = ${OS_VERSION} must be higher than last version $PREVREL." + exit 1 + fi + else + bbdebug 2 "Stubbing out empty latest.version file" + touch ${DEPLOY_DIR_SWUPD}/image/latest.version + PREVREL="0" + PREVFORMAT=${SWUPD_FORMAT} + PREVTOOLSFORMAT=${SWUPD_FORMAT} + fi + + # swupd-server >= 3.2.8 uses a different name. Support old and new names + # via symlinking. + ln -sf latest.version ${DEPLOY_DIR_SWUPD}/image/LAST_VER + + ${SWUPD_LOG_FN} "Generating update from $PREVREL (format $PREVFORMAT) to ${OS_VERSION} (format ${SWUPD_FORMAT})" + # Generate swupd-server configuration bbdebug 2 "Writing ${DEPLOY_DIR_SWUPD}/server.ini" if [ -e "${DEPLOY_DIR_SWUPD}/server.ini" ]; then @@ -337,17 +437,6 @@ outputdir=${DEPLOY_DIR_SWUPD}/www/ emptydir=${DEPLOY_DIR_SWUPD}/empty/ END - # do_fetch_swupd_inputs() creates this file when a content - # URL was set, so creating an empty file shouldn't be necessary - # in most cases. - if [ -e ${DEPLOY_DIR_SWUPD}/image/latest.version ]; then - PREVREL=`cat ${DEPLOY_DIR_SWUPD}/image/latest.version` - else - bbdebug 2 "Stubbing out empty latest.version file" - touch ${DEPLOY_DIR_SWUPD}/image/latest.version - PREVREL="0" - fi - GROUPS_INI="${DEPLOY_DIR_SWUPD}/groups.ini" bbdebug 2 "Writing ${GROUPS_INI}" if [ -e "${DEPLOY_DIR_SWUPD}/groups.ini" ]; then @@ -400,64 +489,91 @@ END done } - ${SWUPD_LOG_FN} "Generating update from $PREVREL to ${OS_VERSION}" - # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd - invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_create_update --log-stdout -S ${DEPLOY_DIR_SWUPD} --osversion ${OS_VERSION} --format ${SWUPD_FORMAT} - - ${SWUPD_LOG_FN} "Generating fullfiles for ${OS_VERSION}" - # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd - invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles --log-stdout -S ${DEPLOY_DIR_SWUPD} ${OS_VERSION} - if [ "${SWUPD_CONTENT_URL}" ]; then content_url_parameter="--content-url ${SWUPD_CONTENT_URL}" else content_url_parameter="" fi - ${SWUPD_LOG_FN} "Generating zero packs, this can take some time." - # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd - # Generating zero packs isn't parallelized internally. Mostly it just - # spends its time compressing a single tar archive. Therefore we parallelize - # by forking each command and then waiting for all of them to complete. - jobs="" - for bndl in ${ALL_BUNDLES}; do - ${SWUPD_LOG_FN} "Generating zero pack for $bndl" - # The zero packs are used by the swupd client when adding bundles. - # The zero pack for os-core is not needed by the swupd client itself; - # in Clear Linux OS it is used by the installer. We could use some - # space by skipping the os-core zero bundle, but for now it gets - # generated, just in case that it has some future use. - invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} 0 ${OS_VERSION} $bndl & - jobs="$jobs $!" - done + create_version () { + swupd_format=$1 + tool_format=$2 + os_version=$3 + + # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-create-update.tar.gz -C ${DEPLOY_DIR} swupd + invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_create_update_$tool_format --log-stdout -S ${DEPLOY_DIR_SWUPD} --osversion $os_version --format $swupd_format + + ${SWUPD_LOG_FN} "Generating fullfiles for $os_version" + # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-fullfiles.tar.gz -C ${DEPLOY_DIR} swupd + invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_fullfiles_$tool_format --log-stdout -S ${DEPLOY_DIR_SWUPD} $os_version - # Generate delta-packs against previous versions chosen by our caller. - # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd - for prevver in ${SWUPD_DELTAPACK_VERSIONS}; do + ${SWUPD_LOG_FN} "Generating zero packs, this can take some time." + # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-zero-pack.tar.gz -C ${DEPLOY_DIR} swupd + # Generating zero packs isn't parallelized internally. Mostly it just + # spends its time compressing a single tar archive. Therefore we parallelize + # by forking each command and then waiting for all of them to complete. + jobs="" for bndl in ${ALL_BUNDLES}; do - bndlcnt=0 - ${SWUPD_LOG_FN} "Generating delta pack from $prevver to ${OS_VERSION} for $bndl" - invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl | sed -u -e "s/^/$prevver $bndl: /" & + # The zero packs are used by the swupd client when adding bundles. + # The zero pack for os-core is not needed by the swupd client itself; + # in Clear Linux OS it is used by the installer. We could use some + # space by skipping the os-core zero bundle, but for now it gets + # generated, just in case that it has some future use. + invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack_$tool_format --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} 0 $os_version $bndl | sed -u -e "s/^/$bndl: /" & jobs="$jobs $!" done - done - waitall $jobs + # Generate delta-packs against previous versions chosen by our caller, + # if possible. Different formats make this useless because the previous + # version won't be able to update to the new version directly. + # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-before-make-delta-pack.tar.gz -C ${DEPLOY_DIR} swupd + for prevver in ${SWUPD_DELTAPACK_VERSIONS}; do + old_swupd_format=`swupd_format_of_version $prevver` + if [ $old_swupd_format -eq $swupd_format ]; then + for bndl in ${ALL_BUNDLES}; do + ${SWUPD_LOG_FN} "Generating delta pack from $prevver to $os_version for $bndl" + invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack_$tool_format --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} $prevver $os_version $bndl | sed -u -e "s/^/$prevver $bndl: /" & + jobs="$jobs $!" + done + fi + done + + waitall $jobs - # Write version to www/version/format${SWUPD_FORMAT}/latest and image/latest.version - bbdebug 2 "Writing latest file" - mkdir -p ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT} - echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest + # Write version to www/version/format$swupd_format/latest. + bbdebug 2 "Writing latest file" + mkdir -p ${DEPLOY_DIR_SWUPD}/www/version/format$swupd_format + echo $os_version > ${DEPLOY_DIR_SWUPD}/www/version/format$swupd_format/latest + # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd + } + + if [ $PREVFORMAT -ne ${SWUPD_FORMAT} ]; then + # Exact same content (including the OS_VERSION in the os-release file), + # just different tool and/or format in the manifests. + ln -sf ${OS_VERSION} ${DEPLOY_DIR_SWUPD}/image/${OS_VERSION_INTERIM} + echo $PREVREL > ${DEPLOY_DIR_SWUPD}/image/latest.version + create_version $PREVFORMAT $PREVTOOLSFORMAT ${OS_VERSION_INTERIM} + fi + echo $PREVREL > ${DEPLOY_DIR_SWUPD}/image/latest.version + create_version ${SWUPD_FORMAT} ${SWUPD_TOOLS_FORMAT} ${OS_VERSION} echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version - # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd } SWUPDDEPENDS = "\ virtual/fakeroot-native:do_populate_sysroot \ rsync-native:do_populate_sysroot \ bsdiff-native:do_populate_sysroot \ - swupd-server-native:do_populate_sysroot \ " + +# We don't know exactly which formats will be in use during +# do_swupd_update. It depends on the content of the update +# repo, which is unavailable when dependencies are evaluated +# in preparation of the build. +# +# For now we simply build all supported server versions. +SWUPD_SERVER_FORMATS = "3 4" +SWUPDDEPENDS += "${@ ' '.join(['swupd-server-format%s-native:do_populate_sysroot' % x for x in '${SWUPD_SERVER_FORMATS}'.split()])}" + addtask swupd_update after do_image_complete before do_build do_swupd_update[depends] = "${SWUPDDEPENDS}" diff --git a/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch b/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch index 649f7da..06f138c 100644 --- a/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch +++ b/recipes-core/swupd-client/swupd-client/0001-Add-configure-option-to-re-enable-updating-of-config.patch @@ -1,22 +1,24 @@ -From d648898c8f9823a8d511507f93390079954bc24a Mon Sep 17 00:00:00 2001 +From 72bcbe256a6612954ea24175538660864e65e26d Mon Sep 17 00:00:00 2001 From: Joshua Lock <joshua.g.lock@intel.com> Date: Mon, 18 Apr 2016 13:30:18 +0100 -Subject: [PATCH] Add configure option to re-enable updating of config files +Subject: [PATCH 2/4] Add configure option to re-enable updating of config + files Upstream-Status: Pending Signed-off-by: Joshua Lock <joshua.g.lock@intel.com> + --- configure.ac | 7 +++++++ src/heuristics.c | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac -index b1905f0..4a5c029 100644 +index ee05258..5cbc450 100644 --- a/configure.ac +++ b/configure.ac -@@ -63,6 +63,13 @@ AS_IF([test "x$enable_bsdtar" = "xyes" ], - [AC_DEFINE(SWUPD_WITHOUT_BSDTAR, 1, [Use default tar])] +@@ -153,6 +153,13 @@ AS_IF([test "x$enable_tar_selinux" = "xyes"], + [TARSELINUX=no] ) +AC_ARG_ENABLE( @@ -30,7 +32,7 @@ index b1905f0..4a5c029 100644 [path to systemd system service dir @<:@default=/usr/lib/systemd/system@:>@]), [unitpath=${withval}], [unitpath="$($PKG_CONFIG --variable=systemdsystemunitdir systemd)"]) diff --git a/src/heuristics.c b/src/heuristics.c -index e7c0632..c488e17 100644 +index 1b5d6c9..a2615f3 100644 --- a/src/heuristics.c +++ b/src/heuristics.c @@ -27,6 +27,7 @@ @@ -53,5 +55,5 @@ index e7c0632..c488e17 100644 is_state(file->filename) || // ideally we trust the manifest but short term reapply check here (file->is_boot && file->is_deleted) || -- -2.5.5 +2.1.4 diff --git a/recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch b/recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch deleted file mode 100644 index 59875b7..0000000 --- a/recipes-core/swupd-client/swupd-client/0001-downloads-minimize-syscalls-to-improve-performance.patch +++ /dev/null @@ -1,186 +0,0 @@ -From 9bc713b7ed0dba91304c5d7ed4905f5924ad8e42 Mon Sep 17 00:00:00 2001 -From: Patrick Ohly <patrick.ohly@intel.com> -Date: Thu, 14 Apr 2016 11:03:31 +0200 -Subject: [PATCH 1/3] downloads: minimize syscalls to improve performance - -The previous approach was to open/fdopen/fclose the file for each -chunk that gets passed from curl. This incurrs a huge performance hit -when close() triggers a hashing of the file content on systems where -integrity protection via IMA is enabled. - -Now the file is opened only once and kept open until the download is -complete. In addition, the unnecessary usage of C file IO is avoided. - -The semantic is changed as little as possible: -- file gets created only after the first chunk of data arrived -- file descriptors do not leak to child processes (O_CLOEXEC) -- data gets appended to existing files (via O_APPEND, used - to keep the code simple and avoid an additional lseek) -- data gets flushed explicitly for each chunk (via fdatasync(), - which somewhat approximates the effect that an explicit - close() may have had) - -As an additional improvement, failures during close() are checked. To -keep error handling as much as before, the completion function which has -the close() takes the current curl error code and replaces it if it -encounters a write error. - -[v2 of the patch with fixes by Dmitry Rozhkov, see https://github.com/pohly/swupd-client/pull/1] - -Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> ---- - include/swupd.h | 3 +++ - src/curl.c | 63 ++++++++++++++++++++++++++++++++++++++++----------------- - src/download.c | 8 +++++++- - 3 files changed, 54 insertions(+), 20 deletions(-) - -diff --git a/include/swupd.h b/include/swupd.h -index 14e65ab..3bac8b2 100644 ---- a/include/swupd.h -+++ b/include/swupd.h -@@ -89,6 +89,7 @@ struct file { - int last_change; - struct update_stat stat; - -+ unsigned int fd_valid : 1; - unsigned int is_dir : 1; - unsigned int is_file : 1; - unsigned int is_link : 1; -@@ -109,6 +110,7 @@ struct file { - - char *staging; /* output name used during download & staging */ - CURL *curl; /* curl handle if downloading */ -+ int fd; /* file written into during downloading, unset when fd_valid is false */ - }; - - extern bool download_only; -@@ -199,6 +201,7 @@ extern void swupd_curl_set_current_version(int v); - extern void swupd_curl_set_requested_version(int v); - extern double swupd_query_url_content_size(char *url); - extern size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata); -+extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file); - extern int swupd_curl_get_file(const char *url, char *filename, struct file *file, - struct version_container *tmp_version, bool pack); - #define SWUPD_CURL_LOW_SPEED_LIMIT 1 -diff --git a/src/curl.c b/src/curl.c -index b14193b..cab1ef2 100644 ---- a/src/curl.c -+++ b/src/curl.c -@@ -165,35 +165,57 @@ size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata) - const char *outfile; - int fd; - FILE *f; -- size_t written; -+ size_t written, remaining; - - outfile = file->staging; -+ if (file->fd_valid) { -+ fd = file->fd; -+ } else { -+ fd = open(outfile, O_CREAT | O_RDWR | O_CLOEXEC | O_APPEND, 00600); -+ if (fd < 0) { -+ fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", -+ outfile, strerror(errno)); -+ return -1; -+ } -+ file->fd = fd; -+ file->fd_valid = 1; -+ } - -- fd = open(outfile, O_CREAT | O_RDWR, 00600); -- if (fd < 0) { -- printf("Error: Cannot open %s for write: %s\n", -- outfile, strerror(errno)); -- return -1; -+ /* handle short writes with repeated write() calls */ -+ for (remaining = size * nmemb; remaining; remaining -= written) { -+ written = write(fd, ptr, size*nmemb); -+ if (written < 0) { -+ if (errno == EINTR) { -+ written = 0; -+ continue; -+ } -+ fprintf(stderr, "write error \\*outfile=\"%s\",strerror=\"%s\"*\\\n", -+ outfile, strerror(errno)); -+ return -1; -+ } - } - -- f = fdopen(fd, "a"); -- if (!f) { -- printf("Error: Cannot fdopen %s for write: %s\n", -- outfile, strerror(errno)); -- close(fd); -+ if (fdatasync(fd)) { -+ fprintf(stderr, "fdatasync \\*outfile=\"%s\",strerror=\"%s\"*\\\n", outfile, strerror(errno)); - return -1; - } - -- written = fwrite(ptr, size * nmemb, 1, f); -- -- fflush(f); -- fclose(f); -+ return size*nmemb; -+} - -- if (written != 1) { -- return -1; -+CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file) -+{ -+ if (file->fd_valid) { -+ if (close(file->fd)) { -+ fprintf(stderr, "Cannot close file after write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", -+ file->staging, strerror(errno)); -+ if (curl_ret == CURLE_OK) { -+ curl_ret = CURLE_WRITE_ERROR; -+ } -+ } -+ file->fd_valid = 0; - } -- -- return size * nmemb; -+ return curl_ret; - } - - /* Download a single file SYNCHRONOUSLY -@@ -281,6 +303,9 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file, - } - - exit: -+ if (local) { -+ curl_ret = swupd_download_file_complete(curl_ret, local); -+ } - if (curl_ret == CURLE_OK) { - /* curl command succeeded, download might've failed, let our caller handle */ - switch (ret) { -diff --git a/src/download.c b/src/download.c -index 6d81d81..c4a7a07 100644 ---- a/src/download.c -+++ b/src/download.c -@@ -164,6 +164,7 @@ static void free_curl_list_data(void *data) - { - struct file *file = (struct file *)data; - CURL *curl = file->curl; -+ (void) swupd_download_file_complete(CURLE_OK, file); - if (curl != NULL) { - curl_multi_remove_handle(mcurl, curl); - curl_easy_cleanup(curl); -@@ -368,9 +369,14 @@ static int perform_curl_io_and_complete(int *left) - continue; - } - -+ /* Get error code from easy handle and augment it if -+ * completing the download encounters further problems. */ -+ curl_ret = msg->data.result; -+ curl_ret = swupd_download_file_complete(curl_ret, file); -+ - /* The easy handle may have an error set, even if the server returns - * HTTP 200, so retry the download for this case. */ -- if (ret == 200 && msg->data.result != CURLE_OK) { -+ if (ret == 200 && curl_ret != CURLE_OK) { - printf("Error for %s download: %s\n", file->hash, - curl_easy_strerror(msg->data.result)); - failed = list_prepend_data(failed, file); --- -2.1.4 - diff --git a/recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch b/recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch new file mode 100644 index 0000000..b3539df --- /dev/null +++ b/recipes-core/swupd-client/swupd-client/0001-fix-enable-xattr.patch @@ -0,0 +1,56 @@ +From a8c7a7a93e19b328a1a6e8114f21b2bff9ad4f69 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Thu, 17 Nov 2016 18:00:46 +0100 +Subject: [PATCH] fix --enable-xattr + +Commit fc0f570d added a check that prevents using --enable-xattr +together with --enable-bsdtar, perhaps because it was assumed that this +wouldn't work because there is no special tar option as in the GNU tar +case. + +But that combination works fine for (and is needed by) Ostro OS with +IMA and Smack xattrs, so the check needs to be removed. + +Besides that, enabling xattrs also had no effect because xattrs.c +never got to see the SWUPD_WITH_XATTRS define due to not including +config.h. + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-client/commit/a2b80dc6958b78885ec395f22d34996e71a5f58a] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + configure.ac | 5 +---- + src/xattrs.c | 1 + + 2 files changed, 2 insertions(+), 4 deletions(-) + +diff --git a/configure.ac b/configure.ac +index 5cbc450..2166fd0 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -135,10 +135,7 @@ AC_ARG_ENABLE( + AS_HELP_STRING([--enable-xattr],[Use extended file attributes (unused by default)]) + ) + AS_IF([test "x$enable_xattr" = "xyes"], +- [AC_DEFINE(SWUPD_WITH_XATTRS,1,[Use extended file attributes]) +- AS_IF(test "x$enable_bsdtar" = "xyes", +- echo "Options --enable-bsdtar and --enable-xattr are incompatible" >&2 +- AS_EXIT(1))], ++ [AC_DEFINE(SWUPD_WITH_XATTRS,1,[Use extended file attributes])], + [XATTR=no] + ) + TARSELINUX="yes" +diff --git a/src/xattrs.c b/src/xattrs.c +index 36d4241..bbca237 100644 +--- a/src/xattrs.c ++++ b/src/xattrs.c +@@ -27,6 +27,7 @@ + #include <string.h> + #include <sys/xattr.h> + ++#include "config.h" + #include "swupd.h" + #include "xattrs.h" + +-- +2.1.4 + diff --git a/recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch b/recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch deleted file mode 100644 index 4d8339a..0000000 --- a/recipes-core/swupd-client/swupd-client/0002-downloads-open-FILE-in-advance-and-use-default-write.patch +++ /dev/null @@ -1,184 +0,0 @@ -From 26c603ad25469d3e37fc00b78ad161b34093f5fa Mon Sep 17 00:00:00 2001 -From: Patrick Ohly <patrick.ohly@intel.com> -Date: Tue, 15 Nov 2016 15:01:38 +0100 -Subject: [PATCH 2/3] downloads: open FILE in advance and use default write - handler - -Now that the number of pending downloads is kept below a certain limit -(see poll_fewer_than()) it is possible to open files before starting -the transfer. Using the default curl write handler and explicit -open/close of the file makes the code simpler. - -Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> ---- - include/swupd.h | 5 ++--- - src/curl.c | 59 +++++++++++++-------------------------------------------- - src/download.c | 6 ++++-- - 3 files changed, 19 insertions(+), 51 deletions(-) - -diff --git a/include/swupd.h b/include/swupd.h -index 3bac8b2..ed4b82c 100644 ---- a/include/swupd.h -+++ b/include/swupd.h -@@ -89,7 +89,6 @@ struct file { - int last_change; - struct update_stat stat; - -- unsigned int fd_valid : 1; - unsigned int is_dir : 1; - unsigned int is_file : 1; - unsigned int is_link : 1; -@@ -110,7 +109,7 @@ struct file { - - char *staging; /* output name used during download & staging */ - CURL *curl; /* curl handle if downloading */ -- int fd; /* file written into during downloading, unset when fd_valid is false */ -+ FILE *fh; /* file written into during downloading */ - }; - - extern bool download_only; -@@ -200,7 +199,7 @@ extern void swupd_curl_cleanup(void); - extern void swupd_curl_set_current_version(int v); - extern void swupd_curl_set_requested_version(int v); - extern double swupd_query_url_content_size(char *url); --extern size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata); -+extern CURLcode swupd_download_file_start(struct file *file); - extern CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file); - extern int swupd_curl_get_file(const char *url, char *filename, struct file *file, - struct version_container *tmp_version, bool pack); -diff --git a/src/curl.c b/src/curl.c -index cab1ef2..009fdd5 100644 ---- a/src/curl.c -+++ b/src/curl.c -@@ -158,62 +158,28 @@ static size_t swupd_download_version_to_memory(void *ptr, size_t size, size_t nm - return data_len; - } - --/* curl easy CURLOPT_WRITEFUNCTION callback */ --size_t swupd_download_file(void *ptr, size_t size, size_t nmemb, void *userdata) -+CURLcode swupd_download_file_start(struct file *file) - { -- struct file *file = (struct file *)userdata; -- const char *outfile; -- int fd; -- FILE *f; -- size_t written, remaining; -- -- outfile = file->staging; -- if (file->fd_valid) { -- fd = file->fd; -- } else { -- fd = open(outfile, O_CREAT | O_RDWR | O_CLOEXEC | O_APPEND, 00600); -- if (fd < 0) { -- fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", -- outfile, strerror(errno)); -- return -1; -- } -- file->fd = fd; -- file->fd_valid = 1; -- } -- -- /* handle short writes with repeated write() calls */ -- for (remaining = size * nmemb; remaining; remaining -= written) { -- written = write(fd, ptr, size*nmemb); -- if (written < 0) { -- if (errno == EINTR) { -- written = 0; -- continue; -- } -- fprintf(stderr, "write error \\*outfile=\"%s\",strerror=\"%s\"*\\\n", -- outfile, strerror(errno)); -- return -1; -- } -+ file->fh = fopen(file->staging, "w"); -+ if (!file->fh) { -+ fprintf(stderr, "Cannot open file for write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", -+ file->staging, strerror(errno)); -+ return CURLE_WRITE_ERROR; - } -- -- if (fdatasync(fd)) { -- fprintf(stderr, "fdatasync \\*outfile=\"%s\",strerror=\"%s\"*\\\n", outfile, strerror(errno)); -- return -1; -- } -- -- return size*nmemb; -+ return CURLE_OK; - } - - CURLcode swupd_download_file_complete(CURLcode curl_ret, struct file *file) - { -- if (file->fd_valid) { -- if (close(file->fd)) { -+ if (file->fh) { -+ if (fclose(file->fh)) { - fprintf(stderr, "Cannot close file after write \\*outfile=\"%s\",strerror=\"%s\"*\\\n", - file->staging, strerror(errno)); - if (curl_ret == CURLE_OK) { - curl_ret = CURLE_WRITE_ERROR; - } - } -- file->fd_valid = 0; -+ file->fh = NULL; - } - return curl_ret; - } -@@ -246,6 +212,7 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file, - - if (file) { - local = file; -+ local->fh = NULL; - } else { - local = calloc(1, sizeof(struct file)); - if (!local) { -@@ -266,11 +233,11 @@ int swupd_curl_get_file(const char *url, char *filename, struct file *file, - if (curl_ret != CURLE_OK) { - goto exit; - } -- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, swupd_download_file); -+ curl_ret = swupd_download_file_start(local); - if (curl_ret != CURLE_OK) { - goto exit; - } -- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)local); -+ curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)local->fh); - if (curl_ret != CURLE_OK) { - goto exit; - } -diff --git a/src/download.c b/src/download.c -index c4a7a07..397d56c 100644 ---- a/src/download.c -+++ b/src/download.c -@@ -475,6 +475,7 @@ void full_download(struct file *file) - CURLMcode curlm_ret = CURLM_OK; - CURLcode curl_ret = CURLE_OK; - -+ file->fh = NULL; - ret = swupd_curl_hashmap_insert(file); - if (ret > 0) { /* no download needed */ - /* File already exists - report success */ -@@ -510,11 +511,11 @@ void full_download(struct file *file) - if (curl_ret != CURLE_OK) { - goto out_bad; - } -- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, swupd_download_file); -+ curl_ret = swupd_download_file_start(file); - if (curl_ret != CURLE_OK) { - goto out_bad; - } -- curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file); -+ curl_ret = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)file->fh); - if (curl_ret != CURLE_OK) { - goto out_bad; - } -@@ -536,6 +537,7 @@ void full_download(struct file *file) - goto out_good; - - out_bad: -+ (void) swupd_download_file_complete(CURLE_OK, file); - failed = list_prepend_data(failed, file); - if (curl != NULL) { - /* Must remove handle out of multi queue first!*/ --- -2.1.4 - diff --git a/recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch b/recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch deleted file mode 100644 index 4326a58..0000000 --- a/recipes-core/swupd-client/swupd-client/Make-pinned-pubkey-configurable.patch +++ /dev/null @@ -1,118 +0,0 @@ -From 6792cfef0ebfbe83e41bc81df6bc675604d7c943 Mon Sep 17 00:00:00 2001 -From: Dmitry Rozhkov <dmitry.rozhkov@linux.intel.com> -Date: Tue, 16 Aug 2016 10:55:15 +0300 -Subject: [PATCH] Make pinned pubkey configurable - -The server may move to a new location where a different pubkey -needs to be used and the hardcoded one won't work. - -This patch makes pinned pubkey configurable. - -Upstream-Status: Submitted [https://github.com/clearlinux/swupd-client/pull/110] - -Signed-off-by: Dmitry Rozhkov <dmitry.rozhkov@linux.intel.com> ---- - configure.ac | 2 ++ - include/swupd.h | 1 + - src/curl.c | 2 +- - src/globals.c | 14 ++++++++++++++ - 4 files changed, 18 insertions(+), 1 deletion(-) - -diff --git a/configure.ac b/configure.ac -index 83007aa..883553a 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -114,6 +114,7 @@ AH_TEMPLATE([LOCK_DIR],[Directory for lock file]) - AH_TEMPLATE([BUNDLES_DIR],[Directory to use for bundles]) - AH_TEMPLATE([UPDATE_CA_CERTS_PATH],[Location of CA certificates]) - AH_TEMPLATE([MOTD_FILE],[motd file path]) -+AH_TEMPLATE([PINNED_PUBKEY_PATH],[Path to pinned public key]) - - if test "$enable_linux_rootfs_build" = "yes"; then - AC_DEFINE([SWUPD_LINUX_ROOTFS],1) -@@ -124,6 +125,7 @@ if test "$enable_linux_rootfs_build" = "yes"; then - AC_DEFINE([BUNDLES_DIR],["/usr/share/clear/bundles"]) - AC_DEFINE_UNQUOTED([UPDATE_CA_CERTS_PATH],["$certs_path"]) - AC_DEFINE([MOTD_FILE],["/usr/lib/motd.d/001-new-release"]) -+ AC_DEFINE_UNQUOTED([PINNED_PUBKEY_PATH],["${certs_path}/425b0f6b.key"]) - else - AC_MSG_ERROR([Unknown build variant]) - fi -diff --git a/include/swupd.h b/include/swupd.h -index 5c722d3..3658dc8 100644 ---- a/include/swupd.h -+++ b/include/swupd.h -@@ -132,6 +132,7 @@ extern void *tm_dlhandle; - extern char *bundle_to_add; - extern struct timeval start_time; - extern char *state_dir; -+extern char *pinned_pubkey_path; - - extern char *version_url; - extern char *content_url; -diff --git a/src/curl.c b/src/curl.c -index 6b6099f..b14193b 100644 ---- a/src/curl.c -+++ b/src/curl.c -@@ -447,7 +447,7 @@ static CURLcode swupd_curl_set_security_opts(CURL *curl) - goto exit; - } - -- curl_ret = curl_easy_setopt(curl, CURLOPT_PINNEDPUBLICKEY, "/usr/share/clear/update-ca/425b0f6b.key"); -+ curl_ret = curl_easy_setopt(curl, CURLOPT_PINNEDPUBLICKEY, pinned_pubkey_path); - if (curl_ret != CURLE_OK) { - goto exit; - } -diff --git a/src/globals.c b/src/globals.c -index f2f1200..d0858df 100644 ---- a/src/globals.c -+++ b/src/globals.c -@@ -51,6 +51,7 @@ char *mounted_dirs = NULL; - char *bundle_to_add = NULL; - struct timeval start_time; - char *state_dir = NULL; -+char *pinned_pubkey_path = NULL; - - /* NOTE: Today the content and version server urls are the same in - * all cases. It is highly likely these will eventually differ, eg: -@@ -68,6 +69,7 @@ long update_server_port = -1; - static const char *default_version_url_path = "/usr/share/defaults/swupd/versionurl"; - static const char *default_content_url_path = "/usr/share/defaults/swupd/contenturl"; - static const char *default_format_path = "/usr/share/defaults/swupd/format"; -+static const char *default_pinnedpubkey_path = "/usr/share/defaults/swupd/pinnedpubkey"; - - static int set_default_value(char **global, const char *path) - { -@@ -194,6 +196,16 @@ bool set_state_dir(char *path) - return true; - } - -+void set_pinned_pubkey_path() -+{ -+ int ret; -+ -+ ret = set_default_value(&pinned_pubkey_path, default_pinnedpubkey_path); -+ if (ret < 0) { -+ string_or_die(&pinned_pubkey_path, "%s", PINNED_PUBKEY_PATH); -+ } -+} -+ - bool set_format_string(char *userinput) - { - int ret; -@@ -322,6 +334,7 @@ bool init_globals(void) - (void)set_format_string(NULL); - set_version_url(NULL); - set_content_url(NULL); -+ set_pinned_pubkey_path(); - - /* must set this global after version_url and content_url */ - set_local_download(); -@@ -337,6 +350,7 @@ void free_globals(void) - free(format_string); - free(mounted_dirs); - free(state_dir); -+ free(pinned_pubkey_path); - if (bundle_to_add != NULL) { - free(bundle_to_add); - } diff --git a/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch b/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch index 7410b1d..b3dd47b 100644 --- a/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch +++ b/recipes-core/swupd-client/swupd-client/ignore-xattrs-when-verifying-Manifest-files.patch @@ -1,7 +1,7 @@ -From c16e1e7fc16933669ed4be63858edd4082509183 Mon Sep 17 00:00:00 2001 +From cc44bbfb2eaa90284a67ad6d42706e6433abd7ff Mon Sep 17 00:00:00 2001 From: Patrick Ohly <patrick.ohly@intel.com> Date: Thu, 3 Nov 2016 11:47:53 +0100 -Subject: [PATCH] ignore xattrs when verifying Manifest files +Subject: [PATCH 1/3] verify_file: ignore xattrs when verifying Manifest files When IMA or Smack are active on the client, the downloaded Manifest files will be assigned certain xattrs (security.ima @@ -12,170 +12,33 @@ Manifest hashes even if they existed (see write_manifest_plain() in src/manifest.c). Therefore the client must ignore xattrs when verifying Manifest files. -This is the only place where verification gets relaxed. All other locations -still use xattrs, just as before. + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-client/commit/09c26658d346cdd80ea54188d991db3493983176] Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> --- - include/swupd.h | 2 +- - src/delta.c | 2 +- - src/download.c | 6 +++--- - src/hash.c | 6 +++--- - src/helpers.c | 2 +- - src/manifest.c | 2 +- - src/scripts.c | 2 +- - src/verify.c | 4 ++-- - 8 files changed, 13 insertions(+), 13 deletions(-) + src/hash.c | 9 ++++++++- -diff --git a/include/swupd.h b/include/swupd.h -index e1e1f3d..14e65ab 100644 ---- a/include/swupd.h -+++ b/include/swupd.h -@@ -225,7 +225,7 @@ extern struct list *recurse_manifest(struct manifest *manifest, const char *comp - extern struct list *consolidate_files(struct list *files); - extern void debug_write_manifest(struct manifest *manifest, char *filename); - extern void populate_file_struct(struct file *file, char *filename); --extern bool verify_file(struct file *file, char *filename); -+extern bool verify_file(struct file *file, char *filename, bool use_xattrs); - extern int verify_bundle_hash(struct manifest *manifest, struct file *bundle); - extern void unlink_all_staged_content(struct file *file); - extern void link_renames(struct list *newfiles, struct manifest *from_manifest); -diff --git a/src/delta.c b/src/delta.c -index 8172b67..317adde 100644 ---- a/src/delta.c -+++ b/src/delta.c -@@ -109,7 +109,7 @@ static void do_delta(struct file *file) - } - xattrs_copy(origin, filename); - -- if (!verify_file(file, filename)) { -+ if (!verify_file(file, filename, true)) { - unlink_all_staged_content(file); - goto out; - } -diff --git a/src/download.c b/src/download.c -index 9ea957d..6d81d81 100644 ---- a/src/download.c -+++ b/src/download.c -@@ -98,7 +98,7 @@ static int swupd_curl_hashmap_insert(struct file *file) - string_or_die(&targetfile, "%s/staged/%s", state_dir, file->hash); - - if (lstat(targetfile, &stat) == 0) { -- if (verify_file(file, targetfile)) { -+ if (verify_file(file, targetfile, true)) { - free(targetfile); - pthread_mutex_unlock(&bucket->mutex); - return 1; -@@ -260,7 +260,7 @@ int untar_full_download(void *data) - * NOTE: this should NEVER happen given the checking that happens - * ahead of queueing a download. But... */ - if (lstat(targetfile, &stat) == 0) { -- if (verify_file(file, targetfile)) { -+ if (verify_file(file, targetfile, true)) { - unlink(tar_dotfile); - unlink(tarfile); - free(tar_dotfile); -@@ -316,7 +316,7 @@ int untar_full_download(void *data) - } - - err = lstat(targetfile, &stat); -- if (!err && !verify_file(file, targetfile)) { -+ if (!err && !verify_file(file, targetfile, true)) { - /* Download was successful but the hash was bad. This is fatal*/ - printf("Error: File content hash mismatch for %s (bad server data?)\n", targetfile); - exit(EXIT_FAILURE); diff --git a/src/hash.c b/src/hash.c -index 34da6eb..00a6802 100644 +index 1e61454..9553644 100644 --- a/src/hash.c +++ b/src/hash.c -@@ -226,7 +226,7 @@ int compute_hash(struct file *file, char *filename) - return 0; - } - --bool verify_file(struct file *file, char *filename) -+bool verify_file(struct file *file, char *filename, bool use_xattrs) - { - struct file *local = calloc(1, sizeof(struct file)); - -@@ -235,7 +235,7 @@ bool verify_file(struct file *file, char *filename) +@@ -236,7 +236,14 @@ bool verify_file(struct file *file, char *filename) } local->filename = file->filename; - local->use_xattrs = true; -+ local->use_xattrs = use_xattrs; ++ /* ++ * xattrs are currently not supported for manifest files. ++ * They are data files produced by the swupd-server and ++ * therefore do not have any of the xattrs normally ++ * set for the actual system files (like security.ima ++ * when using IMA or security.SMACK64 when using Smack). ++ */ ++ local->use_xattrs = !file->is_manifest; populate_file_struct(local, filename); if (compute_hash(local, filename) != 0) { -@@ -275,7 +275,7 @@ int verify_bundle_hash(struct manifest *manifest, struct file *bundle) - string_or_die(&local, "%s/%i/Manifest.%s", state_dir, - current->last_change, current->filename); - -- if (!verify_file(bundle, local)) { -+ if (!verify_file(bundle, local, false)) { - printf("Warning: hash check failed for Manifest.%s\n", - current->filename); - ret = 0; -diff --git a/src/helpers.c b/src/helpers.c -index e71688c..01fd4a3 100644 ---- a/src/helpers.c -+++ b/src/helpers.c -@@ -787,7 +787,7 @@ int verify_fix_path(char *targetpath, struct manifest *target_MoM) - - ret = stat(target, &sb); - if (ret == 0) { -- if (verify_file(file, target)) { -+ if (verify_file(file, target, true)) { - continue; - } - printf("Hash did not match for path : %s\n", path); -diff --git a/src/manifest.c b/src/manifest.c -index 2b57d3d..ee6d29a 100644 ---- a/src/manifest.c -+++ b/src/manifest.c -@@ -674,7 +674,7 @@ struct list *create_update_list(struct manifest *current, struct manifest *serve - if (fullname == NULL) { - abort(); - } -- if (verify_file(file, fullname)) { -+ if (verify_file(file, fullname, true)) { - free(fullname); - continue; - } -diff --git a/src/scripts.c b/src/scripts.c -index 59417af..c2157f7 100644 ---- a/src/scripts.c -+++ b/src/scripts.c -@@ -127,7 +127,7 @@ void run_preupdate_scripts(struct manifest *manifest) - } - - /* Check that system file matches file in manifest */ -- if (verify_file(file, script)) { -+ if (verify_file(file, script, true)) { - system(script); - break; - } -diff --git a/src/verify.c b/src/verify.c -index 1514988..eaf9dd8 100644 ---- a/src/verify.c -+++ b/src/verify.c -@@ -462,7 +462,7 @@ static void deal_with_hash_mismatches(struct manifest *official_manifest, bool r - if (fullname == NULL) { - abort(); - } -- if (verify_file(file, fullname)) { -+ if (verify_file(file, fullname, true)) { - free(fullname); - continue; - } else { -@@ -483,7 +483,7 @@ static void deal_with_hash_mismatches(struct manifest *official_manifest, bool r - } - - /* at the end of all this, verify the hash again to judge success */ -- if (verify_file(file, fullname)) { -+ if (verify_file(file, fullname, true)) { - file_fixed_count++; - printf("\tfixed\n"); - } else { -- 2.1.4 diff --git a/recipes-core/swupd-client/swupd-client_git.bb b/recipes-core/swupd-client/swupd-client_git.bb index fd73223..d603cb0 100644 --- a/recipes-core/swupd-client/swupd-client_git.bb +++ b/recipes-core/swupd-client/swupd-client_git.bb @@ -5,17 +5,14 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=f8d90fb802930e30e49c39c8126a959e" DEPENDS = "glib-2.0 curl openssl libarchive bsdiff" -PV = "3.6.0+git${SRCPV}" -SRC_URI = "\ - git://github.com/clearlinux/swupd-client.git;protocol=https \ - file://Change-systemctl-path-to-OE-systemctl-path.patch \ - file://0001-Add-configure-option-to-re-enable-updating-of-config.patch \ - file://Make-pinned-pubkey-configurable.patch \ - file://ignore-xattrs-when-verifying-Manifest-files.patch \ - file://0001-downloads-minimize-syscalls-to-improve-performance.patch \ - file://0002-downloads-open-FILE-in-advance-and-use-default-write.patch \ -" -SRCREV = "f4000c5b22be47ec1af2f8748fd71a36148b5dc4" +PV = "3.7.2+git${SRCPV}" +SRC_URI = "git://github.com/clearlinux/swupd-client.git;protocol=https \ + file://Change-systemctl-path-to-OE-systemctl-path.patch \ + file://0001-Add-configure-option-to-re-enable-updating-of-config.patch \ + file://ignore-xattrs-when-verifying-Manifest-files.patch \ + file://0001-fix-enable-xattr.patch \ + " +SRCREV = "b43ad9748ea690f69f924b6cae9a83c3886514b6" S = "${WORKDIR}/git" @@ -30,18 +27,18 @@ RRECOMMENDS_${PN}_class-target = "os-release" # and bump the number by one for each update of the recipe where we # switch to a source that has a format change. # -# To switch to a client with a new format also update SWUPD_FORMAT in +# To switch to a client with a new format also update SWUPD_TOOLS_FORMAT in # swupd-image.bbclass. -RPROVIDES_${PN} = "swupd-client-format3" +SWUPD_CLIENT_FORMAT = "4" +RPROVIDES_${PN} = "swupd-client-format${SWUPD_CLIENT_FORMAT}" -# TODO: we inherit autotools-brokensep because the Makefile calls a perl script -# in ${S} during one of its steps. -inherit pkgconfig autotools-brokensep systemd +inherit pkgconfig autotools systemd EXTRA_OECONF = "\ --with-systemdsystemunitdir=${systemd_system_unitdir} \ --enable-bsdtar \ --disable-tests \ + --enable-xattr \ " PACKAGECONFIG ??= "stateless" @@ -53,7 +50,7 @@ FILES_${PN} += "\ /var/lib/swupd \ " -SYSTEMD_SERVICE_${PN} = "check-update.timer check-update.service" +SYSTEMD_SERVICE_${PN} = "check-update.timer check-update.service swupd-update.timer swupd-update.service" SYSTEMD_AUTO_ENABLE_${PN} = "disable" BBCLASSEXTEND = "native" diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch new file mode 100644 index 0000000..fd91b78 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch @@ -0,0 +1,257 @@ +From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 16 Nov 2016 14:26:30 +0100 +Subject: [PATCH 1/3] create_pack: rely less on previous builds + +When a file has not been modified in the current build, then by +definition the current copy of the file is the same as in the build +were it was last changed and thus it does not matter whether we use +<current build>/full/<file> or <last change>/full/<file>. But using +the current copy is better for a CI system which starts without local +access to older rootfs directories. It might also be a bit more +efficient (file access less scattered between different "full" +directories). + +Staging directories is better than staging .tar archives containing +those directories for the same reason (the .tar archive might not be +available in the CI system) and probably also improves efficiency (no +need to invoke bsdtar just to create a directory; impact not +measured). + +Also fix a slight flaw in the "target file exists already" handling: +when that occured for whatever reason (likely only during manual +debugging), the code would add the original fullfile .tar although it +isn't needed. Clearing the "ret" variable in that particular error +case avoids that. + +make_pack_full_files() and make_final_pack() used the exact same code +for populating the "staged" directory. Now that common code is in +stage_entry(). + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 2 +- + src/delta.c | 4 +-- + src/pack.c | 105 +++++++++++++++++++++++++++++++------------------------- + 3 files changed, 62 insertions(+), 49 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index c1c0e96..cf384e3 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest); + + extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); + extern void link_renames(GList *newfiles, struct manifest *from_manifest); +-extern void __create_delta(struct file *file, int from_version); ++extern void __create_delta(struct file *file, int from_version, int to_version); + + extern void account_delta_hit(void); + extern void account_delta_miss(void); +diff --git a/src/delta.c b/src/delta.c +index 7e978b1..8fff4c9 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -35,7 +35,7 @@ + #include "swupd.h" + #include "xattrs.h" + +-void __create_delta(struct file *file, int from_version) ++void __create_delta(struct file *file, int from_version, int to_version) + { + char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; + char *tmpdir = NULL; +@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version) + } + + conf = config_image_base(); +- string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename); ++ string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename); + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + +diff --git a/src/pack.c b/src/pack.c +index 984c2d6..ccb28bd 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -37,6 +37,7 @@ + #include <unistd.h> + + #include "swupd.h" ++#include "xattrs.h" + + static void empty_pack_stage(int full, int from_version, int to_version, char *module) + { +@@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack) + link_renames(pack->end_manifest->files, manifest); + } + ++static int stage_entry(struct file *file, ++ const char *fullfrom, const char *fullto, ++ const char *tarfrom, const char *tarto, ++ const char *packname) ++{ ++ int ret; ++ ++ /* Prefer to hardlink uncompressed files or replicate ++ * directories first, and fall back to the compressed ++ * versions if that failed. ++ */ ++ if (!file->is_dir) { ++ ret = link(fullfrom, fullto); ++ if (ret && errno == EEXIST) { ++ ret = 0; ++ } else if (ret) { ++ LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); ++ } ++ } else { ++ /* Replicate directory. */ ++ struct stat st; ++ if ((stat(fullfrom, &st) || ++ mkdir(fullto, 0) || ++ chmod(fullto, st.st_mode) || ++ chown(fullto, st.st_uid, st.st_gid) || ++ (xattrs_copy(fullfrom, fullto), false)) && ++ errno != EEXIST) { ++ LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); ++ rmdir(fullto); ++ ret = -1; ++ } else { ++ ret = 0; ++ } ++ } ++ ++ if (ret) { ++ ret = link(tarfrom, tarto); ++ if (ret && errno != EEXIST) { ++ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); ++ } ++ } ++ ++ return ret; ++} ++ + static void make_pack_full_files(struct packdata *pack) + { + GList *item; +@@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack) + char *fullfrom, *fullto; + + /* hardlink each file that is in <end> but not in <X> */ +- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); ++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); + string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash); + string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + +- ret = -1; +- errno = 0; +- +- /* Prefer to hardlink uncompressed files (excluding +- * directories) first, and fall back to the compressed +- * versions if the hardlink fails. ++ /* Prefer to hardlink uncompressed files or replicate ++ * directories first, and fall back to the compressed ++ * versions if that failed. + */ +- if (!file->is_dir) { +- ret = link(fullfrom, fullto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno); +- } +- } +- if (ret) { +- ret = link(from, to); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno); +- } +- } ++ ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile"); + + if (ret == 0) { + pack->fullcount++; +@@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack) + return files; + } + +-static void create_delta(gpointer data, __unused__ gpointer user_data) ++static void create_delta(gpointer data, gpointer user_data) + { + struct file *file = data; ++ int *to_version = user_data; + + /* if the file was not found in the from version, skip delta creation */ + if (file->peer) { +- __create_delta(file, file->peer->last_change); ++ __create_delta(file, file->peer->last_change, *to_version); + } + } + +-static void make_pack_deltas(GList *files) ++static void make_pack_deltas(GList *files, int to_version) + { + GThreadPool *threadpool; + GList *item; +@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files) + sysconf(_SC_NPROCESSORS_ONLN); + + LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); +- threadpool = g_thread_pool_new(create_delta, NULL, ++ threadpool = g_thread_pool_new(create_delta, &to_version, + numthreads, FALSE, NULL); + + item = g_list_first(files); +@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack) + file->last_change, file->hash); + string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, + pack->module, pack->from, pack->to, file->hash); +- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); ++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); + string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + +@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack) + } + } + } else { +- ret = -1; +- errno = 0; +- +- /* Prefer to hardlink uncompressed files (excluding +- * directories) first, and fall back to the compressed +- * versions if the hardlink fails. +- */ +- if (!file->is_dir) { +- ret = link(fullfrom, fullto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno); +- } +- } +- +- if (ret) { +- ret = link(tarfrom, tarto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno); +- } +- } +- ++ ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final"); + if (ret == 0) { + pack->fullcount++; + } +@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack) + + /* step 2: consolidate delta list & create all delta files*/ + delta_list = consolidate_packs_delta_files(delta_list, pack); +- make_pack_deltas(delta_list); ++ make_pack_deltas(delta_list, pack->to); + g_list_free(delta_list); + + /* step 3: complete pack creation */ +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server/0001-delta.c-fix-xattr-test-after-patching.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch index 78fb85a..78fb85a 100644 --- a/recipes-core/swupd-server/swupd-server/0001-delta.c-fix-xattr-test-after-patching.patch +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch new file mode 100644 index 0000000..5920afc --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch @@ -0,0 +1,349 @@ +From e1f0d54a940eb7d04e2fbd59bd995a819331425f Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Fri, 30 Sep 2016 08:42:08 +0200 +Subject: [PATCH 1/2] swupd-create-update: alternative input layout + +In Ostro OS, we already have a "full" directory with all files. +Splitting it up into bundles just so that swupd-create-update can +reconstruct the "full" directory is a waste of IO, and noticably slow +when run under pseudo. + +To streamline the required work, a new layout for the "image" input +directory gets introduced: +- The "full" directory gets created by the caller before invoking + swupd-create-update. +- For each bundle, instead of a <bundle> directory, there is a + <bundle>.content.txt file, listing all entries (including directories) + of the bundle. + +The traditional mode of operation still works as before because each operation +which normally works with a bundle directory checks whether there is such a +directory and if not, switches to the new mode. + +That way it is even possible to mix the two modes, i.e. replacing only +some bundles with a content list, although that's probably not all +that useful. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/55] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/analyze_fs.c | 171 ++++++++++++++++++++++++++++++++++++++-------------- + src/chroot.c | 24 +++++--- + src/create_update.c | 6 +- + src/fullfiles.c | 2 + + 4 files changed, 150 insertions(+), 53 deletions(-) + +diff --git a/src/analyze_fs.c b/src/analyze_fs.c +index 0f16343..d534587 100644 +--- a/src/analyze_fs.c ++++ b/src/analyze_fs.c +@@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data) + /* disallow characters which can do unexpected things when the filename is + * used on a tar command line via system("tar [args] filename [more args]"); + */ +-static bool illegal_characters(char *filename) ++static bool illegal_characters(const char *filename) + { + char c; + int i; +@@ -301,25 +301,145 @@ static bool illegal_characters(char *filename) + return false; + } + ++static void add_file(struct manifest *manifest, ++ const char *entry_name, ++ char *sub_filename, ++ char *fullname, ++ bool do_hash) ++{ ++ GError *err = NULL; ++ struct file *file; ++ ++ if (illegal_characters(entry_name)) { ++ printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename); ++ free(sub_filename); ++ free(fullname); ++ return; ++ } ++ ++ file = calloc(1, sizeof(struct file)); ++ assert(file); ++ ++ file->last_change = manifest->version; ++ file->filename = sub_filename; ++ ++ populate_file_struct(file, fullname); ++ if (file->is_deleted) { ++ /* ++ * populate_file_struct() logs a stat() failure, but ++ * does not abort. When adding files that should ++ * exist, this case is an error. ++ */ ++ LOG(NULL, "file not found", "%s", fullname); ++ assert(0); ++ } ++ ++ ++ /* if for some reason there is a file in the official build ++ * which should not be included in the Manifest, then open a bug ++ * to get it removed, and work around its presence by ++ * excluding it here, eg: ++ if (strncmp(file->filename, "/dev/", 5) == 0) { ++ continue; ++ } ++ */ ++ ++ if (do_hash) { ++ /* compute the hash from a thread */ ++ int ret; ++ ret = g_thread_pool_push(threadpool, file, &err); ++ if (ret == FALSE) { ++ printf("GThread hash computation push error\n"); ++ printf("%s\n", err->message); ++ assert(0); ++ } ++ } ++ manifest->files = g_list_prepend(manifest->files, file); ++ manifest->count++; ++} ++ ++ + static void iterate_directory(struct manifest *manifest, char *pathprefix, + char *subpath, bool do_hash) + { + DIR *dir; + struct dirent *entry; + char *fullpath; +- int ret; +- GError *err = NULL; + + string_or_die(&fullpath, "%s/%s", pathprefix, subpath); + + dir = opendir(fullpath); + if (!dir) { ++ FILE *content; ++ int len; ++ free(fullpath); ++ if (errno != ENOENT) { ++ return; ++ } ++ /* ++ * If there is a <dir>.content.txt instead of ++ * the actual directory, then read that ++ * file. It has a list of path names, ++ * including all directories. The ++ * corresponding file system entry is then ++ * expected to be in a pre-populated "full" ++ * directory. ++ */ ++ if (subpath[0]) { ++ string_or_die(&fullpath, "%s/%s.content.txt", pathprefix, len, subpath); ++ } else { ++ string_or_die(&fullpath, "%s.content.txt", pathprefix); ++ } ++ content = fopen(fullpath, "r"); + free(fullpath); ++ fullpath = NULL; ++ if (content) { ++ char *line = NULL; ++ size_t len = 0; ++ ssize_t read; ++ const char *full; ++ int full_len; ++ /* ++ * determine path to "full" directory: it is assumed to be alongside ++ * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit, ++ * so we have to strip the last path component. ++ */ ++ full = strrchr(pathprefix, '/'); ++ if (full) { ++ full_len = full - pathprefix + 1; ++ full = pathprefix; ++ } else { ++ full = ""; ++ full_len = 0; ++ } ++ while ((read = getline(&line, &len, content)) != -1) { ++ if (read) { ++ const char *entry_name = strrchr(line, '/'); ++ if (entry_name) { ++ entry_name++; ++ } else { ++ entry_name = line; ++ } ++ if (line[read - 1] == '\n') { ++ line[read - 1] = 0; ++ } ++ string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line); ++ add_file(manifest, ++ entry_name, ++ strdup(line), ++ fullpath, ++ do_hash); ++ } ++ } ++ free(line); ++ } ++ ++ // If both directory and content file are missing, silently (?) ++ // don't add anything to the manifest. + return; + } + + while (dir) { +- struct file *file; + char *sub_filename; + char *fullname; + +@@ -334,50 +454,13 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix, + } + + string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name); +- +- if (illegal_characters(entry->d_name)) { +- printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename); +- free(sub_filename); +- continue; +- } +- +- file = calloc(1, sizeof(struct file)); +- if (!file) { +- break; +- } +- +- file->last_change = manifest->version; +- file->filename = sub_filename; +- + string_or_die(&fullname, "%s/%s", fullpath, entry->d_name); +- populate_file_struct(file, fullname); +- free(fullname); +- + if (entry->d_type == DT_DIR) { +- iterate_directory(manifest, pathprefix, file->filename, do_hash); ++ iterate_directory(manifest, pathprefix, sub_filename, do_hash); + } ++ /* takes ownership of the strings */ ++ add_file(manifest, entry->d_name, sub_filename, fullname, do_hash); + +- /* if for some reason there is a file in the official build +- * which should not be included in the Manifest, then open a bug +- * to get it removed, and work around its presence by +- * excluding it here, eg: +- if (strncmp(file->filename, "/dev/", 5) == 0) { +- continue; +- } +- */ +- +- if (do_hash) { +- /* compute the hash from a thread */ +- ret = g_thread_pool_push(threadpool, file, &err); +- if (ret == FALSE) { +- printf("GThread hash computation push error\n"); +- printf("%s\n", err->message); +- closedir(dir); +- return; +- } +- } +- manifest->files = g_list_prepend(manifest->files, file); +- manifest->count++; + } + closedir(dir); + free(fullpath); +diff --git a/src/chroot.c b/src/chroot.c +index 32ed997..f3832e1 100644 +--- a/src/chroot.c ++++ b/src/chroot.c +@@ -39,15 +39,21 @@ void chroot_create_full(int newversion) + char *full_dir; + + string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion); ++ if (!access(full_dir, R_OK|X_OK)) { ++ free(full_dir); ++ return; ++ } + + g_mkdir_with_parents(full_dir, S_IRWXU); + + /* start with base */ +- LOG(NULL, "Copying chroot os-core to full", ""); + string_or_die(¶m, "%s/%i/os-core/", image_dir, newversion); +- char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL }; +- if (system_argv(rsynccmd) != 0) { +- assert(0); ++ if (!access(param, F_OK)) { ++ LOG(NULL, "Copying chroot os-core to full", ""); ++ char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL }; ++ if (system_argv(rsynccmd) != 0) { ++ assert(0); ++ } + } + free(param); + +@@ -58,11 +64,13 @@ void chroot_create_full(int newversion) + break; + } + +- LOG(NULL, "Overlaying bundle chroot onto full", "%s", group); + string_or_die(¶m, "%s/%i/%s/", image_dir, newversion, group); +- char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL }; +- if (system_argv(rsynccmd) != 0) { +- assert(0); ++ if (!access(param, F_OK)) { ++ LOG(NULL, "Overlaying bundle chroot onto full", "%s", group); ++ char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL }; ++ if (system_argv(rsynccmd) != 0) { ++ assert(0); ++ } + } + free(param); + } +diff --git a/src/create_update.c b/src/create_update.c +index 766609b..74d5376 100644 +--- a/src/create_update.c ++++ b/src/create_update.c +@@ -141,6 +141,7 @@ static bool parse_options(int argc, char **argv) + static void populate_dirs(int version) + { + char *newversiondir; ++ char *newversiondircontent = NULL; + + string_or_die(&newversiondir, "%s/%d", image_dir, version); + +@@ -182,9 +183,11 @@ static void populate_dirs(int version) + } + + string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group); ++ string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group); + + /* Create the bundle directory(s) as needed */ +- if (access(newversiondir, F_OK | R_OK) != 0) { ++ if (access(newversiondir, F_OK | R_OK) != 0 && ++ access(newversiondircontent, F_OK | R_OK) != 0) { + printf("%s does not exist...creating\n", group); + if (mkdir(newversiondir, 0755) != 0) { + printf("Failed to create %s subdirectory\n", group); +@@ -193,6 +196,7 @@ static void populate_dirs(int version) + } + } + free(newversiondir); ++ free(newversiondircontent); + } + + static int check_build_env(void) +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 9fdfc2f..214b7b4 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -136,6 +136,8 @@ static void create_fullfile(struct file *file) + } + done += curr; + } ++ close(fd); ++ fd = -1; + } + + for (int i = 0; compression_filters[i]; i++) { +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch new file mode 100644 index 0000000..56ea17d --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch @@ -0,0 +1,200 @@ +From 2e3eb8abcdef496d0ce30d03da7befcb9978aceb Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Sat, 1 Oct 2016 13:51:02 +0200 +Subject: [PATCH 2/2] add logging to stdout + +When a CI system (like the one from Ostro) captures the output of +commands, but not necessarily intermediate log files, then it is +useful to also log to stdout. Another use case is calling the tools +interactively during development. + +The new --log-stdout option in all three commands enables logging to +stdout in addition to the traditional log files. + +The implementation recycles the existing init_log_stdout() (not used +before) and gives it the slightly different meaning of "also log to +stdout". + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/72dd27a886ad9b2c66bb7cdb5c4cadb24f783654] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/create_update.c | 5 +++++ + src/log.c | 27 +++++++++++++++++---------- + src/make_fullfiles.c | 5 +++++ + src/make_packs.c | 5 +++++ + 4 files changed, 32 insertions(+), 10 deletions(-) + +diff --git a/src/create_update.c b/src/create_update.c +index 74d5376..f1c840d 100644 +--- a/src/create_update.c ++++ b/src/create_update.c +@@ -50,6 +50,7 @@ static void banner(void) + static const struct option prog_opts[] = { + { "help", no_argument, 0, 'h' }, + { "version", no_argument, 0, 'v' }, ++ { "log-stdout", no_argument, 0, 'l' }, + { "osversion", required_argument, 0, 'o' }, + { "minversion", required_argument, 0, 'm' }, + { "format", required_argument, 0, 'F' }, +@@ -68,6 +69,7 @@ static void print_help(const char *name) + printf(" -v, --version Show software version\n"); + printf("\n"); + printf("Application Options:\n"); ++ printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -o, --osversion The OS version for which to create an update\n"); + printf(" -m, --minversion Optional minimum file version to write into manifests per file\n"); + printf(" -F, --format Format number for the update\n"); +@@ -87,6 +89,9 @@ static bool parse_options(int argc, char **argv) + case 'h': + print_help(argv[0]); + return false; ++ case 'l': ++ init_log_stdout(); ++ break; + case 'v': + banner(); + return false; +diff --git a/src/log.c b/src/log.c +index e8bf9c3..45a4d66 100644 +--- a/src/log.c ++++ b/src/log.c +@@ -33,7 +33,7 @@ + + #include "swupd.h" + +-static FILE *logfile; ++static FILE *logfile[2]; + + static struct timeval start_time; + +@@ -41,13 +41,13 @@ void init_log(const char *prefix, const char *bundle, int start, int end) + { + char *filename; + string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end); +- logfile = fopen(filename, "w"); ++ logfile[0] = fopen(filename, "w"); + free(filename); + gettimeofday(&start_time, NULL); + } + void init_log_stdout(void) + { +- logfile = stdout; ++ logfile[1] = stdout; + gettimeofday(&start_time, NULL); + } + +@@ -91,8 +91,9 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con + char *logstring = NULL; + char filebuf[4096]; + char filebuf2[4096]; ++ int i; + +- if (!logfile) { ++ if (!logfile[0] && !logfile[1]) { + return; + } + +@@ -119,12 +120,16 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con + strcat(filebuf2, " "); + } + +- fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n", +- (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf); ++ for (i = 0; i < 2; i++) { ++ if (logfile[i]) { ++ fprintf(logfile[i], "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n", ++ (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf); ++ fflush(logfile[i]); ++ } ++ } + + free(logstring); + free(buf); +- fflush(logfile); + } + + void close_log(int version, int exit_status) +@@ -133,7 +138,7 @@ void close_log(int version, int exit_status) + int t_sec; + int t_msec; + +- if (!logfile) { ++ if (!logfile[0] && !logfile[1]) { + return; + } + +@@ -159,6 +164,8 @@ void close_log(int version, int exit_status) + printf("Update build failed for version %i\n", version); + } + +- fclose(logfile); +- logfile = NULL; ++ if (logfile[0]) { ++ fclose(logfile[0]); ++ logfile[0] = NULL; ++ } + } +diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c +index 2a1e2e9..0216e91 100644 +--- a/src/make_fullfiles.c ++++ b/src/make_fullfiles.c +@@ -33,6 +33,7 @@ + + static const struct option prog_opts[] = { + { "help", no_argument, 0, 'h' }, ++ { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { 0, 0, 0, 0 } + }; +@@ -43,6 +44,7 @@ static void usage(const char *name) + printf(" %s <version>\n\n", name); + printf("Help options:\n"); + printf(" -h, --help Show help options\n"); ++ printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf("\n"); + } +@@ -57,6 +59,9 @@ static bool parse_options(int argc, char **argv) + case 'h': + usage(argv[0]); + return false; ++ case 'l': ++ init_log_stdout(); ++ break; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument '%s'\n\n", optarg); +diff --git a/src/make_packs.c b/src/make_packs.c +index 8560b3f..2d3e25e 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -46,6 +46,7 @@ static void banner(void) + + static const struct option prog_opts[] = { + { "help", no_argument, 0, 'h' }, ++ { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { "signcontent", no_argument, 0, 's' }, + { 0, 0, 0, 0 } +@@ -57,6 +58,7 @@ static void usage(const char *name) + printf(" %s <start version> <latest version> <bundle>\n\n", name); + printf("Help options:\n"); + printf(" -h, --help Show help options\n"); ++ printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); + printf("\n"); +@@ -72,6 +74,9 @@ static bool parse_options(int argc, char **argv) + case 'h': + usage(argv[0]); + return false; ++ case 'l': ++ init_log_stdout(); ++ break; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument ''%s'\n\n", optarg); +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch new file mode 100644 index 0000000..440cca8 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch @@ -0,0 +1,42 @@ +From c9e9fb971ab0494047b4d8c0e656e0a06ad9b236 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:39:49 +0100 +Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs + +The fullfile .tar is needed for a pack as fallback when linking from +the full rootfs isn't possible. In practice this shouldn't happen. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/pack.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/src/pack.c b/src/pack.c +index ccb28bd..e331da2 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -187,6 +187,19 @@ static int stage_entry(struct file *file, + + if (ret) { + ret = link(tarfrom, tarto); ++ if (ret && errno == ENOENT && content_url) { ++ LOG(NULL, "Download fallback for pack", "%s: %s to %s", packname, tarfrom, tarto); ++ ++ /* Must be "tarfrom" that is missing. Download directly into target location.*/ ++ char *cmd; ++ string_or_die(&cmd, "curl -s -o '%s' %s/%d/files/%s.tar", ++ tarto, content_url, file->last_change, file->hash); ++ if (system(cmd)) { ++ LOG(file, "Downloading failed", "%s", cmd); ++ } else { ++ ret = 0; ++ } ++ } + if (ret && errno != EEXIST) { + LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); + } +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch new file mode 100644 index 0000000..4cc3af4 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch @@ -0,0 +1,38 @@ +From 7b9e1bc78b704a2fb9610dddc6f0925bb6412f55 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:34:44 +0100 +Subject: [PATCH 2/3] pack.c: do not clean packstage + +This works around a bug in pseudo +(https://bugzilla.yoctoproject.org/show_bug.cgi?id=10623) where +unlinking the linked entries in the staging area also removes the +xattrs of the original files in the full directory tree. + +This works as long as the swupd_create_pack command does not get +called again without cleaning the packstage. + +Upstream-Status: Inappropriate [workaround] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/pack.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/src/pack.c b/src/pack.c +index 46c7c68..984c2d6 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -46,7 +46,9 @@ static void empty_pack_stage(int full, int from_version, int to_version, char *m + // clean any stale data (eg: re-run after a failure) + string_or_die(¶m, "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version); + char *const rmcmd[] = { "rm", "-fr", param, NULL }; +- if (system_argv(rmcmd) != 0) { ++ if (true) { ++ LOG(NULL, "Skipping removal of packstage", "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version); ++ } else if (system_argv(rmcmd) != 0) { + fprintf(stderr, "Failed to clean %s/%s/%i_to_%i\n", + packstage_dir, module, from_version, to_version); + free(param); +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch new file mode 100644 index 0000000..7e981e3 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch @@ -0,0 +1,60 @@ +From 88fd362785ae8871537573fd7073e499542b0a0d Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Thu, 17 Nov 2016 10:09:27 +0100 +Subject: [PATCH 3/3] create_pack: abort delta handling early when impossible + due to IMA + +Currently, deltas cannot be computed for systems using IMA because IMA +relies on a security.ima xattr which changes each time the file +content changes and swupd doesn't support deltas for files with +different xattrs. + +It would be worthwhile to add such a support, but that's a bit +complicated (needs to be done in client and server and implies a +format change), so for now just abort early when a security.xattr is +found. + +This speeds up delta computation considerably in Ostro OS because it +skips the slow downloading of the old file. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/delta.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/src/delta.c b/src/delta.c +index 8fff4c9..d3c4b35 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -30,6 +30,7 @@ + #include <string.h> + #include <sys/stat.h> + #include <sys/types.h> ++#include <attr/xattr.h> + #include <unistd.h> + + #include "swupd.h" +@@ -64,6 +65,18 @@ void __create_delta(struct file *file, int from_version, int to_version) + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + ++ if (lgetxattr(newfile, "security.ima", NULL, 0) > 0) { ++ /* There is a non-empty security.ima xattr on the new file. ++ * That xattr contains a hash of the file content. We know that ++ * the file content has changed, so the xattr will be different ++ * from the one on the old file and we can bail out early without ++ * even bothering with retrieving the original file. A better ++ * solution for systems with IMA would be to support deltas even ++ * when xattrs are different. ++ */ ++ goto out; ++ } ++ + if (access(original, F_OK) && + content_url) { + /* File does not exist. Try to get it from the online update repo instead. +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch new file mode 100644 index 0000000..3a71d9c --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch @@ -0,0 +1,246 @@ +From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:39:49 +0100 +Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for + diffing + +The new, optional mode gets enabled via the --content-url parameter. +When specified, swupd_create_pack will download the <original +version>/files/<hash>.tar file from the update server if the required +file is missing. + +This mode is meant for use with meta-swupd where new CI builds start +with an empty "image" directory. NFS access to the full, expanded +"image" directories of the previous build is not possible in such a +setup (would require root for special file attributes). + +Downloading and unpacking image directories of previous builds would +be possible, but is expected to be slower (when downloading all files, +not just those needed for diffing) or more complex (when deciding about +required files outside of swupd_create_pack). + +The downside of this new approach is that it relies on not changing +the content or naming of the tar files. For example, switching from +GNU tar to bsdtar would not work. But that's not an issue for the +intended usage in Ostro OS, which already made the switch to +bsdtar. + +If for some reason a format change is necessary, delta computation +becomes impossible and falls back to staging the full files. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 2 ++ + src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- + src/globals.c | 13 ++++++++ + src/make_packs.c | 8 +++++ + 4 files changed, 111 insertions(+), 2 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index 31c7191..c1c0e96 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -147,12 +147,14 @@ extern char *state_dir; + extern char *packstage_dir; + extern char *image_dir; + extern char *staging_dir; ++extern char *content_url; + + extern bool init_globals(void); + extern void free_globals(void); + extern bool set_format(char *); + extern void check_root(void); + extern bool set_state_dir(char *); ++extern bool set_content_url(const char *); + extern bool init_state_globals(void); + extern void free_state_globals(void); + +diff --git a/src/delta.c b/src/delta.c +index 67e7df7..7e978b1 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -37,8 +37,14 @@ + + void __create_delta(struct file *file, int from_version) + { +- char *original, *newfile, *outfile, *dotfile, *testnewfile; +- char *conf, *param1, *param2; ++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; ++ char *tmpdir = NULL; ++ char *url = NULL; ++ char *cmd = NULL; ++ char *conf = NULL, *param1 = NULL, *param2 = NULL; ++ bool delete_original = false; ++ struct manifest *manifest = NULL; ++ GError *gerror = NULL; + int ret; + + if (file->is_link) { +@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version) + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + ++ if (access(original, F_OK) && ++ content_url) { ++ /* File does not exist. Try to get it from the online update repo instead. ++ * This fallback is meant to be used for CI builds which start with no local ++ * state and only HTTP(S) access to the published www directory. ++ * Not being able to retrieve the file is not an error and will merely ++ * prevent computing the delta. ++ */ ++ string_or_die(&tmpdir, "%s/make-pack-tmpdir-XXXXXX", state_dir); ++ tmpdir = g_dir_make_tmp("make-pack-XXXXXX", &gerror); ++ if (!tmpdir) { ++ LOG(NULL, "Failed to create temporary directory for untarring original file", "%s", ++ gerror->message); ++ assert(0); ++ } ++ /* Determine hash of original file in the corresponding Manifest. */ ++ manifest = manifest_from_file(from_version, "full"); ++ if (!manifest) { ++ LOG(NULL, "Failed to read full Manifest", "version %d, cannot retrieve original file", ++ from_version); ++ goto out; ++ } ++ const char *last_hash = NULL; ++ GList *list = g_list_first(manifest->files); ++ while (list) { ++ struct file *original_file = list->data; ++ if (!strcmp(file->filename, original_file->filename)) { ++ last_hash = original_file->hash; ++ break; ++ } ++ list = g_list_next(list); ++ } ++ if (!last_hash) { ++ LOG(NULL, "Original file not found", "%s in full manifest for %d - inconsistent update data?!", ++ file->filename, from_version); ++ goto out; ++ } ++ ++ /* We use a temporary copy because we don't want to ++ * tamper with the original "full" folder which ++ * probably does not even exist. Using a temporary file ++ * file implies re-downloading in the future, but that's ++ * consistent with the intended usage in a CI environment ++ * which always starts from scratch. ++ */ ++ free(original); ++ string_or_die(&original, "%s/%s", tmpdir, last_hash); ++ delete_original = true; ++ ++ /* ++ * This is a proof-of-concept. A real implementation should use ++ * a combination of libcurl + libarchive calls to unpack the files. ++ * For current Ostro OS, deltas despite xattr differences would ++ * be needed, otherwise this code here is of little use (all ++ * modified files fail the xattr sameness check, because security.ima ++ * changes when file content changes). ++ */ ++ string_or_die(&url, "%s/%d/files/%s.tar", content_url, from_version, last_hash); ++ LOG(file, "Downloading original file", "%s to %s", url, original); ++ ++ /* bsdtar can detect compression when reading from stdin, GNU tar can't. */ ++ string_or_die(&cmd, "curl -s %s | bsdtar -C %s -xf -", url, tmpdir); ++ if (system(cmd)) { ++ LOG(file, "Downloading/unpacking failed, skipping delta", "%s", url); ++ goto out; ++ } ++ } ++ + free(conf); + + conf = config_output_dir(); +@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version) + LOG(NULL, "Failed to rename", ""); + } + out: ++ if (delete_original) { ++ unlink(original); ++ } ++ if (tmpdir) { ++ rmdir(tmpdir); ++ g_free(tmpdir); ++ } ++ if (manifest) { ++ free_manifest(manifest); ++ } ++ g_clear_error(&gerror); ++ free(cmd); + free(testnewfile); + free(conf); + free(newfile); +diff --git a/src/globals.c b/src/globals.c +index 74758ce..0e047e2 100644 +--- a/src/globals.c ++++ b/src/globals.c +@@ -40,6 +40,7 @@ char *state_dir = NULL; + char *packstage_dir = NULL; + char *image_dir = NULL; + char *staging_dir = NULL; ++char *content_url = NULL; + + bool set_format(char *userinput) + { +@@ -76,6 +77,17 @@ bool set_state_dir(char *dir) + return true; + } + ++bool set_content_url(const char *url) ++{ ++ if (content_url) { ++ free(content_url); ++ } ++ string_or_die(&content_url, "%s", url); ++ ++ return true; ++} ++ ++ + bool init_globals(void) + { + if (format == 0) { +@@ -123,4 +135,5 @@ void free_state_globals(void) + free(packstage_dir); + free(image_dir); + free(staging_dir); ++ free(content_url); + } +diff --git a/src/make_packs.c b/src/make_packs.c +index 2d3e25e..827be56 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -49,6 +49,7 @@ static const struct option prog_opts[] = { + { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { "signcontent", no_argument, 0, 's' }, ++ { "content-url", required_argument, 0, 'u' }, + { 0, 0, 0, 0 } + }; + +@@ -61,6 +62,7 @@ static void usage(const char *name) + printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); ++ printf(" -u, --content-url Base URL of the update repo (optional, used to retrieve missing files on demand"); + printf("\n"); + } + +@@ -86,6 +88,12 @@ static bool parse_options(int argc, char **argv) + case 's': + enable_signing = true; + break; ++ case 'u': ++ if (!optarg || !set_content_url(optarg)) { ++ printf("Invalid --content-url argument ''%s''\n\n", optarg); ++ return false; ++ } ++ break; + } + } + +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch new file mode 100644 index 0000000..43ae6ff --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch @@ -0,0 +1,56 @@ +From d253e81e001ccbff9d7a8ee1768006c5ac36b259 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 27 Sep 2016 16:43:57 +0200 +Subject: [PATCH 25/29] swupd_make_pack: fix extracting files with bsdtar + +TAR_XATTR_ARGS is no longer used as part of a plain string. Embedding +the empty "" value for bsdtar inside an argv argument list passes an +empty parameter to bsdtar, leading to: + bsdtar: Must specify one of -c, -r, -t, -u, -x + +To allow the the "no parameter" case, it has to be argument list: that +can be empty. If not empty, it has to end with a comma. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/41] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 4 ++-- + src/pack.c | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index 43a5002..0aa6399 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -20,12 +20,12 @@ + #define TAR_COMMAND "bsdtar" + #define TAR_XATTR_ARGS "" + #define TAR_XATTR_ARGS_STRLIST +-#define TAR_WARN_ARGS "" ++#define TAR_WARN_ARGS_STRLIST + #else + #define TAR_COMMAND "tar" + #define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'" + #define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'", +-#define TAR_WARN_ARGS "--warning=no-timestamp" ++#define TAR_WARN_ARGS_STRLIST "--warning=no-timestamp", + #endif + + #if SWUPD_WITH_SELINUX +diff --git a/src/pack.c b/src/pack.c +index a1fbc51..12d7443 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -116,7 +116,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module) + * time on the client... + */ + string_or_die(¶m, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version); +- char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL }; ++ char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS_STRLIST TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL }; + if (system_argv(tarcmd) == 0) { + unlink(path); + } +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch new file mode 100644 index 0000000..abc3082 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch @@ -0,0 +1,30 @@ +From a8b77a7d4479df7662faf3a15f5aca6828d1c67a Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 27 Sep 2016 08:12:49 +0200 +Subject: [PATCH 26/29] fullfiles.c: fix invalid LOG() call + +LOG() takes an additional fixed string before the format string. + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/7e38f013eff01b84a8df88557f5312dc070ef0b3] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/fullfiles.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 31b9ab5..3be43d1 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -72,7 +72,7 @@ static void create_fullfile(struct file *file) + string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename); + if (lstat(origin, &sbuf) < 0) { + /* no input file: means earlier phase of update creation failed */ +- LOG(NULL, "Failed to stat %s\n", origin); ++ LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno)); + assert(0); + } + +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch new file mode 100644 index 0000000..0a68667 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch @@ -0,0 +1,89 @@ +From 8686189b3b080446fae732a85b72528b7fe68ba6 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 27 Sep 2016 08:25:40 +0200 +Subject: [PATCH 27/29] update control over parallelism + +The SWUPD_NUM_THREADS env variable is now understood by all three +commands and overrides the default number of threads. Setting it to 1 +is useful while debugging the code that runs inside threads (only one +thread hits breakpoints there). + +The hard-coded parallelism of 12 threads when analysing the file system +gets replaced with n, where n is the number of available CPUs. The default +is the same as before elsewhere (n for packing, 3 * n for fullfiles). + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/4e0fdd4193a8ce9dcf3cfc5e488dfd4b23b7e7d9] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/analyze_fs.c | 5 ++++- + src/fullfiles.c | 7 +++++-- + src/pack.c | 7 +++++-- + 3 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/src/analyze_fs.c b/src/analyze_fs.c +index 3bfb288..0f16343 100644 +--- a/src/analyze_fs.c ++++ b/src/analyze_fs.c +@@ -387,6 +387,9 @@ struct manifest *full_manifest_from_directory(int version) + { + struct manifest *manifest; + char *dir; ++ int numthreads = getenv("SWUPD_NUM_THREADS") ? ++ atoi(getenv("SWUPD_NUM_THREADS")) : ++ sysconf(_SC_NPROCESSORS_ONLN); + + LOG(NULL, "Computing hashes", "for %i/full", version); + +@@ -394,7 +397,7 @@ struct manifest *full_manifest_from_directory(int version) + + string_or_die(&dir, "%s/%i/full", image_dir, version); + +- threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL); ++ threadpool = g_thread_pool_new(get_hash, dir, numthreads, FALSE, NULL); + + iterate_directory(manifest, dir, "", true); + +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 3be43d1..216a1d7 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -291,10 +291,13 @@ static void submit_fullfile_tasks(GList *files) + int ret; + int count = 0; + GError *err = NULL; ++ int numthreads = getenv("SWUPD_NUM_THREADS") ? ++ atoi(getenv("SWUPD_NUM_THREADS")) : ++ sysconf(_SC_NPROCESSORS_ONLN) * 3; + +- LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3); ++ LOG(NULL, "fullfile threadpool", "%d threads", numthreads); + threadpool = g_thread_pool_new(create_fullfile_task, NULL, +- sysconf(_SC_NPROCESSORS_ONLN) * 3, ++ numthreads, + TRUE, NULL); + + printf("Starting downloadable fullfiles data creation\n"); +diff --git a/src/pack.c b/src/pack.c +index 12d7443..3ffb88a 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -285,10 +285,13 @@ static void make_pack_deltas(GList *files) + struct file *file; + int ret; + GError *err = NULL; ++ int numthreads = getenv("SWUPD_NUM_THREADS") ? ++ atoi(getenv("SWUPD_NUM_THREADS")) : ++ sysconf(_SC_NPROCESSORS_ONLN); + +- LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN)); ++ LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); + threadpool = g_thread_pool_new(create_delta, NULL, +- sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL); ++ numthreads, FALSE, NULL); + + item = g_list_first(files); + while (item) { +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch new file mode 100644 index 0000000..0322735 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch @@ -0,0 +1,100 @@ +From f74807f9aebbb7b8feb1f50107e268bd869f2691 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 28 Sep 2016 16:55:22 +0200 +Subject: [PATCH 1/3] enable locales in all programs + +This is a pre-condition for using libarchive directly: libarchive +needs to know what the encoding of filenames is, and it uses the +current locale for that. Without setlocale(), the locale is "C", which +only supports ASCII filenames, leading to warnings about "Can't +encode..." from libarchive when it is forced to fall back to copying +strings verbatim when writing archives that require UTF-8 encoding. + +As a side effect, error messages from libc will get translated +according to the user's environment. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/44] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + +--- + src/create_update.c | 6 ++++++ + src/make_fullfiles.c | 6 ++++++ + src/make_packs.c | 6 ++++++ + 3 files changed, 18 insertions(+) + +diff --git a/src/create_update.c b/src/create_update.c +index 97045e5..766609b 100644 +--- a/src/create_update.c ++++ b/src/create_update.c +@@ -29,6 +29,7 @@ + #include <errno.h> + #include <getopt.h> + #include <glib.h> ++#include <locale.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> +@@ -240,6 +241,11 @@ int main(int argc, char **argv) + /* keep valgrind working well */ + setenv("G_SLICE", "always-malloc", 0); + ++ if (!setlocale(LC_ALL, "")) { ++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]); ++ return EXIT_FAILURE; ++ } ++ + if (!parse_options(argc, argv)) { + free_globals(); + return EXIT_FAILURE; +diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c +index 4ea2f01..2a1e2e9 100644 +--- a/src/make_fullfiles.c ++++ b/src/make_fullfiles.c +@@ -23,6 +23,7 @@ + #define _GNU_SOURCE + #include <assert.h> + #include <getopt.h> ++#include <locale.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> +@@ -88,6 +89,11 @@ int main(int argc, char **argv) + /* keep valgrind working well */ + setenv("G_SLICE", "always-malloc", 0); + ++ if (!setlocale(LC_ALL, "")) { ++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]); ++ return EXIT_FAILURE; ++ } ++ + if (!parse_options(argc, argv)) { + free_state_globals(); + return EXIT_FAILURE; +diff --git a/src/make_packs.c b/src/make_packs.c +index 4002cd9..8560b3f 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -27,6 +27,7 @@ + #include <getopt.h> + #include <getopt.h> + #include <glib.h> ++#include <locale.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> +@@ -101,6 +102,11 @@ int main(int argc, char **argv) + int exit_status = EXIT_FAILURE; + char *file_path = NULL; + ++ if (!setlocale(LC_ALL, "")) { ++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]); ++ return EXIT_FAILURE; ++ } ++ + if (!parse_options(argc, argv)) { + free_state_globals(); + return EXIT_FAILURE; +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch new file mode 100644 index 0000000..d599261 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch @@ -0,0 +1,626 @@ +From 4ebc886b910fa4cb83864658069a30dd133caca8 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 30 Mar 2016 13:14:42 +0200 +Subject: [PATCH 3/6] fullfiles: use libarchive directly + +Calling an external tar command makes the code fairly complicated, +because it is necessary to set up a suitable temporary directory with +the desired content. By calling libarchive directly, no temporary copy +is needed and directories and files can be treated (almost) the same +way. The only difference is that for files, data has to be added to +the archive. + +More important, performance under bitbake took a big hit because of +the external commands. Launching them when running under pseudo is a +lot slower compared to running natively as root. For example, the +unmodified swupd-make-fullfiles took over 20min for +ostro-image-swupd. With this change, it completes in 3:30min. When +running natively as root, there is also some improvement because less +work needs to be done, but total runtime only decreases from 2:28min +to 2:14min. + +By calling libarchive directly, swupd also gets better control over +error handling. bsdtar emits a warning for invalid filename or +linkname encoding, which was silently ignored when using the external +command. Now it is treated as an error. + +Archives always get created using the restricted pax interchange +format (see +https://github.com/libarchive/libarchive/wiki/ManPageLibarchiveFormats5), +same as with using bsdtar as external command. GNU tar should be able +to decode them properly as long as no extensions are needed (for +example, for xattrs). + +Only the smallest archive really gets written to disk. Until then, the +compressed archive is kept in memory. As an additional optimization, +creating an archive gets aborted once it is already larger than the +currently best one. In practice, that particular optimization did not +have any significant impact on performance. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/48] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + +--- + Makefile.am | 6 +- + configure.ac | 1 + + include/libarchive_helper.h | 42 ++++++ + src/fullfiles.c | 324 +++++++++++++++++++++++++------------------- + src/in_memory_archive.c | 67 +++++++++ + 5 files changed, 300 insertions(+), 140 deletions(-) + create mode 100644 include/libarchive_helper.h + create mode 100644 src/in_memory_archive.c + +diff --git a/Makefile.am b/Makefile.am +index 528f02f..b14feb0 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -23,6 +23,7 @@ swupd_create_update_SOURCES = \ + src/helpers.c \ + src/heuristics.c \ + src/log.c \ ++ src/in_memory_archive.c \ + src/manifest.c \ + src/pack.c \ + src/rename.c \ +@@ -54,6 +55,7 @@ swupd_make_fullfiles_SOURCES = \ + src/fullfiles.c \ + src/globals.c \ + src/helpers.c \ ++ src/in_memory_archive.c \ + src/log.c \ + src/make_fullfiles.c \ + src/manifest.c \ +@@ -63,12 +65,13 @@ swupd_make_fullfiles_SOURCES = \ + src/stats.c \ + src/xattrs.c + +-AM_CPPFLAGS = $(glib_CFLAGS) -I$(top_srcdir)/include ++AM_CPPFLAGS = $(glib_CFLAGS) $(libarchive_CFLAGS) -I$(top_srcdir)/include + + swupd_create_update_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ ++ $(libarchive_LIBS) \ + $(bsdiff_LIBS) + + swupd_make_pack_LDADD = \ +@@ -81,6 +84,7 @@ swupd_make_fullfiles_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ ++ $(libarchive_LIBS) \ + $(bsdiff_LIBS) + + if ENABLE_LZMA +diff --git a/configure.ac b/configure.ac +index 8819206..3e2d933 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -36,6 +36,7 @@ AC_ARG_ENABLE( + AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command])), + AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command]) + ) ++PKG_CHECK_MODULES([libarchive], [libarchive]) + + AC_ARG_ENABLE( + [tests], +diff --git a/include/libarchive_helper.h b/include/libarchive_helper.h +new file mode 100644 +index 0000000..ad28def +--- /dev/null ++++ b/include/libarchive_helper.h +@@ -0,0 +1,42 @@ ++/* ++ * Software Updater - server side ++ * ++ * Copyright © 2016 Intel Corporation. ++ * ++ * This program is free software: you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation, version 2 or later of the License. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program. If not, see <http://www.gnu.org/licenses/>. ++ * ++ * Authors: ++ * Patrick Ohly <patrick.ohly@intel.com> ++ * ++ */ ++ ++#ifndef __INCLUDE_GUARD_LIBARCHIVE_HELPER_H ++#define __INCLUDE_GUARD_LIBARCHIVE_HELPER_H ++ ++#include <archive.h> ++#include <stdint.h> ++ ++/* ++ * Used by archive_write_open() callbacks to store the resulting archive in memory. ++ */ ++struct in_memory_archive { ++ uint8_t *buffer; ++ size_t allocated; ++ size_t used; ++ /* If not 0, aborts writing when the used data would become larger than this. */ ++ size_t maxsize; ++}; ++ ++ssize_t in_memory_write(struct archive *, void *client_data, const void *buffer, size_t length); ++ ++#endif /* __INCLUDE_GUARD_LIBARCHIVE_HELPER_H */ +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 216a1d7..9fdfc2f 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -22,6 +22,8 @@ + */ + + #define _GNU_SOURCE ++#include <archive.h> ++#include <archive_entry.h> + #include <assert.h> + #include <errno.h> + #include <fcntl.h> +@@ -33,24 +35,27 @@ + #include <string.h> + #include <sys/stat.h> + #include <sys/types.h> ++#include <sys/xattr.h> + #include <unistd.h> + + #include "swupd.h" ++#include "libarchive_helper.h" + + /* output must be a file, which is a (compressed) tar file, of the file denoted by "file", without any of its + directory paths etc etc */ + static void create_fullfile(struct file *file) + { +- char *origin; ++ char *origin = NULL; + char *tarname = NULL; +- char *rename_source = NULL; +- char *rename_target = NULL; +- char *rename_tmpdir = NULL; +- int ret; + struct stat sbuf; + char *empty, *indir, *outdir; +- char *param1, *param2; +- int stderrfd; ++ struct archive_entry *entry = NULL; ++ struct archive *from = NULL, *to = NULL; ++ struct in_memory_archive best = { .buffer = NULL }; ++ struct in_memory_archive current = { .buffer = NULL }; ++ uint8_t *file_content = NULL; ++ size_t file_size; ++ int fd = -1; + + if (file->is_deleted) { + return; /* file got deleted -> by definition we cannot tar it up */ +@@ -59,15 +64,17 @@ static void create_fullfile(struct file *file) + empty = config_empty_dir(); + indir = config_image_base(); + outdir = config_output_dir(); ++ entry = archive_entry_new(); ++ assert(entry); ++ from = archive_read_disk_new(); ++ assert(from); + + string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); + if (access(tarname, R_OK) == 0) { + /* output file already exists...done */ +- free(tarname); ++ goto done; + return; + } +- free(tarname); +- //printf("%s was missing\n", file->hash); + + string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename); + if (lstat(origin, &sbuf) < 0) { +@@ -76,156 +83,195 @@ static void create_fullfile(struct file *file) + assert(0); + } + +- if (file->is_dir) { /* directories are easy */ +- char *tmp1, *tmp2, *dir, *base; +- +- tmp1 = strdup(origin); +- assert(tmp1); +- base = basename(tmp1); +- +- tmp2 = strdup(origin); +- assert(tmp2); +- dir = dirname(tmp2); +- +- string_or_die(&rename_tmpdir, "%s/XXXXXX", outdir); +- if (!mkdtemp(rename_tmpdir)) { +- LOG(NULL, "Failed to create temporary directory for %s move", origin); +- assert(0); +- } +- +- string_or_die(¶m1, "--exclude=%s/?*", base); +- string_or_die(¶m2, "./%s", base); +- char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL }; +- char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL }; +- +- stderrfd = open("/dev/null", O_WRONLY); +- if (stderrfd == -1) { +- LOG(NULL, "Failed to open /dev/null", ""); +- assert(0); +- } +- if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) { +- assert(0); +- } +- free(param1); +- free(param2); +- close(stderrfd); +- +- string_or_die(&rename_source, "%s/%s", rename_tmpdir, base); +- string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash); +- if (rename(rename_source, rename_target)) { +- LOG(NULL, "rename failed for %s to %s", rename_source, rename_target); ++ /* step 1: tar it with each compression type */ ++ typedef int (*filter_t)(struct archive *); ++ static const filter_t compression_filters[] = { ++ /* ++ * Start with the compression method that is most likely (*) to produce ++ * the best result. That will allow aborting creation of archives earlier ++ * when they become larger than the currently smallest archive. ++ * ++ * (*) statistics for ostro-image-swupd: ++ * 43682 LZMA ++ * 13398 gzip ++ * 844 bzip2 ++ */ ++ archive_write_add_filter_lzma, ++ archive_write_add_filter_gzip, ++ archive_write_add_filter_bzip2, ++ /* ++ * TODO (?): can archive_write_add_filter_none ever be better than compressing? ++ */ ++ NULL ++ }; ++ file_size = S_ISREG(sbuf.st_mode) ? sbuf.st_size : 0; ++ ++ archive_entry_copy_sourcepath(entry, origin); ++ if (archive_read_disk_entry_from_file(from, entry, -1, &sbuf)) { ++ LOG(NULL, "Getting directory attributes failed", "%s: %s", ++ origin, archive_error_string(from)); ++ assert(0); ++ } ++ archive_entry_copy_pathname(entry, file->hash); ++ if (file_size) { ++ file_content = malloc(file_size); ++ if (!file_content) { ++ LOG(NULL, "out of memory", ""); + assert(0); + } +- free(rename_source); +- +- /* for a directory file, tar up simply with gzip */ +- string_or_die(¶m1, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); +- char *const tarcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param1, file->hash, NULL }; +- +- if (system_argv(tarcmd) != 0) { ++ fd = open(origin, O_RDONLY); ++ if (fd == -1) { ++ LOG(NULL, "Failed to open file", "%s: %s", ++ origin, strerror(errno)); + assert(0); + } +- free(param1); +- +- if (rmdir(rename_target)) { +- LOG(NULL, "rmdir failed for %s", rename_target); +- } +- free(rename_target); +- if (rmdir(rename_tmpdir)) { +- LOG(NULL, "rmdir failed for %s", rename_tmpdir); +- } +- free(rename_tmpdir); +- +- free(tmp1); +- free(tmp2); +- } else { /* files are more complex */ +- char *gzfile = NULL, *bzfile = NULL, *xzfile = NULL; +- char *tempfile; +- uint64_t gz_size = LONG_MAX, bz_size = LONG_MAX, xz_size = LONG_MAX; +- +- /* step 1: hardlink the guy to an empty directory with the hash as the filename */ +- string_or_die(&tempfile, "%s/%s", empty, file->hash); +- if (link(origin, tempfile) < 0) { +- LOG(NULL, "hardlink failed", "%s due to %s (%s -> %s)", file->filename, strerror(errno), origin, tempfile); +- char *const argv[] = { "cp", "-a", origin, tempfile, NULL }; +- if (system_argv(argv) != 0) { ++ size_t done = 0; ++ while (done < file_size) { ++ ssize_t curr; ++ curr = read(fd, file_content + done, file_size - done); ++ if (curr == -1) { ++ LOG(NULL, "Failed to read from file", "%s: %s", ++ origin, strerror(errno)); + assert(0); + } ++ done += curr; + } ++ } + +- /* step 2a: tar it with each compression type */ +- // lzma +- string_or_die(¶m1, "--directory=%s", empty); +- string_or_die(¶m2, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash); +- char *const tarlzmacmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf", param2, file->hash, NULL }; +- +- if (system_argv(tarlzmacmd) != 0) { ++ for (int i = 0; compression_filters[i]; i++) { ++ /* Need to re-initialize the archive handle, it cannot be re-used. */ ++ if (to) { ++ archive_write_free(to); ++ } ++ /* ++ * Use the recommended restricted pax interchange ++ * format. Numeric uid/gid values are stored in the archive ++ * (no uid/gid lookup enabled) because symbolic names can lead ++ * to a hash mismatch during unpacking when /etc/passwd or ++ * /etc/group change during an update (see ++ * https://github.com/clearlinux/swupd-client/issues/101). ++ * ++ * Filenames read from the file system are expected to be ++ * valid according to the current locale. archive_write_header() ++ * will warn about filenames that it cannot properly decode ++ * and proceeds by writing the raw bytes, but we treat this an ++ * error by not distinguishing between ARCHIVE_FATAL ++ * and ARCHIVE_WARN. ++ * ++ * When we fail with "Can't translate" errors, make sure that ++ * LANG and/or LC_ env variables are set. ++ */ ++ to = archive_write_new(); ++ assert(to); ++ if (archive_write_set_format_pax_restricted(to)) { ++ LOG(NULL, "PAX format", "%s", archive_error_string(to)); + assert(0); + } +- free(param1); +- free(param2); +- +- // gzip +- string_or_die(¶m1, "--directory=%s", empty); +- string_or_die(¶m2, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash); +- char *const targzipcmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param2, file->hash, NULL }; +- +- if (system_argv(targzipcmd) != 0) { ++ do { ++ /* Try compression methods until we find one which is supported. */ ++ if (!compression_filters[i](to)) { ++ break; ++ } ++ } while(compression_filters[++i]); ++ /* ++ * Regardless of the block size below, never pad the ++ * last block, it just makes the archive larger. ++ */ ++ if (archive_write_set_bytes_in_last_block(to, 1)) { ++ LOG(NULL, "Removing padding failed", ""); + assert(0); + } +- free(param1); +- free(param2); +- +-#ifdef SWUPD_WITH_BZIP2 +- string_or_die(¶m1, "--directory=%s", empty); +- string_or_die(¶m2, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash); +- char *const tarbzip2cmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-jcf", param2, file->hash, NULL }; +- +- if (system_argv(tarbzip2cmd) != 0) { ++ /* ++ * Invoke in_memory_write() as often as possible and check each ++ * time whether we are already larger than the currently best ++ * algorithm. ++ */ ++ current.maxsize = best.used; ++ if (archive_write_set_bytes_per_block(to, 0)) { ++ LOG(NULL, "Removing blocking failed", ""); + assert(0); + } +- free(param1); +- free(param2); +- +-#endif +- +- /* step 2b: pick the smallest of the three compression formats */ +- string_or_die(&gzfile, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash); +- if (stat(gzfile, &sbuf) == 0) { +- gz_size = sbuf.st_size; ++ /* ++ * We can make an educated guess how large the resulting archive will be. ++ * Avoids realloc() calls when the file is big. ++ */ ++ if (!current.allocated) { ++ current.allocated = file_size + 4096; ++ current.buffer = malloc(current.allocated); + } +- string_or_die(&bzfile, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash); +- if (stat(bzfile, &sbuf) == 0) { +- bz_size = sbuf.st_size; ++ if (!current.buffer) { ++ LOG(NULL, "out of memory", ""); ++ assert(0); + } +- string_or_die(&xzfile, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash); +- if (stat(xzfile, &sbuf) == 0) { +- xz_size = sbuf.st_size; ++ if (archive_write_open(to, ¤t, NULL, in_memory_write, NULL)) { ++ LOG(NULL, "Failed to create archive", "%s", ++ archive_error_string(to)); ++ assert(0); + } +- string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); +- if (gz_size <= xz_size && gz_size <= bz_size) { +- ret = rename(gzfile, tarname); +- } else if (xz_size <= bz_size) { +- ret = rename(xzfile, tarname); +- } else { +- ret = rename(bzfile, tarname); ++ if (archive_write_header(to, entry) || ++ file_content && archive_write_data(to, file_content, file_size) != (ssize_t)file_size || ++ archive_write_close(to)) { ++ if (current.maxsize && current.used >= current.maxsize) { ++ archive_write_free(to); ++ to = NULL; ++ continue; ++ } ++ LOG(NULL, "Failed to store file in archive", "%s: %s", ++ origin, archive_error_string(to)); ++ assert(0); + } +- if (ret != 0) { +- LOG(file, "post-tar rename failed", "ret=%d", ret); ++ if (!best.used || current.used < best.used) { ++ free(best.buffer); ++ best = current; ++ memset(¤t, 0, sizeof(current)); ++ } else { ++ /* Simply re-use the buffer for the next iteration. */ ++ current.used = 0; + } +- unlink(bzfile); +- unlink(xzfile); +- unlink(gzfile); +- free(bzfile); +- free(xzfile); +- free(gzfile); +- free(tarname); +- +- /* step 3: remove the hardlink */ +- unlink(tempfile); +- free(tempfile); ++ } ++ if (!best.used) { ++ LOG(NULL, "creating archive failed with all compression methods", ""); ++ assert(0); + } + ++ /* step 2: write out to disk. Archives are immutable and thus read-only. */ ++ fd = open(tarname, O_CREAT|O_WRONLY, S_IRUSR|S_IRGRP|S_IROTH); ++ if (fd <= 0) { ++ LOG(NULL, "Failed to create archive", "%s: %s", ++ tarname, strerror(errno)); ++ assert(0); ++ } ++ size_t done = 0; ++ while (done < best.used) { ++ ssize_t curr; ++ curr = write(fd, best.buffer + done, best.used - done); ++ if (curr == -1) { ++ LOG(NULL, "Failed to write archive", "%s: %s", ++ tarname, strerror(errno)); ++ assert(0); ++ } ++ done += curr; ++ } ++ if (close(fd)) { ++ LOG(NULL, "Failed to complete writing archive", "%s: %s", ++ tarname, strerror(errno)); ++ assert(0); ++ } ++ fd = -1; ++ free(best.buffer); ++ free(current.buffer); ++ free(file_content); ++ ++ done: ++ if (fd >= 0) { ++ close(fd); ++ } ++ archive_read_free(from); ++ if (to) { ++ archive_write_free(to); ++ } ++ archive_entry_free(entry); ++ free(tarname); + free(indir); + free(outdir); + free(empty); +diff --git a/src/in_memory_archive.c b/src/in_memory_archive.c +new file mode 100644 +index 0000000..abd7e54 +--- /dev/null ++++ b/src/in_memory_archive.c +@@ -0,0 +1,67 @@ ++/* ++ * Software Updater - server side ++ * ++ * Copyright © 2016 Intel Corporation. ++ * ++ * This program is free software: you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation, version 2 or later of the License. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program. If not, see <http://www.gnu.org/licenses/>. ++ * ++ * Authors: ++ * Patrick Ohly <patrick.ohly@intel.com> ++ * ++ */ ++ ++#include <errno.h> ++#include <stdlib.h> ++ ++#include "libarchive_helper.h" ++ ++ssize_t in_memory_write(struct archive *archive, void *client_data, const void *buffer, size_t length) ++{ ++ struct in_memory_archive *in_memory = client_data; ++ void *newbuff; ++ ++ if (in_memory->maxsize && in_memory->used + length >= in_memory->maxsize) { ++ archive_set_error(archive, EFBIG, "resulting archive would become larger than %lu", ++ (unsigned long)in_memory->maxsize); ++ archive_write_fail(archive); ++ /* ++ * Despite the error and archive_write_fail(), libarchive internally calls us ++ * again and when we fail again, overwrites our error with something about ++ * "Failed to clean up compressor". Therefore our caller needs to check for "used == maxsize" ++ * to detect that we caused the failure. ++ */ ++ in_memory->used = in_memory->maxsize; ++ return -1; ++ } ++ ++ if (in_memory->used + length > in_memory->allocated) { ++ /* Start with a small chunk, double in size to avoid too many reallocs. */ ++ size_t new_size = in_memory->allocated ? ++ in_memory->allocated * 2 : ++ 4096; ++ while (new_size < in_memory->used + length) { ++ new_size *= 2; ++ } ++ newbuff = realloc(in_memory->buffer, new_size); ++ if (!newbuff) { ++ archive_set_error(archive, ENOMEM, "failed to enlarge buffer"); ++ return -1; ++ } ++ in_memory->buffer = newbuff; ++ in_memory->allocated = new_size; ++ } ++ ++ memcpy(in_memory->buffer + in_memory->used, buffer, length); ++ in_memory->used += length; ++ return length; ++} +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch b/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch index 4f5ba19..4f5ba19 100644 --- a/recipes-core/swupd-server/swupd-server/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch +++ b/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch diff --git a/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch index a3a72ad..f27b111 100644 --- a/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch +++ b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch @@ -1,7 +1,7 @@ -From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001 +From 4be3d38364cc1bead38c32465cb7e2f3a0be4c88 Mon Sep 17 00:00:00 2001 From: Patrick Ohly <patrick.ohly@intel.com> Date: Wed, 16 Nov 2016 14:26:30 +0100 -Subject: [PATCH 1/3] create_pack: rely less on previous builds +Subject: [PATCH 11/13] create_pack: rely less on previous builds When a file has not been modified in the current build, then by definition the current copy of the file is the same as in the build @@ -28,7 +28,10 @@ make_pack_full_files() and make_final_pack() used the exact same code for populating the "staged" directory. Now that common code is in stage_entry(). +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + --- include/swupd.h | 2 +- src/delta.c | 4 +-- @@ -36,32 +39,32 @@ Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/include/swupd.h b/include/swupd.h -index c1c0e96..cf384e3 100644 +index 2263018..1b3a046 100644 --- a/include/swupd.h +++ b/include/swupd.h -@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest); +@@ -247,7 +247,7 @@ extern void type_change_detection(struct manifest *manifest); extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); extern void link_renames(GList *newfiles, struct manifest *from_manifest); --extern void __create_delta(struct file *file, int from_version); -+extern void __create_delta(struct file *file, int from_version, int to_version); +-extern void __create_delta(struct file *file, int from_version, char *from_hash); ++extern void __create_delta(struct file *file, int from_version, int to_version, char *from_hash); extern void account_delta_hit(void); extern void account_delta_miss(void); diff --git a/src/delta.c b/src/delta.c -index 7e978b1..8fff4c9 100644 +index 563abb9..8c1bc64 100644 --- a/src/delta.c +++ b/src/delta.c @@ -35,7 +35,7 @@ #include "swupd.h" #include "xattrs.h" --void __create_delta(struct file *file, int from_version) -+void __create_delta(struct file *file, int from_version, int to_version) +-void __create_delta(struct file *file, int from_version, char *from_hash) ++void __create_delta(struct file *file, int from_version, int to_version, char *from_hash) { - char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; + char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL, *conf = NULL; char *tmpdir = NULL; -@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version) +@@ -59,7 +59,7 @@ void __create_delta(struct file *file, int from_version, char *from_hash) } conf = config_image_base(); @@ -71,7 +74,7 @@ index 7e978b1..8fff4c9 100644 string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); diff --git a/src/pack.c b/src/pack.c -index 984c2d6..ccb28bd 100644 +index a7094b8..f7770b2 100644 --- a/src/pack.c +++ b/src/pack.c @@ -37,6 +37,7 @@ @@ -184,8 +187,8 @@ index 984c2d6..ccb28bd 100644 /* if the file was not found in the from version, skip delta creation */ if (file->peer) { -- __create_delta(file, file->peer->last_change); -+ __create_delta(file, file->peer->last_change, *to_version); +- __create_delta(file, file->peer->last_change, file->peer->hash); ++ __create_delta(file, file->peer->last_change, *to_version, file->peer->hash); } } @@ -194,8 +197,8 @@ index 984c2d6..ccb28bd 100644 { GThreadPool *threadpool; GList *item; -@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files) - sysconf(_SC_NPROCESSORS_ONLN); +@@ -290,7 +323,7 @@ static void make_pack_deltas(GList *files) + int numthreads = num_threads(1.0); LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); - threadpool = g_thread_pool_new(create_delta, NULL, @@ -203,7 +206,7 @@ index 984c2d6..ccb28bd 100644 numthreads, FALSE, NULL); item = g_list_first(files); -@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack) +@@ -365,7 +398,7 @@ static int make_final_pack(struct packdata *pack) file->last_change, file->hash); string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, pack->module, pack->from, pack->to, file->hash); @@ -212,7 +215,7 @@ index 984c2d6..ccb28bd 100644 string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, pack->module, pack->from, pack->to, file->hash); -@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack) +@@ -399,27 +432,7 @@ static int make_final_pack(struct packdata *pack) } } } else { @@ -241,7 +244,7 @@ index 984c2d6..ccb28bd 100644 if (ret == 0) { pack->fullcount++; } -@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack) +@@ -537,7 +550,7 @@ int make_pack(struct packdata *pack) /* step 2: consolidate delta list & create all delta files*/ delta_list = consolidate_packs_delta_files(delta_list, pack); diff --git a/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch b/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch index 7151c5c..db25851 100644 --- a/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch +++ b/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch @@ -1,7 +1,7 @@ -From e1f0d54a940eb7d04e2fbd59bd995a819331425f Mon Sep 17 00:00:00 2001 +From f35a8484f100ac0da0fbe173fcd5e8843321b000 Mon Sep 17 00:00:00 2001 From: Patrick Ohly <patrick.ohly@intel.com> Date: Fri, 30 Sep 2016 08:42:08 +0200 -Subject: [PATCH 1/2] swupd-create-update: alternative input layout +Subject: [PATCH 07/13] swupd-create-update: alternative input layout In Ostro OS, we already have a "full" directory with all files. Splitting it up into bundles just so that swupd-create-update can @@ -24,16 +24,18 @@ That way it is even possible to mix the two modes, i.e. replacing only some bundles with a content list, although that's probably not all that useful. +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/55] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + --- - src/analyze_fs.c | 171 ++++++++++++++++++++++++++++++++++++++-------------- - src/chroot.c | 24 +++++--- + src/analyze_fs.c | 176 +++++++++++++++++++++++++++++++++++++++------------- + src/chroot.c | 24 ++++--- src/create_update.c | 6 +- - src/fullfiles.c | 2 + - 4 files changed, 150 insertions(+), 53 deletions(-) + 3 files changed, 153 insertions(+), 53 deletions(-) diff --git a/src/analyze_fs.c b/src/analyze_fs.c -index 0f16343..d534587 100644 +index ce30393..ac3731c 100644 --- a/src/analyze_fs.c +++ b/src/analyze_fs.c @@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data) @@ -45,15 +47,15 @@ index 0f16343..d534587 100644 { char c; int i; -@@ -301,25 +301,145 @@ static bool illegal_characters(char *filename) +@@ -301,27 +301,151 @@ static bool illegal_characters(char *filename) return false; } -+static void add_file(struct manifest *manifest, -+ const char *entry_name, -+ char *sub_filename, -+ char *fullname, -+ bool do_hash) ++static struct file *add_file(struct manifest *manifest, ++ const char *entry_name, ++ char *sub_filename, ++ char *fullname, ++ bool do_hash) +{ + GError *err = NULL; + struct file *file; @@ -62,7 +64,7 @@ index 0f16343..d534587 100644 + printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename); + free(sub_filename); + free(fullname); -+ return; ++ return NULL; + } + + file = calloc(1, sizeof(struct file)); @@ -104,6 +106,7 @@ index 0f16343..d534587 100644 + } + manifest->files = g_list_prepend(manifest->files, file); + manifest->count++; ++ return file; +} + + @@ -120,10 +123,11 @@ index 0f16343..d534587 100644 dir = opendir(fullpath); if (!dir) { ++ bool fatal_error = errno != ENOENT; + FILE *content; -+ int len; -+ free(fullpath); -+ if (errno != ENOENT) { ++ + free(fullpath); ++ if (fatal_error) { + return; + } + /* @@ -134,14 +138,15 @@ index 0f16343..d534587 100644 + * corresponding file system entry is then + * expected to be in a pre-populated "full" + * directory. ++ * ++ * Only supported at top level (i.e. empty ++ * subpath) to keep the code and testing ++ * simpler. + */ -+ if (subpath[0]) { -+ string_or_die(&fullpath, "%s/%s.content.txt", pathprefix, len, subpath); -+ } else { -+ string_or_die(&fullpath, "%s.content.txt", pathprefix); -+ } ++ assert(!subpath[0]); ++ string_or_die(&fullpath, "%s.content.txt", pathprefix); + content = fopen(fullpath, "r"); - free(fullpath); ++ free(fullpath); + fullpath = NULL; + if (content) { + char *line = NULL; @@ -193,8 +198,11 @@ index 0f16343..d534587 100644 - struct file *file; char *sub_filename; char *fullname; ++ struct file *file; -@@ -334,50 +454,13 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix, + entry = readdir(dir); + if (!entry) { +@@ -334,50 +458,14 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix, } string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name); @@ -216,13 +224,12 @@ index 0f16343..d534587 100644 string_or_die(&fullname, "%s/%s", fullpath, entry->d_name); - populate_file_struct(file, fullname); - free(fullname); -- - if (entry->d_type == DT_DIR) { + +- if (file->is_dir) { - iterate_directory(manifest, pathprefix, file->filename, do_hash); -+ iterate_directory(manifest, pathprefix, sub_filename, do_hash); - } -+ /* takes ownership of the strings */ -+ add_file(manifest, entry->d_name, sub_filename, fullname, do_hash); +- } ++ /* takes ownership of the strings, so we don't need to free it */ ++ file = add_file(manifest, entry->d_name, sub_filename, fullname, do_hash); - /* if for some reason there is a file in the official build - * which should not be included in the Manifest, then open a bug @@ -242,7 +249,9 @@ index 0f16343..d534587 100644 - closedir(dir); - return; - } -- } ++ if (file && file->is_dir) { ++ iterate_directory(manifest, pathprefix, file->filename, do_hash); + } - manifest->files = g_list_prepend(manifest->files, file); - manifest->count++; } @@ -297,7 +306,7 @@ index 32ed997..f3832e1 100644 free(param); } diff --git a/src/create_update.c b/src/create_update.c -index 766609b..74d5376 100644 +index 4a8156b..4a00a25 100644 --- a/src/create_update.c +++ b/src/create_update.c @@ -141,6 +141,7 @@ static bool parse_options(int argc, char **argv) @@ -329,19 +338,6 @@ index 766609b..74d5376 100644 } static int check_build_env(void) -diff --git a/src/fullfiles.c b/src/fullfiles.c -index 9fdfc2f..214b7b4 100644 ---- a/src/fullfiles.c -+++ b/src/fullfiles.c -@@ -136,6 +136,8 @@ static void create_fullfile(struct file *file) - } - done += curr; - } -+ close(fd); -+ fd = -1; - } - - for (int i = 0; compression_filters[i]; i++) { -- 2.1.4 diff --git a/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch b/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch index 1762d66..56ea17d 100644 --- a/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch +++ b/recipes-core/swupd-server/swupd-server/0002-add-logging-to-stdout.patch @@ -15,6 +15,8 @@ The implementation recycles the existing init_log_stdout() (not used before) and gives it the slightly different meaning of "also log to stdout". +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/72dd27a886ad9b2c66bb7cdb5c4cadb24f783654] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> --- src/create_update.c | 5 +++++ diff --git a/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch index 4a2eac8..440cca8 100644 --- a/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch +++ b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch @@ -6,6 +6,8 @@ Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs The fullfile .tar is needed for a pack as fallback when linking from the full rootfs isn't possible. In practice this shouldn't happen. +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> --- src/pack.c | 13 +++++++++++++ diff --git a/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch index 6cb2c57..7e981e3 100644 --- a/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch +++ b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch @@ -17,6 +17,8 @@ found. This speeds up delta computation considerably in Ostro OS because it skips the slow downloading of the old file. +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> --- src/delta.c | 13 +++++++++++++ diff --git a/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch index 063c701..6e221b9 100644 --- a/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch +++ b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch @@ -1,8 +1,8 @@ -From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001 +From d05c819ff83f04145dac355e183b426b5802d6b4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly <patrick.ohly@intel.com> Date: Tue, 8 Nov 2016 18:39:49 +0100 -Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for - diffing +Subject: [PATCH 09/12] swupd_create_pack: download original files on demand + for diffing The new, optional mode gets enabled via the --content-url parameter. When specified, swupd_create_pack will download the <original @@ -28,19 +28,22 @@ bsdtar. If for some reason a format change is necessary, delta computation becomes impossible and falls back to staging the full files. +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + --- include/swupd.h | 2 ++ - src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- - src/globals.c | 13 ++++++++ - src/make_packs.c | 8 +++++ - 4 files changed, 111 insertions(+), 2 deletions(-) + src/delta.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- + src/globals.c | 13 +++++++++ + src/make_packs.c | 8 ++++++ + 4 files changed, 110 insertions(+), 1 deletion(-) diff --git a/include/swupd.h b/include/swupd.h -index 31c7191..c1c0e96 100644 +index f9c0583..e1fa6ff 100644 --- a/include/swupd.h +++ b/include/swupd.h -@@ -147,12 +147,14 @@ extern char *state_dir; +@@ -149,12 +149,14 @@ extern char *state_dir; extern char *packstage_dir; extern char *image_dir; extern char *staging_dir; @@ -56,27 +59,25 @@ index 31c7191..c1c0e96 100644 extern void free_state_globals(void); diff --git a/src/delta.c b/src/delta.c -index 67e7df7..7e978b1 100644 +index ad40820..563abb9 100644 --- a/src/delta.c +++ b/src/delta.c -@@ -37,8 +37,14 @@ +@@ -37,7 +37,13 @@ - void __create_delta(struct file *file, int from_version) + void __create_delta(struct file *file, int from_version, char *from_hash) { -- char *original, *newfile, *outfile, *dotfile, *testnewfile; -- char *conf, *param1, *param2; -+ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; +- char *original, *newfile, *outfile, *dotfile, *testnewfile, *conf; ++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL, *conf = NULL; + char *tmpdir = NULL; + char *url = NULL; + char *cmd = NULL; -+ char *conf = NULL, *param1 = NULL, *param2 = NULL; + bool delete_original = false; + struct manifest *manifest = NULL; + GError *gerror = NULL; int ret; if (file->is_link) { -@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version) +@@ -57,6 +63,74 @@ void __create_delta(struct file *file, int from_version, char *from_hash) string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); @@ -151,7 +152,7 @@ index 67e7df7..7e978b1 100644 free(conf); conf = config_output_dir(); -@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version) +@@ -136,6 +210,18 @@ void __create_delta(struct file *file, int from_version, char *from_hash) LOG(NULL, "Failed to rename", ""); } out: diff --git a/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch b/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch index 707e1f0..43ae6ff 100644 --- a/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch +++ b/recipes-core/swupd-server/swupd-server/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch @@ -11,6 +11,8 @@ empty parameter to bsdtar, leading to: To allow the the "no parameter" case, it has to be argument list: that can be empty. If not empty, it has to end with a comma. +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/41] + Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> --- include/swupd.h | 4 ++-- diff --git a/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch b/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch index 0a68667..945e067 100644 --- a/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch +++ b/recipes-core/swupd-server/swupd-server/0027-update-control-over-parallelism.patch @@ -1,12 +1,14 @@ -From 8686189b3b080446fae732a85b72528b7fe68ba6 Mon Sep 17 00:00:00 2001 +From 5926a6ea8ca389ba11b467b231038b5bc5320a8b Mon Sep 17 00:00:00 2001 From: Patrick Ohly <patrick.ohly@intel.com> Date: Tue, 27 Sep 2016 08:25:40 +0200 -Subject: [PATCH 27/29] update control over parallelism +Subject: [PATCH 04/13] update control over parallelism The SWUPD_NUM_THREADS env variable is now understood by all three commands and overrides the default number of threads. Setting it to 1 is useful while debugging the code that runs inside threads (only one -thread hits breakpoints there). +thread hits breakpoints there). If SWUPD_NUM_THREADS is invalid, a +warning is printed and the variable gets ignored, i.e. the default +parallelism is used. The hard-coded parallelism of 12 threads when analysing the file system gets replaced with n, where n is the number of available CPUs. The default @@ -15,27 +17,40 @@ is the same as before elsewhere (n for packing, 3 * n for fullfiles). Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/4e0fdd4193a8ce9dcf3cfc5e488dfd4b23b7e7d9] Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + --- - src/analyze_fs.c | 5 ++++- - src/fullfiles.c | 7 +++++-- - src/pack.c | 7 +++++-- - 3 files changed, 14 insertions(+), 5 deletions(-) + include/swupd.h | 1 + + src/analyze_fs.c | 3 ++- + src/fullfiles.c | 5 +++-- + src/helpers.c | 26 ++++++++++++++++++++++++++ + src/pack.c | 5 +++-- + 5 files changed, 35 insertions(+), 5 deletions(-) +diff --git a/include/swupd.h b/include/swupd.h +index f9c0583..f334bdb 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -258,6 +258,7 @@ extern int system_argv(char *const argv[]); + extern int system_argv_fd(char *const argv[], int newstdin, int newstdout, int newstderr); + extern int system_argv_pipe(char *const argvp1[], int stdinp1, int stderrp1, + char *const argvp2[], int stdoutp2, int stderrp2); ++extern int num_threads(float scaling); + + extern bool signature_initialize(void); + extern void signature_terminate(void); diff --git a/src/analyze_fs.c b/src/analyze_fs.c -index 3bfb288..0f16343 100644 +index a220d79..ce30393 100644 --- a/src/analyze_fs.c +++ b/src/analyze_fs.c -@@ -387,6 +387,9 @@ struct manifest *full_manifest_from_directory(int version) +@@ -387,6 +387,7 @@ struct manifest *full_manifest_from_directory(int version) { struct manifest *manifest; char *dir; -+ int numthreads = getenv("SWUPD_NUM_THREADS") ? -+ atoi(getenv("SWUPD_NUM_THREADS")) : -+ sysconf(_SC_NPROCESSORS_ONLN); ++ int numthreads = num_threads(1.0); LOG(NULL, "Computing hashes", "for %i/full", version); -@@ -394,7 +397,7 @@ struct manifest *full_manifest_from_directory(int version) +@@ -394,7 +395,7 @@ struct manifest *full_manifest_from_directory(int version) string_or_die(&dir, "%s/%i/full", image_dir, version); @@ -45,16 +60,14 @@ index 3bfb288..0f16343 100644 iterate_directory(manifest, dir, "", true); diff --git a/src/fullfiles.c b/src/fullfiles.c -index 3be43d1..216a1d7 100644 +index ca8feda..882e83b 100644 --- a/src/fullfiles.c +++ b/src/fullfiles.c -@@ -291,10 +291,13 @@ static void submit_fullfile_tasks(GList *files) +@@ -291,10 +291,11 @@ static void submit_fullfile_tasks(GList *files) int ret; int count = 0; GError *err = NULL; -+ int numthreads = getenv("SWUPD_NUM_THREADS") ? -+ atoi(getenv("SWUPD_NUM_THREADS")) : -+ sysconf(_SC_NPROCESSORS_ONLN) * 3; ++ int numthreads = num_threads(3.0); - LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3); + LOG(NULL, "fullfile threadpool", "%d threads", numthreads); @@ -64,17 +77,49 @@ index 3be43d1..216a1d7 100644 TRUE, NULL); printf("Starting downloadable fullfiles data creation\n"); +diff --git a/src/helpers.c b/src/helpers.c +index 95d000c..c2ae10e 100644 +--- a/src/helpers.c ++++ b/src/helpers.c +@@ -284,3 +284,29 @@ void check_root(void) + exit(EXIT_FAILURE); + } + } ++ ++int num_threads(float scaling) ++{ ++ const char *var = getenv("SWUPD_NUM_THREADS"); ++ int result = sysconf(_SC_NPROCESSORS_ONLN) * scaling; ++ float scaled_result; ++ ++ if (var && *var) { ++ char *endptr; ++ long int value; ++ ++ errno = 0; ++ value = strtol(var, &endptr, 0); ++ ++ if ((errno != 0 && value == 0) || *endptr) { ++ LOG(NULL, "SWUPD_NUM_THREADS must be an integer", "%s", var); ++ } else if ((errno == ERANGE && (value == LONG_MAX || value == LONG_MIN)) || ++ value < 1 || value > INT_MAX) { ++ LOG(NULL, "SWUPD_NUM_THREADS out of range", "%s", var); ++ } else { ++ result = (int)value; ++ } ++ } ++ ++ return result; ++} diff --git a/src/pack.c b/src/pack.c -index 12d7443..3ffb88a 100644 +index 12d7443..2753e1f 100644 --- a/src/pack.c +++ b/src/pack.c -@@ -285,10 +285,13 @@ static void make_pack_deltas(GList *files) +@@ -285,10 +285,11 @@ static void make_pack_deltas(GList *files) struct file *file; int ret; GError *err = NULL; -+ int numthreads = getenv("SWUPD_NUM_THREADS") ? -+ atoi(getenv("SWUPD_NUM_THREADS")) : -+ sysconf(_SC_NPROCESSORS_ONLN); ++ int numthreads = num_threads(1.0); - LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN)); + LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); diff --git a/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch b/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch index d599261..d62cf7c 100644 --- a/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch +++ b/recipes-core/swupd-server/swupd-server/0029-fullfiles-use-libarchive-directly.patch @@ -1,7 +1,7 @@ -From 4ebc886b910fa4cb83864658069a30dd133caca8 Mon Sep 17 00:00:00 2001 +From b02a3ba1a61868db334d516f2577aa7aa7f07d6f Mon Sep 17 00:00:00 2001 From: Patrick Ohly <patrick.ohly@intel.com> Date: Wed, 30 Mar 2016 13:14:42 +0200 -Subject: [PATCH 3/6] fullfiles: use libarchive directly +Subject: [PATCH 06/13] fullfiles: use libarchive directly Calling an external tar command makes the code fairly complicated, because it is necessary to set up a suitable temporary directory with @@ -45,14 +45,14 @@ Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> Makefile.am | 6 +- configure.ac | 1 + include/libarchive_helper.h | 42 ++++++ - src/fullfiles.c | 324 +++++++++++++++++++++++++------------------- + src/fullfiles.c | 326 +++++++++++++++++++++++++------------------- src/in_memory_archive.c | 67 +++++++++ - 5 files changed, 300 insertions(+), 140 deletions(-) + 5 files changed, 302 insertions(+), 140 deletions(-) create mode 100644 include/libarchive_helper.h create mode 100644 src/in_memory_archive.c diff --git a/Makefile.am b/Makefile.am -index 528f02f..b14feb0 100644 +index 0ae2643..c1c8e07 100644 --- a/Makefile.am +++ b/Makefile.am @@ -23,6 +23,7 @@ swupd_create_update_SOURCES = \ @@ -63,15 +63,15 @@ index 528f02f..b14feb0 100644 src/manifest.c \ src/pack.c \ src/rename.c \ -@@ -54,6 +55,7 @@ swupd_make_fullfiles_SOURCES = \ - src/fullfiles.c \ +@@ -56,6 +57,7 @@ swupd_make_fullfiles_SOURCES = \ src/globals.c \ + src/groups.c \ src/helpers.c \ + src/in_memory_archive.c \ src/log.c \ src/make_fullfiles.c \ src/manifest.c \ -@@ -63,12 +65,13 @@ swupd_make_fullfiles_SOURCES = \ +@@ -65,12 +67,13 @@ swupd_make_fullfiles_SOURCES = \ src/stats.c \ src/xattrs.c @@ -86,7 +86,7 @@ index 528f02f..b14feb0 100644 $(bsdiff_LIBS) swupd_make_pack_LDADD = \ -@@ -81,6 +84,7 @@ swupd_make_fullfiles_LDADD = \ +@@ -83,6 +86,7 @@ swupd_make_fullfiles_LDADD = \ $(glib_LIBS) \ $(zlib_LIBS) \ $(openssl_LIBS) \ @@ -95,7 +95,7 @@ index 528f02f..b14feb0 100644 if ENABLE_LZMA diff --git a/configure.ac b/configure.ac -index 8819206..3e2d933 100644 +index 277146b..b4943b1 100644 --- a/configure.ac +++ b/configure.ac @@ -36,6 +36,7 @@ AC_ARG_ENABLE( @@ -155,7 +155,7 @@ index 0000000..ad28def + +#endif /* __INCLUDE_GUARD_LIBARCHIVE_HELPER_H */ diff --git a/src/fullfiles.c b/src/fullfiles.c -index 216a1d7..9fdfc2f 100644 +index 882e83b..e27d9ce 100644 --- a/src/fullfiles.c +++ b/src/fullfiles.c @@ -22,6 +22,8 @@ @@ -223,7 +223,7 @@ index 216a1d7..9fdfc2f 100644 string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename); if (lstat(origin, &sbuf) < 0) { -@@ -76,156 +83,195 @@ static void create_fullfile(struct file *file) +@@ -76,156 +83,197 @@ static void create_fullfile(struct file *file) assert(0); } @@ -349,6 +349,8 @@ index 216a1d7..9fdfc2f 100644 } + done += curr; } ++ close(fd); ++ fd = -1; + } - /* step 2a: tar it with each compression type */ diff --git a/recipes-core/swupd-server/swupd-server_3.2.5.bb b/recipes-core/swupd-server/swupd-server_3.2.5.bb new file mode 100644 index 0000000..632a11a --- /dev/null +++ b/recipes-core/swupd-server/swupd-server_3.2.5.bb @@ -0,0 +1,60 @@ +SUMMARY = "swupd sofware update from Clear Linux - server component" +HOMEPAGE = "https://github.com/clearlinux/swupd-server" +LICENSE = "GPL-2.0" +LIC_FILES_CHKSUM = "file://COPYING;md5=f8d90fb802930e30e49c39c8126a959e" + +DEPENDS = "file glib-2.0 rsync openssl libarchive bsdiff bzip2" +# Need the special "-replacement" variant because bzip2 and file +# are assumed to be provided and would not get built. +DEPENDS_append_class-native = " file-replacement-native bzip2-replacement-native" + +# This matches the SWUPD_TOOLS_FORMAT in swupd-image.bbclass. +# When updating to a new release which changes the format of +# the output, copy the recipe first to ensure that the old +# release is still available if needed by swupd-image.bbclass, +# then bump this number. +# +# The rest of the recipe ensures that different swupd-server +# versions can be build and installed in parallel (format +# number embedded in PN and the resulting files). +SWUPD_SERVER_FORMAT = "3" +PN = "swupd-server-format${SWUPD_SERVER_FORMAT}" +FILESEXTRAPATHS_prepend = "${THISDIR}/swupd-server-3.2.5:" +PV = "3.2.5+git${SRCPV}" +SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \ + file://0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch \ + file://0026-fullfiles.c-fix-invalid-LOG-call.patch \ + file://0027-update-control-over-parallelism.patch \ + file://0028-enable-locales-in-all-programs.patch \ + file://0029-fullfiles-use-libarchive-directly.patch \ + file://0001-swupd-create-update-alternative-input-layout.patch \ + file://0002-add-logging-to-stdout.patch \ + file://swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch \ + file://0001-delta.c-fix-xattr-test-after-patching.patch \ + file://0002-pack.c-do-not-clean-packstage.patch \ + file://0003-swupd_create_pack-download-original-files-on-demand-.patch \ + file://0001-create_pack-rely-less-on-previous-builds.patch \ + file://0002-create_pack-download-fullfile-on-demand-for-packs.patch \ + file://0003-create_pack-abort-delta-handling-early-when-impossib.patch \ + " +SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55" + +S = "${WORKDIR}/git" + +inherit pkgconfig autotools + +EXTRA_OECONF = "--enable-bzip2 --enable-lzma --disable-stateless --disable-tests --enable-bsdtar" + +# safer-calls-to-system-utilities.patch uses for loop initial declaration +CFLAGS_append = " -std=c99" + +RDEPENDS_${PN} = "rsync" +RDEPENDS_${PN}_class-target = " bsdtar" + +BBCLASSEXTEND = "native" + +do_install_append () { + for i in ${D}${bindir}/swupd_*; do + mv $i ${i}_${SWUPD_SERVER_FORMAT} + done +} diff --git a/recipes-core/swupd-server/swupd-server_git.bb b/recipes-core/swupd-server/swupd-server_git.bb index 5f93034..dbe7b19 100644 --- a/recipes-core/swupd-server/swupd-server_git.bb +++ b/recipes-core/swupd-server/swupd-server_git.bb @@ -8,7 +8,19 @@ DEPENDS = "file glib-2.0 rsync openssl libarchive bsdiff bzip2" # are assumed to be provided and would not get built. DEPENDS_append_class-native = " file-replacement-native bzip2-replacement-native" -PV = "3.2.5+git${SRCPV}" +# This matches the SWUPD_TOOLS_FORMAT in swupd-image.bbclass. +# When updating to a new release which changes the format of +# the output, copy the recipe first to ensure that the old +# release is still available if needed by swupd-image.bbclass, +# then bump this number. +# +# The rest of the recipe ensures that different swupd-server +# versions can be build and installed in parallel (format +# number embedded in PN and the resulting files). +SWUPD_SERVER_FORMAT = "4" +PN = "swupd-server-format${SWUPD_SERVER_FORMAT}" +FILESEXTRAPATHS_prepend = "${THISDIR}/swupd-server:" +PV = "3.3.0+git${SRCPV}" SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \ file://0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch \ file://0026-fullfiles.c-fix-invalid-LOG-call.patch \ @@ -17,15 +29,13 @@ SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \ file://0029-fullfiles-use-libarchive-directly.patch \ file://0001-swupd-create-update-alternative-input-layout.patch \ file://0002-add-logging-to-stdout.patch \ - file://swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch \ - file://0001-delta.c-fix-xattr-test-after-patching.patch \ file://0002-pack.c-do-not-clean-packstage.patch \ file://0003-swupd_create_pack-download-original-files-on-demand-.patch \ file://0001-create_pack-rely-less-on-previous-builds.patch \ file://0002-create_pack-download-fullfile-on-demand-for-packs.patch \ file://0003-create_pack-abort-delta-handling-early-when-impossib.patch \ " -SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55" +SRCREV = "9148ea8a8ecd73720d450d30acc4bceec310ab3b" S = "${WORKDIR}/git" @@ -40,3 +50,9 @@ RDEPENDS_${PN} = "rsync" RDEPENDS_${PN}_class-target = " bsdtar" BBCLASSEXTEND = "native" + +do_install_append () { + for i in ${D}${bindir}/swupd_*; do + mv $i ${i}_${SWUPD_SERVER_FORMAT} + done +} |