diff options
7 files changed, 607 insertions, 23 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass index dc1144d..783aca9 100644 --- a/classes/swupd-image.bbclass +++ b/classes/swupd-image.bbclass @@ -265,10 +265,6 @@ python do_fetch_swupd_inputs () { # Get information from remote update repo. swupd.bundles.download_old_versions(d) - # Stage locally cached information about previous builds - # (corresponds to the "archive the files of the current build" - # step in do_swupd_update). - swupd.bundles.copy_old_versions(d) } do_fetch_swupd_inputs[dirs] = "${SWUPDIMAGEDIR}" addtask do_fetch_swupd_inputs before do_swupd_update @@ -427,7 +423,12 @@ END for bndl in ${ALL_BUNDLES}; do bndlcnt=0 ${SWUPD_LOG_FN} "Generating delta pack from $prevver to ${OS_VERSION} for $bndl" - invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl + if [ "${SWUPD_CONTENT_URL}" ]; then + content_url_parameter="--content-url ${SWUPD_CONTENT_URL}" + else + content_url_parameter="" + fi + invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl done done @@ -437,11 +438,6 @@ END echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version # env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd - - # Archive the files of the current build which will be needed in the future - # for a <current version> -> <future version> delta computation. We exclude - # the expanded "full" rootfs, because we already have "full.tar". - (cd ${DEPLOY_DIR_SWUPD}; tar -zcf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}-${OS_VERSION}-swupd.tar --exclude=full --exclude=Manifest.*.tar image/${OS_VERSION} www/${OS_VERSION}/Manifest.*) } SWUPDDEPENDS = "\ diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py index 8a47a09..75321f1 100644 --- a/lib/swupd/bundles.py +++ b/lib/swupd/bundles.py @@ -152,19 +152,6 @@ def copy_bundle_contents(d): for bndl in bundles: stage_empty_bundle(d, bndl) -def copy_old_versions(d): - for prevver in d.getVar('SWUPD_DELTAPACK_VERSIONS', True).split(): - if not os.path.exists(os.path.join(d.expand('${DEPLOY_DIR_SWUPD}/image'), prevver)): - pattern = d.expand('${DEPLOY_DIR_IMAGE}/${IMAGE_BASENAME}*-%s-swupd.tar' % prevver) - prevver_tar = glob.glob(pattern) - if not prevver_tar or len(prevver_tar) > 1 or not os.path.exists(prevver_tar[0]): - bb.fatal("Creating swupd delta packs against %s is not possible because %s is not available." % - (prevver, pattern)) - cmd = ['tar', '-C', d.getVar('DEPLOY_DIR_SWUPD', True), '-xf', prevver_tar[0]] - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - if output: - bb.fatal('Unexpected output from the following command:\n%s\n%s' % (cmd, output)) - def download_manifests(content_url, version, component, to_dir): """ Download one manifest file and recursively all manifests referenced by it. diff --git a/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch new file mode 100644 index 0000000..a3a72ad --- /dev/null +++ b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch @@ -0,0 +1,255 @@ +From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 16 Nov 2016 14:26:30 +0100 +Subject: [PATCH 1/3] create_pack: rely less on previous builds + +When a file has not been modified in the current build, then by +definition the current copy of the file is the same as in the build +were it was last changed and thus it does not matter whether we use +<current build>/full/<file> or <last change>/full/<file>. But using +the current copy is better for a CI system which starts without local +access to older rootfs directories. It might also be a bit more +efficient (file access less scattered between different "full" +directories). + +Staging directories is better than staging .tar archives containing +those directories for the same reason (the .tar archive might not be +available in the CI system) and probably also improves efficiency (no +need to invoke bsdtar just to create a directory; impact not +measured). + +Also fix a slight flaw in the "target file exists already" handling: +when that occured for whatever reason (likely only during manual +debugging), the code would add the original fullfile .tar although it +isn't needed. Clearing the "ret" variable in that particular error +case avoids that. + +make_pack_full_files() and make_final_pack() used the exact same code +for populating the "staged" directory. Now that common code is in +stage_entry(). + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 2 +- + src/delta.c | 4 +-- + src/pack.c | 105 +++++++++++++++++++++++++++++++------------------------- + 3 files changed, 62 insertions(+), 49 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index c1c0e96..cf384e3 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest); + + extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); + extern void link_renames(GList *newfiles, struct manifest *from_manifest); +-extern void __create_delta(struct file *file, int from_version); ++extern void __create_delta(struct file *file, int from_version, int to_version); + + extern void account_delta_hit(void); + extern void account_delta_miss(void); +diff --git a/src/delta.c b/src/delta.c +index 7e978b1..8fff4c9 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -35,7 +35,7 @@ + #include "swupd.h" + #include "xattrs.h" + +-void __create_delta(struct file *file, int from_version) ++void __create_delta(struct file *file, int from_version, int to_version) + { + char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; + char *tmpdir = NULL; +@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version) + } + + conf = config_image_base(); +- string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename); ++ string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename); + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + +diff --git a/src/pack.c b/src/pack.c +index 984c2d6..ccb28bd 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -37,6 +37,7 @@ + #include <unistd.h> + + #include "swupd.h" ++#include "xattrs.h" + + static void empty_pack_stage(int full, int from_version, int to_version, char *module) + { +@@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack) + link_renames(pack->end_manifest->files, manifest); + } + ++static int stage_entry(struct file *file, ++ const char *fullfrom, const char *fullto, ++ const char *tarfrom, const char *tarto, ++ const char *packname) ++{ ++ int ret; ++ ++ /* Prefer to hardlink uncompressed files or replicate ++ * directories first, and fall back to the compressed ++ * versions if that failed. ++ */ ++ if (!file->is_dir) { ++ ret = link(fullfrom, fullto); ++ if (ret && errno == EEXIST) { ++ ret = 0; ++ } else if (ret) { ++ LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); ++ } ++ } else { ++ /* Replicate directory. */ ++ struct stat st; ++ if ((stat(fullfrom, &st) || ++ mkdir(fullto, 0) || ++ chmod(fullto, st.st_mode) || ++ chown(fullto, st.st_uid, st.st_gid) || ++ (xattrs_copy(fullfrom, fullto), false)) && ++ errno != EEXIST) { ++ LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); ++ rmdir(fullto); ++ ret = -1; ++ } else { ++ ret = 0; ++ } ++ } ++ ++ if (ret) { ++ ret = link(tarfrom, tarto); ++ if (ret && errno != EEXIST) { ++ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); ++ } ++ } ++ ++ return ret; ++} ++ + static void make_pack_full_files(struct packdata *pack) + { + GList *item; +@@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack) + char *fullfrom, *fullto; + + /* hardlink each file that is in <end> but not in <X> */ +- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); ++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); + string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash); + string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + +- ret = -1; +- errno = 0; +- +- /* Prefer to hardlink uncompressed files (excluding +- * directories) first, and fall back to the compressed +- * versions if the hardlink fails. ++ /* Prefer to hardlink uncompressed files or replicate ++ * directories first, and fall back to the compressed ++ * versions if that failed. + */ +- if (!file->is_dir) { +- ret = link(fullfrom, fullto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno); +- } +- } +- if (ret) { +- ret = link(from, to); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno); +- } +- } ++ ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile"); + + if (ret == 0) { + pack->fullcount++; +@@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack) + return files; + } + +-static void create_delta(gpointer data, __unused__ gpointer user_data) ++static void create_delta(gpointer data, gpointer user_data) + { + struct file *file = data; ++ int *to_version = user_data; + + /* if the file was not found in the from version, skip delta creation */ + if (file->peer) { +- __create_delta(file, file->peer->last_change); ++ __create_delta(file, file->peer->last_change, *to_version); + } + } + +-static void make_pack_deltas(GList *files) ++static void make_pack_deltas(GList *files, int to_version) + { + GThreadPool *threadpool; + GList *item; +@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files) + sysconf(_SC_NPROCESSORS_ONLN); + + LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); +- threadpool = g_thread_pool_new(create_delta, NULL, ++ threadpool = g_thread_pool_new(create_delta, &to_version, + numthreads, FALSE, NULL); + + item = g_list_first(files); +@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack) + file->last_change, file->hash); + string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, + pack->module, pack->from, pack->to, file->hash); +- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); ++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); + string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + +@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack) + } + } + } else { +- ret = -1; +- errno = 0; +- +- /* Prefer to hardlink uncompressed files (excluding +- * directories) first, and fall back to the compressed +- * versions if the hardlink fails. +- */ +- if (!file->is_dir) { +- ret = link(fullfrom, fullto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno); +- } +- } +- +- if (ret) { +- ret = link(tarfrom, tarto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno); +- } +- } +- ++ ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final"); + if (ret == 0) { + pack->fullcount++; + } +@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack) + + /* step 2: consolidate delta list & create all delta files*/ + delta_list = consolidate_packs_delta_files(delta_list, pack); +- make_pack_deltas(delta_list); ++ make_pack_deltas(delta_list, pack->to); + g_list_free(delta_list); + + /* step 3: complete pack creation */ +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch new file mode 100644 index 0000000..4a2eac8 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch @@ -0,0 +1,40 @@ +From c9e9fb971ab0494047b4d8c0e656e0a06ad9b236 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:39:49 +0100 +Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs + +The fullfile .tar is needed for a pack as fallback when linking from +the full rootfs isn't possible. In practice this shouldn't happen. + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/pack.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/src/pack.c b/src/pack.c +index ccb28bd..e331da2 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -187,6 +187,19 @@ static int stage_entry(struct file *file, + + if (ret) { + ret = link(tarfrom, tarto); ++ if (ret && errno == ENOENT && content_url) { ++ LOG(NULL, "Download fallback for pack", "%s: %s to %s", packname, tarfrom, tarto); ++ ++ /* Must be "tarfrom" that is missing. Download directly into target location.*/ ++ char *cmd; ++ string_or_die(&cmd, "curl -s -o '%s' %s/%d/files/%s.tar", ++ tarto, content_url, file->last_change, file->hash); ++ if (system(cmd)) { ++ LOG(file, "Downloading failed", "%s", cmd); ++ } else { ++ ret = 0; ++ } ++ } + if (ret && errno != EEXIST) { + LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); + } +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch new file mode 100644 index 0000000..6cb2c57 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch @@ -0,0 +1,58 @@ +From 88fd362785ae8871537573fd7073e499542b0a0d Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Thu, 17 Nov 2016 10:09:27 +0100 +Subject: [PATCH 3/3] create_pack: abort delta handling early when impossible + due to IMA + +Currently, deltas cannot be computed for systems using IMA because IMA +relies on a security.ima xattr which changes each time the file +content changes and swupd doesn't support deltas for files with +different xattrs. + +It would be worthwhile to add such a support, but that's a bit +complicated (needs to be done in client and server and implies a +format change), so for now just abort early when a security.xattr is +found. + +This speeds up delta computation considerably in Ostro OS because it +skips the slow downloading of the old file. + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/delta.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/src/delta.c b/src/delta.c +index 8fff4c9..d3c4b35 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -30,6 +30,7 @@ + #include <string.h> + #include <sys/stat.h> + #include <sys/types.h> ++#include <attr/xattr.h> + #include <unistd.h> + + #include "swupd.h" +@@ -64,6 +65,18 @@ void __create_delta(struct file *file, int from_version, int to_version) + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + ++ if (lgetxattr(newfile, "security.ima", NULL, 0) > 0) { ++ /* There is a non-empty security.ima xattr on the new file. ++ * That xattr contains a hash of the file content. We know that ++ * the file content has changed, so the xattr will be different ++ * from the one on the old file and we can bail out early without ++ * even bothering with retrieving the original file. A better ++ * solution for systems with IMA would be to support deltas even ++ * when xattrs are different. ++ */ ++ goto out; ++ } ++ + if (access(original, F_OK) && + content_url) { + /* File does not exist. Try to get it from the online update repo instead. +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch new file mode 100644 index 0000000..063c701 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch @@ -0,0 +1,244 @@ +From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:39:49 +0100 +Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for + diffing + +The new, optional mode gets enabled via the --content-url parameter. +When specified, swupd_create_pack will download the <original +version>/files/<hash>.tar file from the update server if the required +file is missing. + +This mode is meant for use with meta-swupd where new CI builds start +with an empty "image" directory. NFS access to the full, expanded +"image" directories of the previous build is not possible in such a +setup (would require root for special file attributes). + +Downloading and unpacking image directories of previous builds would +be possible, but is expected to be slower (when downloading all files, +not just those needed for diffing) or more complex (when deciding about +required files outside of swupd_create_pack). + +The downside of this new approach is that it relies on not changing +the content or naming of the tar files. For example, switching from +GNU tar to bsdtar would not work. But that's not an issue for the +intended usage in Ostro OS, which already made the switch to +bsdtar. + +If for some reason a format change is necessary, delta computation +becomes impossible and falls back to staging the full files. + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 2 ++ + src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- + src/globals.c | 13 ++++++++ + src/make_packs.c | 8 +++++ + 4 files changed, 111 insertions(+), 2 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index 31c7191..c1c0e96 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -147,12 +147,14 @@ extern char *state_dir; + extern char *packstage_dir; + extern char *image_dir; + extern char *staging_dir; ++extern char *content_url; + + extern bool init_globals(void); + extern void free_globals(void); + extern bool set_format(char *); + extern void check_root(void); + extern bool set_state_dir(char *); ++extern bool set_content_url(const char *); + extern bool init_state_globals(void); + extern void free_state_globals(void); + +diff --git a/src/delta.c b/src/delta.c +index 67e7df7..7e978b1 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -37,8 +37,14 @@ + + void __create_delta(struct file *file, int from_version) + { +- char *original, *newfile, *outfile, *dotfile, *testnewfile; +- char *conf, *param1, *param2; ++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; ++ char *tmpdir = NULL; ++ char *url = NULL; ++ char *cmd = NULL; ++ char *conf = NULL, *param1 = NULL, *param2 = NULL; ++ bool delete_original = false; ++ struct manifest *manifest = NULL; ++ GError *gerror = NULL; + int ret; + + if (file->is_link) { +@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version) + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + ++ if (access(original, F_OK) && ++ content_url) { ++ /* File does not exist. Try to get it from the online update repo instead. ++ * This fallback is meant to be used for CI builds which start with no local ++ * state and only HTTP(S) access to the published www directory. ++ * Not being able to retrieve the file is not an error and will merely ++ * prevent computing the delta. ++ */ ++ string_or_die(&tmpdir, "%s/make-pack-tmpdir-XXXXXX", state_dir); ++ tmpdir = g_dir_make_tmp("make-pack-XXXXXX", &gerror); ++ if (!tmpdir) { ++ LOG(NULL, "Failed to create temporary directory for untarring original file", "%s", ++ gerror->message); ++ assert(0); ++ } ++ /* Determine hash of original file in the corresponding Manifest. */ ++ manifest = manifest_from_file(from_version, "full"); ++ if (!manifest) { ++ LOG(NULL, "Failed to read full Manifest", "version %d, cannot retrieve original file", ++ from_version); ++ goto out; ++ } ++ const char *last_hash = NULL; ++ GList *list = g_list_first(manifest->files); ++ while (list) { ++ struct file *original_file = list->data; ++ if (!strcmp(file->filename, original_file->filename)) { ++ last_hash = original_file->hash; ++ break; ++ } ++ list = g_list_next(list); ++ } ++ if (!last_hash) { ++ LOG(NULL, "Original file not found", "%s in full manifest for %d - inconsistent update data?!", ++ file->filename, from_version); ++ goto out; ++ } ++ ++ /* We use a temporary copy because we don't want to ++ * tamper with the original "full" folder which ++ * probably does not even exist. Using a temporary file ++ * file implies re-downloading in the future, but that's ++ * consistent with the intended usage in a CI environment ++ * which always starts from scratch. ++ */ ++ free(original); ++ string_or_die(&original, "%s/%s", tmpdir, last_hash); ++ delete_original = true; ++ ++ /* ++ * This is a proof-of-concept. A real implementation should use ++ * a combination of libcurl + libarchive calls to unpack the files. ++ * For current Ostro OS, deltas despite xattr differences would ++ * be needed, otherwise this code here is of little use (all ++ * modified files fail the xattr sameness check, because security.ima ++ * changes when file content changes). ++ */ ++ string_or_die(&url, "%s/%d/files/%s.tar", content_url, from_version, last_hash); ++ LOG(file, "Downloading original file", "%s to %s", url, original); ++ ++ /* bsdtar can detect compression when reading from stdin, GNU tar can't. */ ++ string_or_die(&cmd, "curl -s %s | bsdtar -C %s -xf -", url, tmpdir); ++ if (system(cmd)) { ++ LOG(file, "Downloading/unpacking failed, skipping delta", "%s", url); ++ goto out; ++ } ++ } ++ + free(conf); + + conf = config_output_dir(); +@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version) + LOG(NULL, "Failed to rename", ""); + } + out: ++ if (delete_original) { ++ unlink(original); ++ } ++ if (tmpdir) { ++ rmdir(tmpdir); ++ g_free(tmpdir); ++ } ++ if (manifest) { ++ free_manifest(manifest); ++ } ++ g_clear_error(&gerror); ++ free(cmd); + free(testnewfile); + free(conf); + free(newfile); +diff --git a/src/globals.c b/src/globals.c +index 74758ce..0e047e2 100644 +--- a/src/globals.c ++++ b/src/globals.c +@@ -40,6 +40,7 @@ char *state_dir = NULL; + char *packstage_dir = NULL; + char *image_dir = NULL; + char *staging_dir = NULL; ++char *content_url = NULL; + + bool set_format(char *userinput) + { +@@ -76,6 +77,17 @@ bool set_state_dir(char *dir) + return true; + } + ++bool set_content_url(const char *url) ++{ ++ if (content_url) { ++ free(content_url); ++ } ++ string_or_die(&content_url, "%s", url); ++ ++ return true; ++} ++ ++ + bool init_globals(void) + { + if (format == 0) { +@@ -123,4 +135,5 @@ void free_state_globals(void) + free(packstage_dir); + free(image_dir); + free(staging_dir); ++ free(content_url); + } +diff --git a/src/make_packs.c b/src/make_packs.c +index 2d3e25e..827be56 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -49,6 +49,7 @@ static const struct option prog_opts[] = { + { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { "signcontent", no_argument, 0, 's' }, ++ { "content-url", required_argument, 0, 'u' }, + { 0, 0, 0, 0 } + }; + +@@ -61,6 +62,7 @@ static void usage(const char *name) + printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); ++ printf(" -u, --content-url Base URL of the update repo (optional, used to retrieve missing files on demand"); + printf("\n"); + } + +@@ -86,6 +88,12 @@ static bool parse_options(int argc, char **argv) + case 's': + enable_signing = true; + break; ++ case 'u': ++ if (!optarg || !set_content_url(optarg)) { ++ printf("Invalid --content-url argument ''%s''\n\n", optarg); ++ return false; ++ } ++ break; + } + } + +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server_git.bb b/recipes-core/swupd-server/swupd-server_git.bb index 2ee8642..2c00843 100644 --- a/recipes-core/swupd-server/swupd-server_git.bb +++ b/recipes-core/swupd-server/swupd-server_git.bb @@ -18,6 +18,10 @@ SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \ file://swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch \ file://0001-delta.c-fix-xattr-test-after-patching.patch \ file://0002-pack.c-do-not-clean-packstage.patch \ + file://0003-swupd_create_pack-download-original-files-on-demand-.patch \ + file://0001-create_pack-rely-less-on-previous-builds.patch \ + file://0002-create_pack-download-fullfile-on-demand-for-packs.patch \ + file://0003-create_pack-abort-delta-handling-early-when-impossib.patch \ " SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55" |