aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-11-08 21:26:03 +0100
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:12:56 +0100
commitddde21be154d12f37ac22613851a86625d270907 (patch)
treef446fc4b62aa1ebbdc07c27dced46949c576c874
parentf4279241025a28873e1ec9741861129d6c521ce2 (diff)
downloadmeta-swupd-ddde21be154d12f37ac22613851a86625d270907.tar.gz
meta-swupd-ddde21be154d12f37ac22613851a86625d270907.tar.bz2
meta-swupd-ddde21be154d12f37ac22613851a86625d270907.zip
swupd_create_pack: enable delta computation
The previous approaches all relied on somehow carrying additional data across from one build to the next (sstate or additional archives in the deploy directory). The new approach replaces that with downloading required content on a file-by-file basis from the normal update repo when (and not sooner) it is needed by swupd_create_pack. That works for meta-swupd because the format of the files (compressed archive created with bsdtar) is expected to be stable. If a change ever becomes necessary, some backward compatibilty mode would have to be added or deltas simply would be skipped again. Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--classes/swupd-image.bbclass16
-rw-r--r--lib/swupd/bundles.py13
-rw-r--r--recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch255
-rw-r--r--recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch40
-rw-r--r--recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch58
-rw-r--r--recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch244
-rw-r--r--recipes-core/swupd-server/swupd-server_git.bb4
7 files changed, 607 insertions, 23 deletions
diff --git a/classes/swupd-image.bbclass b/classes/swupd-image.bbclass
index dc1144d..783aca9 100644
--- a/classes/swupd-image.bbclass
+++ b/classes/swupd-image.bbclass
@@ -265,10 +265,6 @@ python do_fetch_swupd_inputs () {
# Get information from remote update repo.
swupd.bundles.download_old_versions(d)
- # Stage locally cached information about previous builds
- # (corresponds to the "archive the files of the current build"
- # step in do_swupd_update).
- swupd.bundles.copy_old_versions(d)
}
do_fetch_swupd_inputs[dirs] = "${SWUPDIMAGEDIR}"
addtask do_fetch_swupd_inputs before do_swupd_update
@@ -427,7 +423,12 @@ END
for bndl in ${ALL_BUNDLES}; do
bndlcnt=0
${SWUPD_LOG_FN} "Generating delta pack from $prevver to ${OS_VERSION} for $bndl"
- invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl
+ if [ "${SWUPD_CONTENT_URL}" ]; then
+ content_url_parameter="--content-url ${SWUPD_CONTENT_URL}"
+ else
+ content_url_parameter=""
+ fi
+ invoke_swupd ${STAGING_BINDIR_NATIVE}/swupd_make_pack --log-stdout $content_url_parameter -S ${DEPLOY_DIR_SWUPD} $prevver ${OS_VERSION} $bndl
done
done
@@ -437,11 +438,6 @@ END
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/www/version/format${SWUPD_FORMAT}/latest
echo ${OS_VERSION} > ${DEPLOY_DIR_SWUPD}/image/latest.version
# env $PSEUDO bsdtar -acf ${DEPLOY_DIR}/swupd-done.tar.gz -C ${DEPLOY_DIR} swupd
-
- # Archive the files of the current build which will be needed in the future
- # for a <current version> -> <future version> delta computation. We exclude
- # the expanded "full" rootfs, because we already have "full.tar".
- (cd ${DEPLOY_DIR_SWUPD}; tar -zcf ${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}-${OS_VERSION}-swupd.tar --exclude=full --exclude=Manifest.*.tar image/${OS_VERSION} www/${OS_VERSION}/Manifest.*)
}
SWUPDDEPENDS = "\
diff --git a/lib/swupd/bundles.py b/lib/swupd/bundles.py
index 8a47a09..75321f1 100644
--- a/lib/swupd/bundles.py
+++ b/lib/swupd/bundles.py
@@ -152,19 +152,6 @@ def copy_bundle_contents(d):
for bndl in bundles:
stage_empty_bundle(d, bndl)
-def copy_old_versions(d):
- for prevver in d.getVar('SWUPD_DELTAPACK_VERSIONS', True).split():
- if not os.path.exists(os.path.join(d.expand('${DEPLOY_DIR_SWUPD}/image'), prevver)):
- pattern = d.expand('${DEPLOY_DIR_IMAGE}/${IMAGE_BASENAME}*-%s-swupd.tar' % prevver)
- prevver_tar = glob.glob(pattern)
- if not prevver_tar or len(prevver_tar) > 1 or not os.path.exists(prevver_tar[0]):
- bb.fatal("Creating swupd delta packs against %s is not possible because %s is not available." %
- (prevver, pattern))
- cmd = ['tar', '-C', d.getVar('DEPLOY_DIR_SWUPD', True), '-xf', prevver_tar[0]]
- output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
- if output:
- bb.fatal('Unexpected output from the following command:\n%s\n%s' % (cmd, output))
-
def download_manifests(content_url, version, component, to_dir):
"""
Download one manifest file and recursively all manifests referenced by it.
diff --git a/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch
new file mode 100644
index 0000000..a3a72ad
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch
@@ -0,0 +1,255 @@
+From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Wed, 16 Nov 2016 14:26:30 +0100
+Subject: [PATCH 1/3] create_pack: rely less on previous builds
+
+When a file has not been modified in the current build, then by
+definition the current copy of the file is the same as in the build
+were it was last changed and thus it does not matter whether we use
+<current build>/full/<file> or <last change>/full/<file>. But using
+the current copy is better for a CI system which starts without local
+access to older rootfs directories. It might also be a bit more
+efficient (file access less scattered between different "full"
+directories).
+
+Staging directories is better than staging .tar archives containing
+those directories for the same reason (the .tar archive might not be
+available in the CI system) and probably also improves efficiency (no
+need to invoke bsdtar just to create a directory; impact not
+measured).
+
+Also fix a slight flaw in the "target file exists already" handling:
+when that occured for whatever reason (likely only during manual
+debugging), the code would add the original fullfile .tar although it
+isn't needed. Clearing the "ret" variable in that particular error
+case avoids that.
+
+make_pack_full_files() and make_final_pack() used the exact same code
+for populating the "staged" directory. Now that common code is in
+stage_entry().
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ include/swupd.h | 2 +-
+ src/delta.c | 4 +--
+ src/pack.c | 105 +++++++++++++++++++++++++++++++-------------------------
+ 3 files changed, 62 insertions(+), 49 deletions(-)
+
+diff --git a/include/swupd.h b/include/swupd.h
+index c1c0e96..cf384e3 100644
+--- a/include/swupd.h
++++ b/include/swupd.h
+@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest);
+
+ extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
+ extern void link_renames(GList *newfiles, struct manifest *from_manifest);
+-extern void __create_delta(struct file *file, int from_version);
++extern void __create_delta(struct file *file, int from_version, int to_version);
+
+ extern void account_delta_hit(void);
+ extern void account_delta_miss(void);
+diff --git a/src/delta.c b/src/delta.c
+index 7e978b1..8fff4c9 100644
+--- a/src/delta.c
++++ b/src/delta.c
+@@ -35,7 +35,7 @@
+ #include "swupd.h"
+ #include "xattrs.h"
+
+-void __create_delta(struct file *file, int from_version)
++void __create_delta(struct file *file, int from_version, int to_version)
+ {
+ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL;
+ char *tmpdir = NULL;
+@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version)
+ }
+
+ conf = config_image_base();
+- string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename);
++ string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename);
+
+ string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
+
+diff --git a/src/pack.c b/src/pack.c
+index 984c2d6..ccb28bd 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -37,6 +37,7 @@
+ #include <unistd.h>
+
+ #include "swupd.h"
++#include "xattrs.h"
+
+ static void empty_pack_stage(int full, int from_version, int to_version, char *module)
+ {
+@@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack)
+ link_renames(pack->end_manifest->files, manifest);
+ }
+
++static int stage_entry(struct file *file,
++ const char *fullfrom, const char *fullto,
++ const char *tarfrom, const char *tarto,
++ const char *packname)
++{
++ int ret;
++
++ /* Prefer to hardlink uncompressed files or replicate
++ * directories first, and fall back to the compressed
++ * versions if that failed.
++ */
++ if (!file->is_dir) {
++ ret = link(fullfrom, fullto);
++ if (ret && errno == EEXIST) {
++ ret = 0;
++ } else if (ret) {
++ LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno);
++ }
++ } else {
++ /* Replicate directory. */
++ struct stat st;
++ if ((stat(fullfrom, &st) ||
++ mkdir(fullto, 0) ||
++ chmod(fullto, st.st_mode) ||
++ chown(fullto, st.st_uid, st.st_gid) ||
++ (xattrs_copy(fullfrom, fullto), false)) &&
++ errno != EEXIST) {
++ LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno);
++ rmdir(fullto);
++ ret = -1;
++ } else {
++ ret = 0;
++ }
++ }
++
++ if (ret) {
++ ret = link(tarfrom, tarto);
++ if (ret && errno != EEXIST) {
++ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno);
++ }
++ }
++
++ return ret;
++}
++
+ static void make_pack_full_files(struct packdata *pack)
+ {
+ GList *item;
+@@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack)
+ char *fullfrom, *fullto;
+
+ /* hardlink each file that is in <end> but not in <X> */
+- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename);
+ string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+ string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash);
+ string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+
+- ret = -1;
+- errno = 0;
+-
+- /* Prefer to hardlink uncompressed files (excluding
+- * directories) first, and fall back to the compressed
+- * versions if the hardlink fails.
++ /* Prefer to hardlink uncompressed files or replicate
++ * directories first, and fall back to the compressed
++ * versions if that failed.
+ */
+- if (!file->is_dir) {
+- ret = link(fullfrom, fullto);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno);
+- }
+- }
+- if (ret) {
+- ret = link(from, to);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno);
+- }
+- }
++ ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile");
+
+ if (ret == 0) {
+ pack->fullcount++;
+@@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
+ return files;
+ }
+
+-static void create_delta(gpointer data, __unused__ gpointer user_data)
++static void create_delta(gpointer data, gpointer user_data)
+ {
+ struct file *file = data;
++ int *to_version = user_data;
+
+ /* if the file was not found in the from version, skip delta creation */
+ if (file->peer) {
+- __create_delta(file, file->peer->last_change);
++ __create_delta(file, file->peer->last_change, *to_version);
+ }
+ }
+
+-static void make_pack_deltas(GList *files)
++static void make_pack_deltas(GList *files, int to_version)
+ {
+ GThreadPool *threadpool;
+ GList *item;
+@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files)
+ sysconf(_SC_NPROCESSORS_ONLN);
+
+ LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
+- threadpool = g_thread_pool_new(create_delta, NULL,
++ threadpool = g_thread_pool_new(create_delta, &to_version,
+ numthreads, FALSE, NULL);
+
+ item = g_list_first(files);
+@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack)
+ file->last_change, file->hash);
+ string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename);
+ string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
+ pack->module, pack->from, pack->to, file->hash);
+
+@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack)
+ }
+ }
+ } else {
+- ret = -1;
+- errno = 0;
+-
+- /* Prefer to hardlink uncompressed files (excluding
+- * directories) first, and fall back to the compressed
+- * versions if the hardlink fails.
+- */
+- if (!file->is_dir) {
+- ret = link(fullfrom, fullto);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno);
+- }
+- }
+-
+- if (ret) {
+- ret = link(tarfrom, tarto);
+- if (ret && errno != EEXIST) {
+- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno);
+- }
+- }
+-
++ ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final");
+ if (ret == 0) {
+ pack->fullcount++;
+ }
+@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack)
+
+ /* step 2: consolidate delta list & create all delta files*/
+ delta_list = consolidate_packs_delta_files(delta_list, pack);
+- make_pack_deltas(delta_list);
++ make_pack_deltas(delta_list, pack->to);
+ g_list_free(delta_list);
+
+ /* step 3: complete pack creation */
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch
new file mode 100644
index 0000000..4a2eac8
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server/0002-create_pack-download-fullfile-on-demand-for-packs.patch
@@ -0,0 +1,40 @@
+From c9e9fb971ab0494047b4d8c0e656e0a06ad9b236 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 8 Nov 2016 18:39:49 +0100
+Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs
+
+The fullfile .tar is needed for a pack as fallback when linking from
+the full rootfs isn't possible. In practice this shouldn't happen.
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/pack.c | 13 +++++++++++++
+ 1 file changed, 13 insertions(+)
+
+diff --git a/src/pack.c b/src/pack.c
+index ccb28bd..e331da2 100644
+--- a/src/pack.c
++++ b/src/pack.c
+@@ -187,6 +187,19 @@ static int stage_entry(struct file *file,
+
+ if (ret) {
+ ret = link(tarfrom, tarto);
++ if (ret && errno == ENOENT && content_url) {
++ LOG(NULL, "Download fallback for pack", "%s: %s to %s", packname, tarfrom, tarto);
++
++ /* Must be "tarfrom" that is missing. Download directly into target location.*/
++ char *cmd;
++ string_or_die(&cmd, "curl -s -o '%s' %s/%d/files/%s.tar",
++ tarto, content_url, file->last_change, file->hash);
++ if (system(cmd)) {
++ LOG(file, "Downloading failed", "%s", cmd);
++ } else {
++ ret = 0;
++ }
++ }
+ if (ret && errno != EEXIST) {
+ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno);
+ }
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch
new file mode 100644
index 0000000..6cb2c57
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server/0003-create_pack-abort-delta-handling-early-when-impossib.patch
@@ -0,0 +1,58 @@
+From 88fd362785ae8871537573fd7073e499542b0a0d Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Thu, 17 Nov 2016 10:09:27 +0100
+Subject: [PATCH 3/3] create_pack: abort delta handling early when impossible
+ due to IMA
+
+Currently, deltas cannot be computed for systems using IMA because IMA
+relies on a security.ima xattr which changes each time the file
+content changes and swupd doesn't support deltas for files with
+different xattrs.
+
+It would be worthwhile to add such a support, but that's a bit
+complicated (needs to be done in client and server and implies a
+format change), so for now just abort early when a security.xattr is
+found.
+
+This speeds up delta computation considerably in Ostro OS because it
+skips the slow downloading of the old file.
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/delta.c | 13 +++++++++++++
+ 1 file changed, 13 insertions(+)
+
+diff --git a/src/delta.c b/src/delta.c
+index 8fff4c9..d3c4b35 100644
+--- a/src/delta.c
++++ b/src/delta.c
+@@ -30,6 +30,7 @@
+ #include <string.h>
+ #include <sys/stat.h>
+ #include <sys/types.h>
++#include <attr/xattr.h>
+ #include <unistd.h>
+
+ #include "swupd.h"
+@@ -64,6 +65,18 @@ void __create_delta(struct file *file, int from_version, int to_version)
+
+ string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
+
++ if (lgetxattr(newfile, "security.ima", NULL, 0) > 0) {
++ /* There is a non-empty security.ima xattr on the new file.
++ * That xattr contains a hash of the file content. We know that
++ * the file content has changed, so the xattr will be different
++ * from the one on the old file and we can bail out early without
++ * even bothering with retrieving the original file. A better
++ * solution for systems with IMA would be to support deltas even
++ * when xattrs are different.
++ */
++ goto out;
++ }
++
+ if (access(original, F_OK) &&
+ content_url) {
+ /* File does not exist. Try to get it from the online update repo instead.
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch
new file mode 100644
index 0000000..063c701
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server/0003-swupd_create_pack-download-original-files-on-demand-.patch
@@ -0,0 +1,244 @@
+From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Tue, 8 Nov 2016 18:39:49 +0100
+Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for
+ diffing
+
+The new, optional mode gets enabled via the --content-url parameter.
+When specified, swupd_create_pack will download the <original
+version>/files/<hash>.tar file from the update server if the required
+file is missing.
+
+This mode is meant for use with meta-swupd where new CI builds start
+with an empty "image" directory. NFS access to the full, expanded
+"image" directories of the previous build is not possible in such a
+setup (would require root for special file attributes).
+
+Downloading and unpacking image directories of previous builds would
+be possible, but is expected to be slower (when downloading all files,
+not just those needed for diffing) or more complex (when deciding about
+required files outside of swupd_create_pack).
+
+The downside of this new approach is that it relies on not changing
+the content or naming of the tar files. For example, switching from
+GNU tar to bsdtar would not work. But that's not an issue for the
+intended usage in Ostro OS, which already made the switch to
+bsdtar.
+
+If for some reason a format change is necessary, delta computation
+becomes impossible and falls back to staging the full files.
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ include/swupd.h | 2 ++
+ src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--
+ src/globals.c | 13 ++++++++
+ src/make_packs.c | 8 +++++
+ 4 files changed, 111 insertions(+), 2 deletions(-)
+
+diff --git a/include/swupd.h b/include/swupd.h
+index 31c7191..c1c0e96 100644
+--- a/include/swupd.h
++++ b/include/swupd.h
+@@ -147,12 +147,14 @@ extern char *state_dir;
+ extern char *packstage_dir;
+ extern char *image_dir;
+ extern char *staging_dir;
++extern char *content_url;
+
+ extern bool init_globals(void);
+ extern void free_globals(void);
+ extern bool set_format(char *);
+ extern void check_root(void);
+ extern bool set_state_dir(char *);
++extern bool set_content_url(const char *);
+ extern bool init_state_globals(void);
+ extern void free_state_globals(void);
+
+diff --git a/src/delta.c b/src/delta.c
+index 67e7df7..7e978b1 100644
+--- a/src/delta.c
++++ b/src/delta.c
+@@ -37,8 +37,14 @@
+
+ void __create_delta(struct file *file, int from_version)
+ {
+- char *original, *newfile, *outfile, *dotfile, *testnewfile;
+- char *conf, *param1, *param2;
++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL;
++ char *tmpdir = NULL;
++ char *url = NULL;
++ char *cmd = NULL;
++ char *conf = NULL, *param1 = NULL, *param2 = NULL;
++ bool delete_original = false;
++ struct manifest *manifest = NULL;
++ GError *gerror = NULL;
+ int ret;
+
+ if (file->is_link) {
+@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version)
+
+ string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
+
++ if (access(original, F_OK) &&
++ content_url) {
++ /* File does not exist. Try to get it from the online update repo instead.
++ * This fallback is meant to be used for CI builds which start with no local
++ * state and only HTTP(S) access to the published www directory.
++ * Not being able to retrieve the file is not an error and will merely
++ * prevent computing the delta.
++ */
++ string_or_die(&tmpdir, "%s/make-pack-tmpdir-XXXXXX", state_dir);
++ tmpdir = g_dir_make_tmp("make-pack-XXXXXX", &gerror);
++ if (!tmpdir) {
++ LOG(NULL, "Failed to create temporary directory for untarring original file", "%s",
++ gerror->message);
++ assert(0);
++ }
++ /* Determine hash of original file in the corresponding Manifest. */
++ manifest = manifest_from_file(from_version, "full");
++ if (!manifest) {
++ LOG(NULL, "Failed to read full Manifest", "version %d, cannot retrieve original file",
++ from_version);
++ goto out;
++ }
++ const char *last_hash = NULL;
++ GList *list = g_list_first(manifest->files);
++ while (list) {
++ struct file *original_file = list->data;
++ if (!strcmp(file->filename, original_file->filename)) {
++ last_hash = original_file->hash;
++ break;
++ }
++ list = g_list_next(list);
++ }
++ if (!last_hash) {
++ LOG(NULL, "Original file not found", "%s in full manifest for %d - inconsistent update data?!",
++ file->filename, from_version);
++ goto out;
++ }
++
++ /* We use a temporary copy because we don't want to
++ * tamper with the original "full" folder which
++ * probably does not even exist. Using a temporary file
++ * file implies re-downloading in the future, but that's
++ * consistent with the intended usage in a CI environment
++ * which always starts from scratch.
++ */
++ free(original);
++ string_or_die(&original, "%s/%s", tmpdir, last_hash);
++ delete_original = true;
++
++ /*
++ * This is a proof-of-concept. A real implementation should use
++ * a combination of libcurl + libarchive calls to unpack the files.
++ * For current Ostro OS, deltas despite xattr differences would
++ * be needed, otherwise this code here is of little use (all
++ * modified files fail the xattr sameness check, because security.ima
++ * changes when file content changes).
++ */
++ string_or_die(&url, "%s/%d/files/%s.tar", content_url, from_version, last_hash);
++ LOG(file, "Downloading original file", "%s to %s", url, original);
++
++ /* bsdtar can detect compression when reading from stdin, GNU tar can't. */
++ string_or_die(&cmd, "curl -s %s | bsdtar -C %s -xf -", url, tmpdir);
++ if (system(cmd)) {
++ LOG(file, "Downloading/unpacking failed, skipping delta", "%s", url);
++ goto out;
++ }
++ }
++
+ free(conf);
+
+ conf = config_output_dir();
+@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version)
+ LOG(NULL, "Failed to rename", "");
+ }
+ out:
++ if (delete_original) {
++ unlink(original);
++ }
++ if (tmpdir) {
++ rmdir(tmpdir);
++ g_free(tmpdir);
++ }
++ if (manifest) {
++ free_manifest(manifest);
++ }
++ g_clear_error(&gerror);
++ free(cmd);
+ free(testnewfile);
+ free(conf);
+ free(newfile);
+diff --git a/src/globals.c b/src/globals.c
+index 74758ce..0e047e2 100644
+--- a/src/globals.c
++++ b/src/globals.c
+@@ -40,6 +40,7 @@ char *state_dir = NULL;
+ char *packstage_dir = NULL;
+ char *image_dir = NULL;
+ char *staging_dir = NULL;
++char *content_url = NULL;
+
+ bool set_format(char *userinput)
+ {
+@@ -76,6 +77,17 @@ bool set_state_dir(char *dir)
+ return true;
+ }
+
++bool set_content_url(const char *url)
++{
++ if (content_url) {
++ free(content_url);
++ }
++ string_or_die(&content_url, "%s", url);
++
++ return true;
++}
++
++
+ bool init_globals(void)
+ {
+ if (format == 0) {
+@@ -123,4 +135,5 @@ void free_state_globals(void)
+ free(packstage_dir);
+ free(image_dir);
+ free(staging_dir);
++ free(content_url);
+ }
+diff --git a/src/make_packs.c b/src/make_packs.c
+index 2d3e25e..827be56 100644
+--- a/src/make_packs.c
++++ b/src/make_packs.c
+@@ -49,6 +49,7 @@ static const struct option prog_opts[] = {
+ { "log-stdout", no_argument, 0, 'l' },
+ { "statedir", required_argument, 0, 'S' },
+ { "signcontent", no_argument, 0, 's' },
++ { "content-url", required_argument, 0, 'u' },
+ { 0, 0, 0, 0 }
+ };
+
+@@ -61,6 +62,7 @@ static void usage(const char *name)
+ printf(" -l, --log-stdout Write log messages also to stdout\n");
+ printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR);
+ printf(" -s, --signcontent Enables cryptographic signing of update content\n");
++ printf(" -u, --content-url Base URL of the update repo (optional, used to retrieve missing files on demand");
+ printf("\n");
+ }
+
+@@ -86,6 +88,12 @@ static bool parse_options(int argc, char **argv)
+ case 's':
+ enable_signing = true;
+ break;
++ case 'u':
++ if (!optarg || !set_content_url(optarg)) {
++ printf("Invalid --content-url argument ''%s''\n\n", optarg);
++ return false;
++ }
++ break;
+ }
+ }
+
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server_git.bb b/recipes-core/swupd-server/swupd-server_git.bb
index 2ee8642..2c00843 100644
--- a/recipes-core/swupd-server/swupd-server_git.bb
+++ b/recipes-core/swupd-server/swupd-server_git.bb
@@ -18,6 +18,10 @@ SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \
file://swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch \
file://0001-delta.c-fix-xattr-test-after-patching.patch \
file://0002-pack.c-do-not-clean-packstage.patch \
+ file://0003-swupd_create_pack-download-original-files-on-demand-.patch \
+ file://0001-create_pack-rely-less-on-previous-builds.patch \
+ file://0002-create_pack-download-fullfile-on-demand-for-packs.patch \
+ file://0003-create_pack-abort-delta-handling-early-when-impossib.patch \
"
SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55"