diff options
Diffstat (limited to 'recipes-core/swupd-server/swupd-server-3.2.5')
14 files changed, 2161 insertions, 0 deletions
diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch new file mode 100644 index 0000000..fd91b78 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-create_pack-rely-less-on-previous-builds.patch @@ -0,0 +1,257 @@ +From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 16 Nov 2016 14:26:30 +0100 +Subject: [PATCH 1/3] create_pack: rely less on previous builds + +When a file has not been modified in the current build, then by +definition the current copy of the file is the same as in the build +were it was last changed and thus it does not matter whether we use +<current build>/full/<file> or <last change>/full/<file>. But using +the current copy is better for a CI system which starts without local +access to older rootfs directories. It might also be a bit more +efficient (file access less scattered between different "full" +directories). + +Staging directories is better than staging .tar archives containing +those directories for the same reason (the .tar archive might not be +available in the CI system) and probably also improves efficiency (no +need to invoke bsdtar just to create a directory; impact not +measured). + +Also fix a slight flaw in the "target file exists already" handling: +when that occured for whatever reason (likely only during manual +debugging), the code would add the original fullfile .tar although it +isn't needed. Clearing the "ret" variable in that particular error +case avoids that. + +make_pack_full_files() and make_final_pack() used the exact same code +for populating the "staged" directory. Now that common code is in +stage_entry(). + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 2 +- + src/delta.c | 4 +-- + src/pack.c | 105 +++++++++++++++++++++++++++++++------------------------- + 3 files changed, 62 insertions(+), 49 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index c1c0e96..cf384e3 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest); + + extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); + extern void link_renames(GList *newfiles, struct manifest *from_manifest); +-extern void __create_delta(struct file *file, int from_version); ++extern void __create_delta(struct file *file, int from_version, int to_version); + + extern void account_delta_hit(void); + extern void account_delta_miss(void); +diff --git a/src/delta.c b/src/delta.c +index 7e978b1..8fff4c9 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -35,7 +35,7 @@ + #include "swupd.h" + #include "xattrs.h" + +-void __create_delta(struct file *file, int from_version) ++void __create_delta(struct file *file, int from_version, int to_version) + { + char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; + char *tmpdir = NULL; +@@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version) + } + + conf = config_image_base(); +- string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename); ++ string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename); + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + +diff --git a/src/pack.c b/src/pack.c +index 984c2d6..ccb28bd 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -37,6 +37,7 @@ + #include <unistd.h> + + #include "swupd.h" ++#include "xattrs.h" + + static void empty_pack_stage(int full, int from_version, int to_version, char *module) + { +@@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack) + link_renames(pack->end_manifest->files, manifest); + } + ++static int stage_entry(struct file *file, ++ const char *fullfrom, const char *fullto, ++ const char *tarfrom, const char *tarto, ++ const char *packname) ++{ ++ int ret; ++ ++ /* Prefer to hardlink uncompressed files or replicate ++ * directories first, and fall back to the compressed ++ * versions if that failed. ++ */ ++ if (!file->is_dir) { ++ ret = link(fullfrom, fullto); ++ if (ret && errno == EEXIST) { ++ ret = 0; ++ } else if (ret) { ++ LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); ++ } ++ } else { ++ /* Replicate directory. */ ++ struct stat st; ++ if ((stat(fullfrom, &st) || ++ mkdir(fullto, 0) || ++ chmod(fullto, st.st_mode) || ++ chown(fullto, st.st_uid, st.st_gid) || ++ (xattrs_copy(fullfrom, fullto), false)) && ++ errno != EEXIST) { ++ LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); ++ rmdir(fullto); ++ ret = -1; ++ } else { ++ ret = 0; ++ } ++ } ++ ++ if (ret) { ++ ret = link(tarfrom, tarto); ++ if (ret && errno != EEXIST) { ++ LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); ++ } ++ } ++ ++ return ret; ++} ++ + static void make_pack_full_files(struct packdata *pack) + { + GList *item; +@@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack) + char *fullfrom, *fullto; + + /* hardlink each file that is in <end> but not in <X> */ +- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); ++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); + string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash); + string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + +- ret = -1; +- errno = 0; +- +- /* Prefer to hardlink uncompressed files (excluding +- * directories) first, and fall back to the compressed +- * versions if the hardlink fails. ++ /* Prefer to hardlink uncompressed files or replicate ++ * directories first, and fall back to the compressed ++ * versions if that failed. + */ +- if (!file->is_dir) { +- ret = link(fullfrom, fullto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno); +- } +- } +- if (ret) { +- ret = link(from, to); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno); +- } +- } ++ ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile"); + + if (ret == 0) { + pack->fullcount++; +@@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack) + return files; + } + +-static void create_delta(gpointer data, __unused__ gpointer user_data) ++static void create_delta(gpointer data, gpointer user_data) + { + struct file *file = data; ++ int *to_version = user_data; + + /* if the file was not found in the from version, skip delta creation */ + if (file->peer) { +- __create_delta(file, file->peer->last_change); ++ __create_delta(file, file->peer->last_change, *to_version); + } + } + +-static void make_pack_deltas(GList *files) ++static void make_pack_deltas(GList *files, int to_version) + { + GThreadPool *threadpool; + GList *item; +@@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files) + sysconf(_SC_NPROCESSORS_ONLN); + + LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); +- threadpool = g_thread_pool_new(create_delta, NULL, ++ threadpool = g_thread_pool_new(create_delta, &to_version, + numthreads, FALSE, NULL); + + item = g_list_first(files); +@@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack) + file->last_change, file->hash); + string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, + pack->module, pack->from, pack->to, file->hash); +- string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); ++ string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); + string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, + pack->module, pack->from, pack->to, file->hash); + +@@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack) + } + } + } else { +- ret = -1; +- errno = 0; +- +- /* Prefer to hardlink uncompressed files (excluding +- * directories) first, and fall back to the compressed +- * versions if the hardlink fails. +- */ +- if (!file->is_dir) { +- ret = link(fullfrom, fullto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno); +- } +- } +- +- if (ret) { +- ret = link(tarfrom, tarto); +- if (ret && errno != EEXIST) { +- LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno); +- } +- } +- ++ ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final"); + if (ret == 0) { + pack->fullcount++; + } +@@ -539,7 +552,7 @@ int make_pack(struct packdata *pack) + + /* step 2: consolidate delta list & create all delta files*/ + delta_list = consolidate_packs_delta_files(delta_list, pack); +- make_pack_deltas(delta_list); ++ make_pack_deltas(delta_list, pack->to); + g_list_free(delta_list); + + /* step 3: complete pack creation */ +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch new file mode 100644 index 0000000..78fb85a --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-delta.c-fix-xattr-test-after-patching.patch @@ -0,0 +1,36 @@ +From 048808fc3d88e1f9b7daed87f582ef35cf1dce22 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:32:00 +0100 +Subject: [PATCH 1/3] delta.c: fix xattr test after patching + +At the moment, swupd_create_pack fails when some files have xattrs and +get patched because the xattrs of the test file do not match the +original, unpatched file. + +That's because xattrs_copy() was applied to the wrong target file. + +Fixes: swupd-server/#35 + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/37] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/delta.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/delta.c b/src/delta.c +index 3ddb13a..67e7df7 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -98,7 +98,7 @@ void __create_delta(struct file *file, int from_version) + ret = 0; + goto out; + } +- xattrs_copy(original, newfile); ++ xattrs_copy(original, testnewfile); + + /* does xattrs have been correctly copied?*/ + if (xattrs_compare(original, testnewfile) != 0) { +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch new file mode 100644 index 0000000..5920afc --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0001-swupd-create-update-alternative-input-layout.patch @@ -0,0 +1,349 @@ +From e1f0d54a940eb7d04e2fbd59bd995a819331425f Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Fri, 30 Sep 2016 08:42:08 +0200 +Subject: [PATCH 1/2] swupd-create-update: alternative input layout + +In Ostro OS, we already have a "full" directory with all files. +Splitting it up into bundles just so that swupd-create-update can +reconstruct the "full" directory is a waste of IO, and noticably slow +when run under pseudo. + +To streamline the required work, a new layout for the "image" input +directory gets introduced: +- The "full" directory gets created by the caller before invoking + swupd-create-update. +- For each bundle, instead of a <bundle> directory, there is a + <bundle>.content.txt file, listing all entries (including directories) + of the bundle. + +The traditional mode of operation still works as before because each operation +which normally works with a bundle directory checks whether there is such a +directory and if not, switches to the new mode. + +That way it is even possible to mix the two modes, i.e. replacing only +some bundles with a content list, although that's probably not all +that useful. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/55] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/analyze_fs.c | 171 ++++++++++++++++++++++++++++++++++++++-------------- + src/chroot.c | 24 +++++--- + src/create_update.c | 6 +- + src/fullfiles.c | 2 + + 4 files changed, 150 insertions(+), 53 deletions(-) + +diff --git a/src/analyze_fs.c b/src/analyze_fs.c +index 0f16343..d534587 100644 +--- a/src/analyze_fs.c ++++ b/src/analyze_fs.c +@@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data) + /* disallow characters which can do unexpected things when the filename is + * used on a tar command line via system("tar [args] filename [more args]"); + */ +-static bool illegal_characters(char *filename) ++static bool illegal_characters(const char *filename) + { + char c; + int i; +@@ -301,25 +301,145 @@ static bool illegal_characters(char *filename) + return false; + } + ++static void add_file(struct manifest *manifest, ++ const char *entry_name, ++ char *sub_filename, ++ char *fullname, ++ bool do_hash) ++{ ++ GError *err = NULL; ++ struct file *file; ++ ++ if (illegal_characters(entry_name)) { ++ printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename); ++ free(sub_filename); ++ free(fullname); ++ return; ++ } ++ ++ file = calloc(1, sizeof(struct file)); ++ assert(file); ++ ++ file->last_change = manifest->version; ++ file->filename = sub_filename; ++ ++ populate_file_struct(file, fullname); ++ if (file->is_deleted) { ++ /* ++ * populate_file_struct() logs a stat() failure, but ++ * does not abort. When adding files that should ++ * exist, this case is an error. ++ */ ++ LOG(NULL, "file not found", "%s", fullname); ++ assert(0); ++ } ++ ++ ++ /* if for some reason there is a file in the official build ++ * which should not be included in the Manifest, then open a bug ++ * to get it removed, and work around its presence by ++ * excluding it here, eg: ++ if (strncmp(file->filename, "/dev/", 5) == 0) { ++ continue; ++ } ++ */ ++ ++ if (do_hash) { ++ /* compute the hash from a thread */ ++ int ret; ++ ret = g_thread_pool_push(threadpool, file, &err); ++ if (ret == FALSE) { ++ printf("GThread hash computation push error\n"); ++ printf("%s\n", err->message); ++ assert(0); ++ } ++ } ++ manifest->files = g_list_prepend(manifest->files, file); ++ manifest->count++; ++} ++ ++ + static void iterate_directory(struct manifest *manifest, char *pathprefix, + char *subpath, bool do_hash) + { + DIR *dir; + struct dirent *entry; + char *fullpath; +- int ret; +- GError *err = NULL; + + string_or_die(&fullpath, "%s/%s", pathprefix, subpath); + + dir = opendir(fullpath); + if (!dir) { ++ FILE *content; ++ int len; ++ free(fullpath); ++ if (errno != ENOENT) { ++ return; ++ } ++ /* ++ * If there is a <dir>.content.txt instead of ++ * the actual directory, then read that ++ * file. It has a list of path names, ++ * including all directories. The ++ * corresponding file system entry is then ++ * expected to be in a pre-populated "full" ++ * directory. ++ */ ++ if (subpath[0]) { ++ string_or_die(&fullpath, "%s/%s.content.txt", pathprefix, len, subpath); ++ } else { ++ string_or_die(&fullpath, "%s.content.txt", pathprefix); ++ } ++ content = fopen(fullpath, "r"); + free(fullpath); ++ fullpath = NULL; ++ if (content) { ++ char *line = NULL; ++ size_t len = 0; ++ ssize_t read; ++ const char *full; ++ int full_len; ++ /* ++ * determine path to "full" directory: it is assumed to be alongside ++ * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit, ++ * so we have to strip the last path component. ++ */ ++ full = strrchr(pathprefix, '/'); ++ if (full) { ++ full_len = full - pathprefix + 1; ++ full = pathprefix; ++ } else { ++ full = ""; ++ full_len = 0; ++ } ++ while ((read = getline(&line, &len, content)) != -1) { ++ if (read) { ++ const char *entry_name = strrchr(line, '/'); ++ if (entry_name) { ++ entry_name++; ++ } else { ++ entry_name = line; ++ } ++ if (line[read - 1] == '\n') { ++ line[read - 1] = 0; ++ } ++ string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line); ++ add_file(manifest, ++ entry_name, ++ strdup(line), ++ fullpath, ++ do_hash); ++ } ++ } ++ free(line); ++ } ++ ++ // If both directory and content file are missing, silently (?) ++ // don't add anything to the manifest. + return; + } + + while (dir) { +- struct file *file; + char *sub_filename; + char *fullname; + +@@ -334,50 +454,13 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix, + } + + string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name); +- +- if (illegal_characters(entry->d_name)) { +- printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename); +- free(sub_filename); +- continue; +- } +- +- file = calloc(1, sizeof(struct file)); +- if (!file) { +- break; +- } +- +- file->last_change = manifest->version; +- file->filename = sub_filename; +- + string_or_die(&fullname, "%s/%s", fullpath, entry->d_name); +- populate_file_struct(file, fullname); +- free(fullname); +- + if (entry->d_type == DT_DIR) { +- iterate_directory(manifest, pathprefix, file->filename, do_hash); ++ iterate_directory(manifest, pathprefix, sub_filename, do_hash); + } ++ /* takes ownership of the strings */ ++ add_file(manifest, entry->d_name, sub_filename, fullname, do_hash); + +- /* if for some reason there is a file in the official build +- * which should not be included in the Manifest, then open a bug +- * to get it removed, and work around its presence by +- * excluding it here, eg: +- if (strncmp(file->filename, "/dev/", 5) == 0) { +- continue; +- } +- */ +- +- if (do_hash) { +- /* compute the hash from a thread */ +- ret = g_thread_pool_push(threadpool, file, &err); +- if (ret == FALSE) { +- printf("GThread hash computation push error\n"); +- printf("%s\n", err->message); +- closedir(dir); +- return; +- } +- } +- manifest->files = g_list_prepend(manifest->files, file); +- manifest->count++; + } + closedir(dir); + free(fullpath); +diff --git a/src/chroot.c b/src/chroot.c +index 32ed997..f3832e1 100644 +--- a/src/chroot.c ++++ b/src/chroot.c +@@ -39,15 +39,21 @@ void chroot_create_full(int newversion) + char *full_dir; + + string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion); ++ if (!access(full_dir, R_OK|X_OK)) { ++ free(full_dir); ++ return; ++ } + + g_mkdir_with_parents(full_dir, S_IRWXU); + + /* start with base */ +- LOG(NULL, "Copying chroot os-core to full", ""); + string_or_die(¶m, "%s/%i/os-core/", image_dir, newversion); +- char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL }; +- if (system_argv(rsynccmd) != 0) { +- assert(0); ++ if (!access(param, F_OK)) { ++ LOG(NULL, "Copying chroot os-core to full", ""); ++ char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL }; ++ if (system_argv(rsynccmd) != 0) { ++ assert(0); ++ } + } + free(param); + +@@ -58,11 +64,13 @@ void chroot_create_full(int newversion) + break; + } + +- LOG(NULL, "Overlaying bundle chroot onto full", "%s", group); + string_or_die(¶m, "%s/%i/%s/", image_dir, newversion, group); +- char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL }; +- if (system_argv(rsynccmd) != 0) { +- assert(0); ++ if (!access(param, F_OK)) { ++ LOG(NULL, "Overlaying bundle chroot onto full", "%s", group); ++ char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL }; ++ if (system_argv(rsynccmd) != 0) { ++ assert(0); ++ } + } + free(param); + } +diff --git a/src/create_update.c b/src/create_update.c +index 766609b..74d5376 100644 +--- a/src/create_update.c ++++ b/src/create_update.c +@@ -141,6 +141,7 @@ static bool parse_options(int argc, char **argv) + static void populate_dirs(int version) + { + char *newversiondir; ++ char *newversiondircontent = NULL; + + string_or_die(&newversiondir, "%s/%d", image_dir, version); + +@@ -182,9 +183,11 @@ static void populate_dirs(int version) + } + + string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group); ++ string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group); + + /* Create the bundle directory(s) as needed */ +- if (access(newversiondir, F_OK | R_OK) != 0) { ++ if (access(newversiondir, F_OK | R_OK) != 0 && ++ access(newversiondircontent, F_OK | R_OK) != 0) { + printf("%s does not exist...creating\n", group); + if (mkdir(newversiondir, 0755) != 0) { + printf("Failed to create %s subdirectory\n", group); +@@ -193,6 +196,7 @@ static void populate_dirs(int version) + } + } + free(newversiondir); ++ free(newversiondircontent); + } + + static int check_build_env(void) +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 9fdfc2f..214b7b4 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -136,6 +136,8 @@ static void create_fullfile(struct file *file) + } + done += curr; + } ++ close(fd); ++ fd = -1; + } + + for (int i = 0; compression_filters[i]; i++) { +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch new file mode 100644 index 0000000..56ea17d --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-add-logging-to-stdout.patch @@ -0,0 +1,200 @@ +From 2e3eb8abcdef496d0ce30d03da7befcb9978aceb Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Sat, 1 Oct 2016 13:51:02 +0200 +Subject: [PATCH 2/2] add logging to stdout + +When a CI system (like the one from Ostro) captures the output of +commands, but not necessarily intermediate log files, then it is +useful to also log to stdout. Another use case is calling the tools +interactively during development. + +The new --log-stdout option in all three commands enables logging to +stdout in addition to the traditional log files. + +The implementation recycles the existing init_log_stdout() (not used +before) and gives it the slightly different meaning of "also log to +stdout". + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/72dd27a886ad9b2c66bb7cdb5c4cadb24f783654] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/create_update.c | 5 +++++ + src/log.c | 27 +++++++++++++++++---------- + src/make_fullfiles.c | 5 +++++ + src/make_packs.c | 5 +++++ + 4 files changed, 32 insertions(+), 10 deletions(-) + +diff --git a/src/create_update.c b/src/create_update.c +index 74d5376..f1c840d 100644 +--- a/src/create_update.c ++++ b/src/create_update.c +@@ -50,6 +50,7 @@ static void banner(void) + static const struct option prog_opts[] = { + { "help", no_argument, 0, 'h' }, + { "version", no_argument, 0, 'v' }, ++ { "log-stdout", no_argument, 0, 'l' }, + { "osversion", required_argument, 0, 'o' }, + { "minversion", required_argument, 0, 'm' }, + { "format", required_argument, 0, 'F' }, +@@ -68,6 +69,7 @@ static void print_help(const char *name) + printf(" -v, --version Show software version\n"); + printf("\n"); + printf("Application Options:\n"); ++ printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -o, --osversion The OS version for which to create an update\n"); + printf(" -m, --minversion Optional minimum file version to write into manifests per file\n"); + printf(" -F, --format Format number for the update\n"); +@@ -87,6 +89,9 @@ static bool parse_options(int argc, char **argv) + case 'h': + print_help(argv[0]); + return false; ++ case 'l': ++ init_log_stdout(); ++ break; + case 'v': + banner(); + return false; +diff --git a/src/log.c b/src/log.c +index e8bf9c3..45a4d66 100644 +--- a/src/log.c ++++ b/src/log.c +@@ -33,7 +33,7 @@ + + #include "swupd.h" + +-static FILE *logfile; ++static FILE *logfile[2]; + + static struct timeval start_time; + +@@ -41,13 +41,13 @@ void init_log(const char *prefix, const char *bundle, int start, int end) + { + char *filename; + string_or_die(&filename, "%s%s-from-%i-to-%i.log", prefix, bundle, start, end); +- logfile = fopen(filename, "w"); ++ logfile[0] = fopen(filename, "w"); + free(filename); + gettimeofday(&start_time, NULL); + } + void init_log_stdout(void) + { +- logfile = stdout; ++ logfile[1] = stdout; + gettimeofday(&start_time, NULL); + } + +@@ -91,8 +91,9 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con + char *logstring = NULL; + char filebuf[4096]; + char filebuf2[4096]; ++ int i; + +- if (!logfile) { ++ if (!logfile[0] && !logfile[1]) { + return; + } + +@@ -119,12 +120,16 @@ void __log_message(struct file *file, char *msg, char *filename, int linenr, con + strcat(filebuf2, " "); + } + +- fprintf(logfile, "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n", +- (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf); ++ for (i = 0; i < 2; i++) { ++ if (logfile[i]) { ++ fprintf(logfile[i], "%3i.%03i %5s %s:%03i\t| %s\t| %s\t| %s\n", ++ (int)current_time.tv_sec, (int)current_time.tv_usec / 1000, logstring, filebuf, linenr, filebuf2, msg, buf); ++ fflush(logfile[i]); ++ } ++ } + + free(logstring); + free(buf); +- fflush(logfile); + } + + void close_log(int version, int exit_status) +@@ -133,7 +138,7 @@ void close_log(int version, int exit_status) + int t_sec; + int t_msec; + +- if (!logfile) { ++ if (!logfile[0] && !logfile[1]) { + return; + } + +@@ -159,6 +164,8 @@ void close_log(int version, int exit_status) + printf("Update build failed for version %i\n", version); + } + +- fclose(logfile); +- logfile = NULL; ++ if (logfile[0]) { ++ fclose(logfile[0]); ++ logfile[0] = NULL; ++ } + } +diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c +index 2a1e2e9..0216e91 100644 +--- a/src/make_fullfiles.c ++++ b/src/make_fullfiles.c +@@ -33,6 +33,7 @@ + + static const struct option prog_opts[] = { + { "help", no_argument, 0, 'h' }, ++ { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { 0, 0, 0, 0 } + }; +@@ -43,6 +44,7 @@ static void usage(const char *name) + printf(" %s <version>\n\n", name); + printf("Help options:\n"); + printf(" -h, --help Show help options\n"); ++ printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf("\n"); + } +@@ -57,6 +59,9 @@ static bool parse_options(int argc, char **argv) + case 'h': + usage(argv[0]); + return false; ++ case 'l': ++ init_log_stdout(); ++ break; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument '%s'\n\n", optarg); +diff --git a/src/make_packs.c b/src/make_packs.c +index 8560b3f..2d3e25e 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -46,6 +46,7 @@ static void banner(void) + + static const struct option prog_opts[] = { + { "help", no_argument, 0, 'h' }, ++ { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { "signcontent", no_argument, 0, 's' }, + { 0, 0, 0, 0 } +@@ -57,6 +58,7 @@ static void usage(const char *name) + printf(" %s <start version> <latest version> <bundle>\n\n", name); + printf("Help options:\n"); + printf(" -h, --help Show help options\n"); ++ printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); + printf("\n"); +@@ -72,6 +74,9 @@ static bool parse_options(int argc, char **argv) + case 'h': + usage(argv[0]); + return false; ++ case 'l': ++ init_log_stdout(); ++ break; + case 'S': + if (!optarg || !set_state_dir(optarg)) { + printf("Invalid --statedir argument ''%s'\n\n", optarg); +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch new file mode 100644 index 0000000..440cca8 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-create_pack-download-fullfile-on-demand-for-packs.patch @@ -0,0 +1,42 @@ +From c9e9fb971ab0494047b4d8c0e656e0a06ad9b236 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:39:49 +0100 +Subject: [PATCH 2/3] create_pack: download fullfile on demand for packs + +The fullfile .tar is needed for a pack as fallback when linking from +the full rootfs isn't possible. In practice this shouldn't happen. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/pack.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/src/pack.c b/src/pack.c +index ccb28bd..e331da2 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -187,6 +187,19 @@ static int stage_entry(struct file *file, + + if (ret) { + ret = link(tarfrom, tarto); ++ if (ret && errno == ENOENT && content_url) { ++ LOG(NULL, "Download fallback for pack", "%s: %s to %s", packname, tarfrom, tarto); ++ ++ /* Must be "tarfrom" that is missing. Download directly into target location.*/ ++ char *cmd; ++ string_or_die(&cmd, "curl -s -o '%s' %s/%d/files/%s.tar", ++ tarto, content_url, file->last_change, file->hash); ++ if (system(cmd)) { ++ LOG(file, "Downloading failed", "%s", cmd); ++ } else { ++ ret = 0; ++ } ++ } + if (ret && errno != EEXIST) { + LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); + } +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch new file mode 100644 index 0000000..4cc3af4 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0002-pack.c-do-not-clean-packstage.patch @@ -0,0 +1,38 @@ +From 7b9e1bc78b704a2fb9610dddc6f0925bb6412f55 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:34:44 +0100 +Subject: [PATCH 2/3] pack.c: do not clean packstage + +This works around a bug in pseudo +(https://bugzilla.yoctoproject.org/show_bug.cgi?id=10623) where +unlinking the linked entries in the staging area also removes the +xattrs of the original files in the full directory tree. + +This works as long as the swupd_create_pack command does not get +called again without cleaning the packstage. + +Upstream-Status: Inappropriate [workaround] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/pack.c | 4 +++- + 1 file changed, 3 insertions(+), 1 deletion(-) + +diff --git a/src/pack.c b/src/pack.c +index 46c7c68..984c2d6 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -46,7 +46,9 @@ static void empty_pack_stage(int full, int from_version, int to_version, char *m + // clean any stale data (eg: re-run after a failure) + string_or_die(¶m, "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version); + char *const rmcmd[] = { "rm", "-fr", param, NULL }; +- if (system_argv(rmcmd) != 0) { ++ if (true) { ++ LOG(NULL, "Skipping removal of packstage", "%s/%s/%i_to_%i/", packstage_dir, module, from_version, to_version); ++ } else if (system_argv(rmcmd) != 0) { + fprintf(stderr, "Failed to clean %s/%s/%i_to_%i\n", + packstage_dir, module, from_version, to_version); + free(param); +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch new file mode 100644 index 0000000..7e981e3 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0003-create_pack-abort-delta-handling-early-when-impossib.patch @@ -0,0 +1,60 @@ +From 88fd362785ae8871537573fd7073e499542b0a0d Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Thu, 17 Nov 2016 10:09:27 +0100 +Subject: [PATCH 3/3] create_pack: abort delta handling early when impossible + due to IMA + +Currently, deltas cannot be computed for systems using IMA because IMA +relies on a security.ima xattr which changes each time the file +content changes and swupd doesn't support deltas for files with +different xattrs. + +It would be worthwhile to add such a support, but that's a bit +complicated (needs to be done in client and server and implies a +format change), so for now just abort early when a security.xattr is +found. + +This speeds up delta computation considerably in Ostro OS because it +skips the slow downloading of the old file. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/delta.c | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/src/delta.c b/src/delta.c +index 8fff4c9..d3c4b35 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -30,6 +30,7 @@ + #include <string.h> + #include <sys/stat.h> + #include <sys/types.h> ++#include <attr/xattr.h> + #include <unistd.h> + + #include "swupd.h" +@@ -64,6 +65,18 @@ void __create_delta(struct file *file, int from_version, int to_version) + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + ++ if (lgetxattr(newfile, "security.ima", NULL, 0) > 0) { ++ /* There is a non-empty security.ima xattr on the new file. ++ * That xattr contains a hash of the file content. We know that ++ * the file content has changed, so the xattr will be different ++ * from the one on the old file and we can bail out early without ++ * even bothering with retrieving the original file. A better ++ * solution for systems with IMA would be to support deltas even ++ * when xattrs are different. ++ */ ++ goto out; ++ } ++ + if (access(original, F_OK) && + content_url) { + /* File does not exist. Try to get it from the online update repo instead. +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch new file mode 100644 index 0000000..3a71d9c --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0003-swupd_create_pack-download-original-files-on-demand-.patch @@ -0,0 +1,246 @@ +From ee076ebeb041b725e40041e77d8f368f866f3216 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 8 Nov 2016 18:39:49 +0100 +Subject: [PATCH 3/3] swupd_create_pack: download original files on demand for + diffing + +The new, optional mode gets enabled via the --content-url parameter. +When specified, swupd_create_pack will download the <original +version>/files/<hash>.tar file from the update server if the required +file is missing. + +This mode is meant for use with meta-swupd where new CI builds start +with an empty "image" directory. NFS access to the full, expanded +"image" directories of the previous build is not possible in such a +setup (would require root for special file attributes). + +Downloading and unpacking image directories of previous builds would +be possible, but is expected to be slower (when downloading all files, +not just those needed for diffing) or more complex (when deciding about +required files outside of swupd_create_pack). + +The downside of this new approach is that it relies on not changing +the content or naming of the tar files. For example, switching from +GNU tar to bsdtar would not work. But that's not an issue for the +intended usage in Ostro OS, which already made the switch to +bsdtar. + +If for some reason a format change is necessary, delta computation +becomes impossible and falls back to staging the full files. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 2 ++ + src/delta.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- + src/globals.c | 13 ++++++++ + src/make_packs.c | 8 +++++ + 4 files changed, 111 insertions(+), 2 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index 31c7191..c1c0e96 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -147,12 +147,14 @@ extern char *state_dir; + extern char *packstage_dir; + extern char *image_dir; + extern char *staging_dir; ++extern char *content_url; + + extern bool init_globals(void); + extern void free_globals(void); + extern bool set_format(char *); + extern void check_root(void); + extern bool set_state_dir(char *); ++extern bool set_content_url(const char *); + extern bool init_state_globals(void); + extern void free_state_globals(void); + +diff --git a/src/delta.c b/src/delta.c +index 67e7df7..7e978b1 100644 +--- a/src/delta.c ++++ b/src/delta.c +@@ -37,8 +37,14 @@ + + void __create_delta(struct file *file, int from_version) + { +- char *original, *newfile, *outfile, *dotfile, *testnewfile; +- char *conf, *param1, *param2; ++ char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; ++ char *tmpdir = NULL; ++ char *url = NULL; ++ char *cmd = NULL; ++ char *conf = NULL, *param1 = NULL, *param2 = NULL; ++ bool delete_original = false; ++ struct manifest *manifest = NULL; ++ GError *gerror = NULL; + int ret; + + if (file->is_link) { +@@ -58,6 +64,74 @@ void __create_delta(struct file *file, int from_version) + + string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); + ++ if (access(original, F_OK) && ++ content_url) { ++ /* File does not exist. Try to get it from the online update repo instead. ++ * This fallback is meant to be used for CI builds which start with no local ++ * state and only HTTP(S) access to the published www directory. ++ * Not being able to retrieve the file is not an error and will merely ++ * prevent computing the delta. ++ */ ++ string_or_die(&tmpdir, "%s/make-pack-tmpdir-XXXXXX", state_dir); ++ tmpdir = g_dir_make_tmp("make-pack-XXXXXX", &gerror); ++ if (!tmpdir) { ++ LOG(NULL, "Failed to create temporary directory for untarring original file", "%s", ++ gerror->message); ++ assert(0); ++ } ++ /* Determine hash of original file in the corresponding Manifest. */ ++ manifest = manifest_from_file(from_version, "full"); ++ if (!manifest) { ++ LOG(NULL, "Failed to read full Manifest", "version %d, cannot retrieve original file", ++ from_version); ++ goto out; ++ } ++ const char *last_hash = NULL; ++ GList *list = g_list_first(manifest->files); ++ while (list) { ++ struct file *original_file = list->data; ++ if (!strcmp(file->filename, original_file->filename)) { ++ last_hash = original_file->hash; ++ break; ++ } ++ list = g_list_next(list); ++ } ++ if (!last_hash) { ++ LOG(NULL, "Original file not found", "%s in full manifest for %d - inconsistent update data?!", ++ file->filename, from_version); ++ goto out; ++ } ++ ++ /* We use a temporary copy because we don't want to ++ * tamper with the original "full" folder which ++ * probably does not even exist. Using a temporary file ++ * file implies re-downloading in the future, but that's ++ * consistent with the intended usage in a CI environment ++ * which always starts from scratch. ++ */ ++ free(original); ++ string_or_die(&original, "%s/%s", tmpdir, last_hash); ++ delete_original = true; ++ ++ /* ++ * This is a proof-of-concept. A real implementation should use ++ * a combination of libcurl + libarchive calls to unpack the files. ++ * For current Ostro OS, deltas despite xattr differences would ++ * be needed, otherwise this code here is of little use (all ++ * modified files fail the xattr sameness check, because security.ima ++ * changes when file content changes). ++ */ ++ string_or_die(&url, "%s/%d/files/%s.tar", content_url, from_version, last_hash); ++ LOG(file, "Downloading original file", "%s to %s", url, original); ++ ++ /* bsdtar can detect compression when reading from stdin, GNU tar can't. */ ++ string_or_die(&cmd, "curl -s %s | bsdtar -C %s -xf -", url, tmpdir); ++ if (system(cmd)) { ++ LOG(file, "Downloading/unpacking failed, skipping delta", "%s", url); ++ goto out; ++ } ++ } ++ + free(conf); + + conf = config_output_dir(); +@@ -141,6 +215,18 @@ void __create_delta(struct file *file, int from_version) + LOG(NULL, "Failed to rename", ""); + } + out: ++ if (delete_original) { ++ unlink(original); ++ } ++ if (tmpdir) { ++ rmdir(tmpdir); ++ g_free(tmpdir); ++ } ++ if (manifest) { ++ free_manifest(manifest); ++ } ++ g_clear_error(&gerror); ++ free(cmd); + free(testnewfile); + free(conf); + free(newfile); +diff --git a/src/globals.c b/src/globals.c +index 74758ce..0e047e2 100644 +--- a/src/globals.c ++++ b/src/globals.c +@@ -40,6 +40,7 @@ char *state_dir = NULL; + char *packstage_dir = NULL; + char *image_dir = NULL; + char *staging_dir = NULL; ++char *content_url = NULL; + + bool set_format(char *userinput) + { +@@ -76,6 +77,17 @@ bool set_state_dir(char *dir) + return true; + } + ++bool set_content_url(const char *url) ++{ ++ if (content_url) { ++ free(content_url); ++ } ++ string_or_die(&content_url, "%s", url); ++ ++ return true; ++} ++ ++ + bool init_globals(void) + { + if (format == 0) { +@@ -123,4 +135,5 @@ void free_state_globals(void) + free(packstage_dir); + free(image_dir); + free(staging_dir); ++ free(content_url); + } +diff --git a/src/make_packs.c b/src/make_packs.c +index 2d3e25e..827be56 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -49,6 +49,7 @@ static const struct option prog_opts[] = { + { "log-stdout", no_argument, 0, 'l' }, + { "statedir", required_argument, 0, 'S' }, + { "signcontent", no_argument, 0, 's' }, ++ { "content-url", required_argument, 0, 'u' }, + { 0, 0, 0, 0 } + }; + +@@ -61,6 +62,7 @@ static void usage(const char *name) + printf(" -l, --log-stdout Write log messages also to stdout\n"); + printf(" -S, --statedir Optional directory to use for state [ default:=%s ]\n", SWUPD_SERVER_STATE_DIR); + printf(" -s, --signcontent Enables cryptographic signing of update content\n"); ++ printf(" -u, --content-url Base URL of the update repo (optional, used to retrieve missing files on demand"); + printf("\n"); + } + +@@ -86,6 +88,12 @@ static bool parse_options(int argc, char **argv) + case 's': + enable_signing = true; + break; ++ case 'u': ++ if (!optarg || !set_content_url(optarg)) { ++ printf("Invalid --content-url argument ''%s''\n\n", optarg); ++ return false; ++ } ++ break; + } + } + +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch new file mode 100644 index 0000000..43ae6ff --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0025-swupd_make_pack-fix-extracting-files-with-bsdtar.patch @@ -0,0 +1,56 @@ +From d253e81e001ccbff9d7a8ee1768006c5ac36b259 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 27 Sep 2016 16:43:57 +0200 +Subject: [PATCH 25/29] swupd_make_pack: fix extracting files with bsdtar + +TAR_XATTR_ARGS is no longer used as part of a plain string. Embedding +the empty "" value for bsdtar inside an argv argument list passes an +empty parameter to bsdtar, leading to: + bsdtar: Must specify one of -c, -r, -t, -u, -x + +To allow the the "no parameter" case, it has to be argument list: that +can be empty. If not empty, it has to end with a comma. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/41] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + include/swupd.h | 4 ++-- + src/pack.c | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/include/swupd.h b/include/swupd.h +index 43a5002..0aa6399 100644 +--- a/include/swupd.h ++++ b/include/swupd.h +@@ -20,12 +20,12 @@ + #define TAR_COMMAND "bsdtar" + #define TAR_XATTR_ARGS "" + #define TAR_XATTR_ARGS_STRLIST +-#define TAR_WARN_ARGS "" ++#define TAR_WARN_ARGS_STRLIST + #else + #define TAR_COMMAND "tar" + #define TAR_XATTR_ARGS "--xattrs --xattrs-include='*'" + #define TAR_XATTR_ARGS_STRLIST "--xattrs", "--xattrs-include='*'", +-#define TAR_WARN_ARGS "--warning=no-timestamp" ++#define TAR_WARN_ARGS_STRLIST "--warning=no-timestamp", + #endif + + #if SWUPD_WITH_SELINUX +diff --git a/src/pack.c b/src/pack.c +index a1fbc51..12d7443 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -116,7 +116,7 @@ static void explode_pack_stage(int from_version, int to_version, char *module) + * time on the client... + */ + string_or_die(¶m, "%s/%s/%i_to_%i/staged", packstage_dir, module, from_version, to_version); +- char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL }; ++ char *const tarcmd[] = { TAR_COMMAND, "-C", param, TAR_WARN_ARGS_STRLIST TAR_PERM_ATTR_ARGS_STRLIST, "-xf", path, NULL }; + if (system_argv(tarcmd) == 0) { + unlink(path); + } +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch new file mode 100644 index 0000000..abc3082 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0026-fullfiles.c-fix-invalid-LOG-call.patch @@ -0,0 +1,30 @@ +From a8b77a7d4479df7662faf3a15f5aca6828d1c67a Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 27 Sep 2016 08:12:49 +0200 +Subject: [PATCH 26/29] fullfiles.c: fix invalid LOG() call + +LOG() takes an additional fixed string before the format string. + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/7e38f013eff01b84a8df88557f5312dc070ef0b3] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/fullfiles.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 31b9ab5..3be43d1 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -72,7 +72,7 @@ static void create_fullfile(struct file *file) + string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename); + if (lstat(origin, &sbuf) < 0) { + /* no input file: means earlier phase of update creation failed */ +- LOG(NULL, "Failed to stat %s\n", origin); ++ LOG(NULL, "Failed to stat", "%s: %s", origin, strerror(errno)); + assert(0); + } + +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch new file mode 100644 index 0000000..0a68667 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0027-update-control-over-parallelism.patch @@ -0,0 +1,89 @@ +From 8686189b3b080446fae732a85b72528b7fe68ba6 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Tue, 27 Sep 2016 08:25:40 +0200 +Subject: [PATCH 27/29] update control over parallelism + +The SWUPD_NUM_THREADS env variable is now understood by all three +commands and overrides the default number of threads. Setting it to 1 +is useful while debugging the code that runs inside threads (only one +thread hits breakpoints there). + +The hard-coded parallelism of 12 threads when analysing the file system +gets replaced with n, where n is the number of available CPUs. The default +is the same as before elsewhere (n for packing, 3 * n for fullfiles). + +Upstream-Status: Backported [https://github.com/clearlinux/swupd-server/commit/4e0fdd4193a8ce9dcf3cfc5e488dfd4b23b7e7d9] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/analyze_fs.c | 5 ++++- + src/fullfiles.c | 7 +++++-- + src/pack.c | 7 +++++-- + 3 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/src/analyze_fs.c b/src/analyze_fs.c +index 3bfb288..0f16343 100644 +--- a/src/analyze_fs.c ++++ b/src/analyze_fs.c +@@ -387,6 +387,9 @@ struct manifest *full_manifest_from_directory(int version) + { + struct manifest *manifest; + char *dir; ++ int numthreads = getenv("SWUPD_NUM_THREADS") ? ++ atoi(getenv("SWUPD_NUM_THREADS")) : ++ sysconf(_SC_NPROCESSORS_ONLN); + + LOG(NULL, "Computing hashes", "for %i/full", version); + +@@ -394,7 +397,7 @@ struct manifest *full_manifest_from_directory(int version) + + string_or_die(&dir, "%s/%i/full", image_dir, version); + +- threadpool = g_thread_pool_new(get_hash, dir, 12, FALSE, NULL); ++ threadpool = g_thread_pool_new(get_hash, dir, numthreads, FALSE, NULL); + + iterate_directory(manifest, dir, "", true); + +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 3be43d1..216a1d7 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -291,10 +291,13 @@ static void submit_fullfile_tasks(GList *files) + int ret; + int count = 0; + GError *err = NULL; ++ int numthreads = getenv("SWUPD_NUM_THREADS") ? ++ atoi(getenv("SWUPD_NUM_THREADS")) : ++ sysconf(_SC_NPROCESSORS_ONLN) * 3; + +- LOG(NULL, "fullfile threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN) * 3); ++ LOG(NULL, "fullfile threadpool", "%d threads", numthreads); + threadpool = g_thread_pool_new(create_fullfile_task, NULL, +- sysconf(_SC_NPROCESSORS_ONLN) * 3, ++ numthreads, + TRUE, NULL); + + printf("Starting downloadable fullfiles data creation\n"); +diff --git a/src/pack.c b/src/pack.c +index 12d7443..3ffb88a 100644 +--- a/src/pack.c ++++ b/src/pack.c +@@ -285,10 +285,13 @@ static void make_pack_deltas(GList *files) + struct file *file; + int ret; + GError *err = NULL; ++ int numthreads = getenv("SWUPD_NUM_THREADS") ? ++ atoi(getenv("SWUPD_NUM_THREADS")) : ++ sysconf(_SC_NPROCESSORS_ONLN); + +- LOG(NULL, "pack deltas threadpool", "%d threads", sysconf(_SC_NPROCESSORS_ONLN)); ++ LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); + threadpool = g_thread_pool_new(create_delta, NULL, +- sysconf(_SC_NPROCESSORS_ONLN), FALSE, NULL); ++ numthreads, FALSE, NULL); + + item = g_list_first(files); + while (item) { +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch new file mode 100644 index 0000000..0322735 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0028-enable-locales-in-all-programs.patch @@ -0,0 +1,100 @@ +From f74807f9aebbb7b8feb1f50107e268bd869f2691 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 28 Sep 2016 16:55:22 +0200 +Subject: [PATCH 1/3] enable locales in all programs + +This is a pre-condition for using libarchive directly: libarchive +needs to know what the encoding of filenames is, and it uses the +current locale for that. Without setlocale(), the locale is "C", which +only supports ASCII filenames, leading to warnings about "Can't +encode..." from libarchive when it is forced to fall back to copying +strings verbatim when writing archives that require UTF-8 encoding. + +As a side effect, error messages from libc will get translated +according to the user's environment. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/44] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + +--- + src/create_update.c | 6 ++++++ + src/make_fullfiles.c | 6 ++++++ + src/make_packs.c | 6 ++++++ + 3 files changed, 18 insertions(+) + +diff --git a/src/create_update.c b/src/create_update.c +index 97045e5..766609b 100644 +--- a/src/create_update.c ++++ b/src/create_update.c +@@ -29,6 +29,7 @@ + #include <errno.h> + #include <getopt.h> + #include <glib.h> ++#include <locale.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> +@@ -240,6 +241,11 @@ int main(int argc, char **argv) + /* keep valgrind working well */ + setenv("G_SLICE", "always-malloc", 0); + ++ if (!setlocale(LC_ALL, "")) { ++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]); ++ return EXIT_FAILURE; ++ } ++ + if (!parse_options(argc, argv)) { + free_globals(); + return EXIT_FAILURE; +diff --git a/src/make_fullfiles.c b/src/make_fullfiles.c +index 4ea2f01..2a1e2e9 100644 +--- a/src/make_fullfiles.c ++++ b/src/make_fullfiles.c +@@ -23,6 +23,7 @@ + #define _GNU_SOURCE + #include <assert.h> + #include <getopt.h> ++#include <locale.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> +@@ -88,6 +89,11 @@ int main(int argc, char **argv) + /* keep valgrind working well */ + setenv("G_SLICE", "always-malloc", 0); + ++ if (!setlocale(LC_ALL, "")) { ++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]); ++ return EXIT_FAILURE; ++ } ++ + if (!parse_options(argc, argv)) { + free_state_globals(); + return EXIT_FAILURE; +diff --git a/src/make_packs.c b/src/make_packs.c +index 4002cd9..8560b3f 100644 +--- a/src/make_packs.c ++++ b/src/make_packs.c +@@ -27,6 +27,7 @@ + #include <getopt.h> + #include <getopt.h> + #include <glib.h> ++#include <locale.h> + #include <stdio.h> + #include <stdlib.h> + #include <string.h> +@@ -101,6 +102,11 @@ int main(int argc, char **argv) + int exit_status = EXIT_FAILURE; + char *file_path = NULL; + ++ if (!setlocale(LC_ALL, "")) { ++ fprintf(stderr, "%s: setlocale() failed\n", argv[0]); ++ return EXIT_FAILURE; ++ } ++ + if (!parse_options(argc, argv)) { + free_state_globals(); + return EXIT_FAILURE; +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch b/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch new file mode 100644 index 0000000..d599261 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/0029-fullfiles-use-libarchive-directly.patch @@ -0,0 +1,626 @@ +From 4ebc886b910fa4cb83864658069a30dd133caca8 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Wed, 30 Mar 2016 13:14:42 +0200 +Subject: [PATCH 3/6] fullfiles: use libarchive directly + +Calling an external tar command makes the code fairly complicated, +because it is necessary to set up a suitable temporary directory with +the desired content. By calling libarchive directly, no temporary copy +is needed and directories and files can be treated (almost) the same +way. The only difference is that for files, data has to be added to +the archive. + +More important, performance under bitbake took a big hit because of +the external commands. Launching them when running under pseudo is a +lot slower compared to running natively as root. For example, the +unmodified swupd-make-fullfiles took over 20min for +ostro-image-swupd. With this change, it completes in 3:30min. When +running natively as root, there is also some improvement because less +work needs to be done, but total runtime only decreases from 2:28min +to 2:14min. + +By calling libarchive directly, swupd also gets better control over +error handling. bsdtar emits a warning for invalid filename or +linkname encoding, which was silently ignored when using the external +command. Now it is treated as an error. + +Archives always get created using the restricted pax interchange +format (see +https://github.com/libarchive/libarchive/wiki/ManPageLibarchiveFormats5), +same as with using bsdtar as external command. GNU tar should be able +to decode them properly as long as no extensions are needed (for +example, for xattrs). + +Only the smallest archive really gets written to disk. Until then, the +compressed archive is kept in memory. As an additional optimization, +creating an archive gets aborted once it is already larger than the +currently best one. In practice, that particular optimization did not +have any significant impact on performance. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/48] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> + +--- + Makefile.am | 6 +- + configure.ac | 1 + + include/libarchive_helper.h | 42 ++++++ + src/fullfiles.c | 324 +++++++++++++++++++++++++------------------- + src/in_memory_archive.c | 67 +++++++++ + 5 files changed, 300 insertions(+), 140 deletions(-) + create mode 100644 include/libarchive_helper.h + create mode 100644 src/in_memory_archive.c + +diff --git a/Makefile.am b/Makefile.am +index 528f02f..b14feb0 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -23,6 +23,7 @@ swupd_create_update_SOURCES = \ + src/helpers.c \ + src/heuristics.c \ + src/log.c \ ++ src/in_memory_archive.c \ + src/manifest.c \ + src/pack.c \ + src/rename.c \ +@@ -54,6 +55,7 @@ swupd_make_fullfiles_SOURCES = \ + src/fullfiles.c \ + src/globals.c \ + src/helpers.c \ ++ src/in_memory_archive.c \ + src/log.c \ + src/make_fullfiles.c \ + src/manifest.c \ +@@ -63,12 +65,13 @@ swupd_make_fullfiles_SOURCES = \ + src/stats.c \ + src/xattrs.c + +-AM_CPPFLAGS = $(glib_CFLAGS) -I$(top_srcdir)/include ++AM_CPPFLAGS = $(glib_CFLAGS) $(libarchive_CFLAGS) -I$(top_srcdir)/include + + swupd_create_update_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ ++ $(libarchive_LIBS) \ + $(bsdiff_LIBS) + + swupd_make_pack_LDADD = \ +@@ -81,6 +84,7 @@ swupd_make_fullfiles_LDADD = \ + $(glib_LIBS) \ + $(zlib_LIBS) \ + $(openssl_LIBS) \ ++ $(libarchive_LIBS) \ + $(bsdiff_LIBS) + + if ENABLE_LZMA +diff --git a/configure.ac b/configure.ac +index 8819206..3e2d933 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -36,6 +36,7 @@ AC_ARG_ENABLE( + AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command])), + AC_DEFINE([SWUPD_WITH_BSDTAR], 0, [Use default tar command]) + ) ++PKG_CHECK_MODULES([libarchive], [libarchive]) + + AC_ARG_ENABLE( + [tests], +diff --git a/include/libarchive_helper.h b/include/libarchive_helper.h +new file mode 100644 +index 0000000..ad28def +--- /dev/null ++++ b/include/libarchive_helper.h +@@ -0,0 +1,42 @@ ++/* ++ * Software Updater - server side ++ * ++ * Copyright © 2016 Intel Corporation. ++ * ++ * This program is free software: you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation, version 2 or later of the License. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program. If not, see <http://www.gnu.org/licenses/>. ++ * ++ * Authors: ++ * Patrick Ohly <patrick.ohly@intel.com> ++ * ++ */ ++ ++#ifndef __INCLUDE_GUARD_LIBARCHIVE_HELPER_H ++#define __INCLUDE_GUARD_LIBARCHIVE_HELPER_H ++ ++#include <archive.h> ++#include <stdint.h> ++ ++/* ++ * Used by archive_write_open() callbacks to store the resulting archive in memory. ++ */ ++struct in_memory_archive { ++ uint8_t *buffer; ++ size_t allocated; ++ size_t used; ++ /* If not 0, aborts writing when the used data would become larger than this. */ ++ size_t maxsize; ++}; ++ ++ssize_t in_memory_write(struct archive *, void *client_data, const void *buffer, size_t length); ++ ++#endif /* __INCLUDE_GUARD_LIBARCHIVE_HELPER_H */ +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 216a1d7..9fdfc2f 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -22,6 +22,8 @@ + */ + + #define _GNU_SOURCE ++#include <archive.h> ++#include <archive_entry.h> + #include <assert.h> + #include <errno.h> + #include <fcntl.h> +@@ -33,24 +35,27 @@ + #include <string.h> + #include <sys/stat.h> + #include <sys/types.h> ++#include <sys/xattr.h> + #include <unistd.h> + + #include "swupd.h" ++#include "libarchive_helper.h" + + /* output must be a file, which is a (compressed) tar file, of the file denoted by "file", without any of its + directory paths etc etc */ + static void create_fullfile(struct file *file) + { +- char *origin; ++ char *origin = NULL; + char *tarname = NULL; +- char *rename_source = NULL; +- char *rename_target = NULL; +- char *rename_tmpdir = NULL; +- int ret; + struct stat sbuf; + char *empty, *indir, *outdir; +- char *param1, *param2; +- int stderrfd; ++ struct archive_entry *entry = NULL; ++ struct archive *from = NULL, *to = NULL; ++ struct in_memory_archive best = { .buffer = NULL }; ++ struct in_memory_archive current = { .buffer = NULL }; ++ uint8_t *file_content = NULL; ++ size_t file_size; ++ int fd = -1; + + if (file->is_deleted) { + return; /* file got deleted -> by definition we cannot tar it up */ +@@ -59,15 +64,17 @@ static void create_fullfile(struct file *file) + empty = config_empty_dir(); + indir = config_image_base(); + outdir = config_output_dir(); ++ entry = archive_entry_new(); ++ assert(entry); ++ from = archive_read_disk_new(); ++ assert(from); + + string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); + if (access(tarname, R_OK) == 0) { + /* output file already exists...done */ +- free(tarname); ++ goto done; + return; + } +- free(tarname); +- //printf("%s was missing\n", file->hash); + + string_or_die(&origin, "%s/%i/full/%s", indir, file->last_change, file->filename); + if (lstat(origin, &sbuf) < 0) { +@@ -76,156 +83,195 @@ static void create_fullfile(struct file *file) + assert(0); + } + +- if (file->is_dir) { /* directories are easy */ +- char *tmp1, *tmp2, *dir, *base; +- +- tmp1 = strdup(origin); +- assert(tmp1); +- base = basename(tmp1); +- +- tmp2 = strdup(origin); +- assert(tmp2); +- dir = dirname(tmp2); +- +- string_or_die(&rename_tmpdir, "%s/XXXXXX", outdir); +- if (!mkdtemp(rename_tmpdir)) { +- LOG(NULL, "Failed to create temporary directory for %s move", origin); +- assert(0); +- } +- +- string_or_die(¶m1, "--exclude=%s/?*", base); +- string_or_die(¶m2, "./%s", base); +- char *const tarcfcmd[] = { TAR_COMMAND, "-C", dir, TAR_PERM_ATTR_ARGS_STRLIST, "-cf", "-", param1, param2, NULL }; +- char *const tarxfcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-xf", "-", NULL }; +- +- stderrfd = open("/dev/null", O_WRONLY); +- if (stderrfd == -1) { +- LOG(NULL, "Failed to open /dev/null", ""); +- assert(0); +- } +- if (system_argv_pipe(tarcfcmd, -1, stderrfd, tarxfcmd, -1, stderrfd) != 0) { +- assert(0); +- } +- free(param1); +- free(param2); +- close(stderrfd); +- +- string_or_die(&rename_source, "%s/%s", rename_tmpdir, base); +- string_or_die(&rename_target, "%s/%s", rename_tmpdir, file->hash); +- if (rename(rename_source, rename_target)) { +- LOG(NULL, "rename failed for %s to %s", rename_source, rename_target); ++ /* step 1: tar it with each compression type */ ++ typedef int (*filter_t)(struct archive *); ++ static const filter_t compression_filters[] = { ++ /* ++ * Start with the compression method that is most likely (*) to produce ++ * the best result. That will allow aborting creation of archives earlier ++ * when they become larger than the currently smallest archive. ++ * ++ * (*) statistics for ostro-image-swupd: ++ * 43682 LZMA ++ * 13398 gzip ++ * 844 bzip2 ++ */ ++ archive_write_add_filter_lzma, ++ archive_write_add_filter_gzip, ++ archive_write_add_filter_bzip2, ++ /* ++ * TODO (?): can archive_write_add_filter_none ever be better than compressing? ++ */ ++ NULL ++ }; ++ file_size = S_ISREG(sbuf.st_mode) ? sbuf.st_size : 0; ++ ++ archive_entry_copy_sourcepath(entry, origin); ++ if (archive_read_disk_entry_from_file(from, entry, -1, &sbuf)) { ++ LOG(NULL, "Getting directory attributes failed", "%s: %s", ++ origin, archive_error_string(from)); ++ assert(0); ++ } ++ archive_entry_copy_pathname(entry, file->hash); ++ if (file_size) { ++ file_content = malloc(file_size); ++ if (!file_content) { ++ LOG(NULL, "out of memory", ""); + assert(0); + } +- free(rename_source); +- +- /* for a directory file, tar up simply with gzip */ +- string_or_die(¶m1, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); +- char *const tarcmd[] = { TAR_COMMAND, "-C", rename_tmpdir, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param1, file->hash, NULL }; +- +- if (system_argv(tarcmd) != 0) { ++ fd = open(origin, O_RDONLY); ++ if (fd == -1) { ++ LOG(NULL, "Failed to open file", "%s: %s", ++ origin, strerror(errno)); + assert(0); + } +- free(param1); +- +- if (rmdir(rename_target)) { +- LOG(NULL, "rmdir failed for %s", rename_target); +- } +- free(rename_target); +- if (rmdir(rename_tmpdir)) { +- LOG(NULL, "rmdir failed for %s", rename_tmpdir); +- } +- free(rename_tmpdir); +- +- free(tmp1); +- free(tmp2); +- } else { /* files are more complex */ +- char *gzfile = NULL, *bzfile = NULL, *xzfile = NULL; +- char *tempfile; +- uint64_t gz_size = LONG_MAX, bz_size = LONG_MAX, xz_size = LONG_MAX; +- +- /* step 1: hardlink the guy to an empty directory with the hash as the filename */ +- string_or_die(&tempfile, "%s/%s", empty, file->hash); +- if (link(origin, tempfile) < 0) { +- LOG(NULL, "hardlink failed", "%s due to %s (%s -> %s)", file->filename, strerror(errno), origin, tempfile); +- char *const argv[] = { "cp", "-a", origin, tempfile, NULL }; +- if (system_argv(argv) != 0) { ++ size_t done = 0; ++ while (done < file_size) { ++ ssize_t curr; ++ curr = read(fd, file_content + done, file_size - done); ++ if (curr == -1) { ++ LOG(NULL, "Failed to read from file", "%s: %s", ++ origin, strerror(errno)); + assert(0); + } ++ done += curr; + } ++ } + +- /* step 2a: tar it with each compression type */ +- // lzma +- string_or_die(¶m1, "--directory=%s", empty); +- string_or_die(¶m2, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash); +- char *const tarlzmacmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-Jcf", param2, file->hash, NULL }; +- +- if (system_argv(tarlzmacmd) != 0) { ++ for (int i = 0; compression_filters[i]; i++) { ++ /* Need to re-initialize the archive handle, it cannot be re-used. */ ++ if (to) { ++ archive_write_free(to); ++ } ++ /* ++ * Use the recommended restricted pax interchange ++ * format. Numeric uid/gid values are stored in the archive ++ * (no uid/gid lookup enabled) because symbolic names can lead ++ * to a hash mismatch during unpacking when /etc/passwd or ++ * /etc/group change during an update (see ++ * https://github.com/clearlinux/swupd-client/issues/101). ++ * ++ * Filenames read from the file system are expected to be ++ * valid according to the current locale. archive_write_header() ++ * will warn about filenames that it cannot properly decode ++ * and proceeds by writing the raw bytes, but we treat this an ++ * error by not distinguishing between ARCHIVE_FATAL ++ * and ARCHIVE_WARN. ++ * ++ * When we fail with "Can't translate" errors, make sure that ++ * LANG and/or LC_ env variables are set. ++ */ ++ to = archive_write_new(); ++ assert(to); ++ if (archive_write_set_format_pax_restricted(to)) { ++ LOG(NULL, "PAX format", "%s", archive_error_string(to)); + assert(0); + } +- free(param1); +- free(param2); +- +- // gzip +- string_or_die(¶m1, "--directory=%s", empty); +- string_or_die(¶m2, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash); +- char *const targzipcmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-zcf", param2, file->hash, NULL }; +- +- if (system_argv(targzipcmd) != 0) { ++ do { ++ /* Try compression methods until we find one which is supported. */ ++ if (!compression_filters[i](to)) { ++ break; ++ } ++ } while(compression_filters[++i]); ++ /* ++ * Regardless of the block size below, never pad the ++ * last block, it just makes the archive larger. ++ */ ++ if (archive_write_set_bytes_in_last_block(to, 1)) { ++ LOG(NULL, "Removing padding failed", ""); + assert(0); + } +- free(param1); +- free(param2); +- +-#ifdef SWUPD_WITH_BZIP2 +- string_or_die(¶m1, "--directory=%s", empty); +- string_or_die(¶m2, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash); +- char *const tarbzip2cmd[] = { TAR_COMMAND, param1, TAR_PERM_ATTR_ARGS_STRLIST, "-jcf", param2, file->hash, NULL }; +- +- if (system_argv(tarbzip2cmd) != 0) { ++ /* ++ * Invoke in_memory_write() as often as possible and check each ++ * time whether we are already larger than the currently best ++ * algorithm. ++ */ ++ current.maxsize = best.used; ++ if (archive_write_set_bytes_per_block(to, 0)) { ++ LOG(NULL, "Removing blocking failed", ""); + assert(0); + } +- free(param1); +- free(param2); +- +-#endif +- +- /* step 2b: pick the smallest of the three compression formats */ +- string_or_die(&gzfile, "%s/%i/files/%s.tar.gz", outdir, file->last_change, file->hash); +- if (stat(gzfile, &sbuf) == 0) { +- gz_size = sbuf.st_size; ++ /* ++ * We can make an educated guess how large the resulting archive will be. ++ * Avoids realloc() calls when the file is big. ++ */ ++ if (!current.allocated) { ++ current.allocated = file_size + 4096; ++ current.buffer = malloc(current.allocated); + } +- string_or_die(&bzfile, "%s/%i/files/%s.tar.bz2", outdir, file->last_change, file->hash); +- if (stat(bzfile, &sbuf) == 0) { +- bz_size = sbuf.st_size; ++ if (!current.buffer) { ++ LOG(NULL, "out of memory", ""); ++ assert(0); + } +- string_or_die(&xzfile, "%s/%i/files/%s.tar.xz", outdir, file->last_change, file->hash); +- if (stat(xzfile, &sbuf) == 0) { +- xz_size = sbuf.st_size; ++ if (archive_write_open(to, ¤t, NULL, in_memory_write, NULL)) { ++ LOG(NULL, "Failed to create archive", "%s", ++ archive_error_string(to)); ++ assert(0); + } +- string_or_die(&tarname, "%s/%i/files/%s.tar", outdir, file->last_change, file->hash); +- if (gz_size <= xz_size && gz_size <= bz_size) { +- ret = rename(gzfile, tarname); +- } else if (xz_size <= bz_size) { +- ret = rename(xzfile, tarname); +- } else { +- ret = rename(bzfile, tarname); ++ if (archive_write_header(to, entry) || ++ file_content && archive_write_data(to, file_content, file_size) != (ssize_t)file_size || ++ archive_write_close(to)) { ++ if (current.maxsize && current.used >= current.maxsize) { ++ archive_write_free(to); ++ to = NULL; ++ continue; ++ } ++ LOG(NULL, "Failed to store file in archive", "%s: %s", ++ origin, archive_error_string(to)); ++ assert(0); + } +- if (ret != 0) { +- LOG(file, "post-tar rename failed", "ret=%d", ret); ++ if (!best.used || current.used < best.used) { ++ free(best.buffer); ++ best = current; ++ memset(¤t, 0, sizeof(current)); ++ } else { ++ /* Simply re-use the buffer for the next iteration. */ ++ current.used = 0; + } +- unlink(bzfile); +- unlink(xzfile); +- unlink(gzfile); +- free(bzfile); +- free(xzfile); +- free(gzfile); +- free(tarname); +- +- /* step 3: remove the hardlink */ +- unlink(tempfile); +- free(tempfile); ++ } ++ if (!best.used) { ++ LOG(NULL, "creating archive failed with all compression methods", ""); ++ assert(0); + } + ++ /* step 2: write out to disk. Archives are immutable and thus read-only. */ ++ fd = open(tarname, O_CREAT|O_WRONLY, S_IRUSR|S_IRGRP|S_IROTH); ++ if (fd <= 0) { ++ LOG(NULL, "Failed to create archive", "%s: %s", ++ tarname, strerror(errno)); ++ assert(0); ++ } ++ size_t done = 0; ++ while (done < best.used) { ++ ssize_t curr; ++ curr = write(fd, best.buffer + done, best.used - done); ++ if (curr == -1) { ++ LOG(NULL, "Failed to write archive", "%s: %s", ++ tarname, strerror(errno)); ++ assert(0); ++ } ++ done += curr; ++ } ++ if (close(fd)) { ++ LOG(NULL, "Failed to complete writing archive", "%s: %s", ++ tarname, strerror(errno)); ++ assert(0); ++ } ++ fd = -1; ++ free(best.buffer); ++ free(current.buffer); ++ free(file_content); ++ ++ done: ++ if (fd >= 0) { ++ close(fd); ++ } ++ archive_read_free(from); ++ if (to) { ++ archive_write_free(to); ++ } ++ archive_entry_free(entry); ++ free(tarname); + free(indir); + free(outdir); + free(empty); +diff --git a/src/in_memory_archive.c b/src/in_memory_archive.c +new file mode 100644 +index 0000000..abd7e54 +--- /dev/null ++++ b/src/in_memory_archive.c +@@ -0,0 +1,67 @@ ++/* ++ * Software Updater - server side ++ * ++ * Copyright © 2016 Intel Corporation. ++ * ++ * This program is free software: you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation, version 2 or later of the License. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program. If not, see <http://www.gnu.org/licenses/>. ++ * ++ * Authors: ++ * Patrick Ohly <patrick.ohly@intel.com> ++ * ++ */ ++ ++#include <errno.h> ++#include <stdlib.h> ++ ++#include "libarchive_helper.h" ++ ++ssize_t in_memory_write(struct archive *archive, void *client_data, const void *buffer, size_t length) ++{ ++ struct in_memory_archive *in_memory = client_data; ++ void *newbuff; ++ ++ if (in_memory->maxsize && in_memory->used + length >= in_memory->maxsize) { ++ archive_set_error(archive, EFBIG, "resulting archive would become larger than %lu", ++ (unsigned long)in_memory->maxsize); ++ archive_write_fail(archive); ++ /* ++ * Despite the error and archive_write_fail(), libarchive internally calls us ++ * again and when we fail again, overwrites our error with something about ++ * "Failed to clean up compressor". Therefore our caller needs to check for "used == maxsize" ++ * to detect that we caused the failure. ++ */ ++ in_memory->used = in_memory->maxsize; ++ return -1; ++ } ++ ++ if (in_memory->used + length > in_memory->allocated) { ++ /* Start with a small chunk, double in size to avoid too many reallocs. */ ++ size_t new_size = in_memory->allocated ? ++ in_memory->allocated * 2 : ++ 4096; ++ while (new_size < in_memory->used + length) { ++ new_size *= 2; ++ } ++ newbuff = realloc(in_memory->buffer, new_size); ++ if (!newbuff) { ++ archive_set_error(archive, ENOMEM, "failed to enlarge buffer"); ++ return -1; ++ } ++ in_memory->buffer = newbuff; ++ in_memory->allocated = new_size; ++ } ++ ++ memcpy(in_memory->buffer + in_memory->used, buffer, length); ++ in_memory->used += length; ++ return length; ++} +-- +2.1.4 + diff --git a/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch b/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch new file mode 100644 index 0000000..4f5ba19 --- /dev/null +++ b/recipes-core/swupd-server/swupd-server-3.2.5/swupd_create_fullfiles-avoid-segfault-when-nothing-c.patch @@ -0,0 +1,32 @@ +From edd16ae9b9aa2315aa6d1b8ee73239a3baa2f998 Mon Sep 17 00:00:00 2001 +From: Patrick Ohly <patrick.ohly@intel.com> +Date: Mon, 7 Nov 2016 09:05:40 +0100 +Subject: [PATCH] swupd_create_fullfiles: avoid segfault when nothing changes + +In the (unlikely) case that nothing changed between two builds, +get_deduplicated_fullfile_list() segfaults because it uses +manifest->files without checking for NULL, aka the empty list. + +Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/36] + +Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> +--- + src/fullfiles.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/fullfiles.c b/src/fullfiles.c +index 214b7b4..0d6ecf2 100644 +--- a/src/fullfiles.c ++++ b/src/fullfiles.c +@@ -300,7 +300,7 @@ static GList *get_deduplicated_fullfile_list(struct manifest *manifest) + manifest->files = g_list_sort(manifest->files, file_sort_hash); + + list = g_list_first(manifest->files); +- while (prev == NULL) { ++ while (prev == NULL && list != NULL) { + tmp = list->data; + list = g_list_next(list); + +-- +2.1.4 + |