From ecd62bee2dc3df9a181319a3f55c9cccab838aaf Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 16 Nov 2016 14:26:30 +0100 Subject: [PATCH 1/3] create_pack: rely less on previous builds When a file has not been modified in the current build, then by definition the current copy of the file is the same as in the build were it was last changed and thus it does not matter whether we use /full/ or /full/. But using the current copy is better for a CI system which starts without local access to older rootfs directories. It might also be a bit more efficient (file access less scattered between different "full" directories). Staging directories is better than staging .tar archives containing those directories for the same reason (the .tar archive might not be available in the CI system) and probably also improves efficiency (no need to invoke bsdtar just to create a directory; impact not measured). Also fix a slight flaw in the "target file exists already" handling: when that occured for whatever reason (likely only during manual debugging), the code would add the original fullfile .tar although it isn't needed. Clearing the "ret" variable in that particular error case avoids that. make_pack_full_files() and make_final_pack() used the exact same code for populating the "staged" directory. Now that common code is in stage_entry(). Signed-off-by: Patrick Ohly --- include/swupd.h | 2 +- src/delta.c | 4 +-- src/pack.c | 105 +++++++++++++++++++++++++++++++------------------------- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/include/swupd.h b/include/swupd.h index c1c0e96..cf384e3 100644 --- a/include/swupd.h +++ b/include/swupd.h @@ -244,7 +244,7 @@ extern void type_change_detection(struct manifest *manifest); extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list); extern void link_renames(GList *newfiles, struct manifest *from_manifest); -extern void __create_delta(struct file *file, int from_version); +extern void __create_delta(struct file *file, int from_version, int to_version); extern void account_delta_hit(void); extern void account_delta_miss(void); diff --git a/src/delta.c b/src/delta.c index 7e978b1..8fff4c9 100644 --- a/src/delta.c +++ b/src/delta.c @@ -35,7 +35,7 @@ #include "swupd.h" #include "xattrs.h" -void __create_delta(struct file *file, int from_version) +void __create_delta(struct file *file, int from_version, int to_version) { char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL; char *tmpdir = NULL; @@ -60,7 +60,7 @@ void __create_delta(struct file *file, int from_version) } conf = config_image_base(); - string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename); + string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename); string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename); diff --git a/src/pack.c b/src/pack.c index 984c2d6..ccb28bd 100644 --- a/src/pack.c +++ b/src/pack.c @@ -37,6 +37,7 @@ #include #include "swupd.h" +#include "xattrs.h" static void empty_pack_stage(int full, int from_version, int to_version, char *module) { @@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack) link_renames(pack->end_manifest->files, manifest); } +static int stage_entry(struct file *file, + const char *fullfrom, const char *fullto, + const char *tarfrom, const char *tarto, + const char *packname) +{ + int ret; + + /* Prefer to hardlink uncompressed files or replicate + * directories first, and fall back to the compressed + * versions if that failed. + */ + if (!file->is_dir) { + ret = link(fullfrom, fullto); + if (ret && errno == EEXIST) { + ret = 0; + } else if (ret) { + LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); + } + } else { + /* Replicate directory. */ + struct stat st; + if ((stat(fullfrom, &st) || + mkdir(fullto, 0) || + chmod(fullto, st.st_mode) || + chown(fullto, st.st_uid, st.st_gid) || + (xattrs_copy(fullfrom, fullto), false)) && + errno != EEXIST) { + LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno); + rmdir(fullto); + ret = -1; + } else { + ret = 0; + } + } + + if (ret) { + ret = link(tarfrom, tarto); + if (ret && errno != EEXIST) { + LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno); + } + } + + return ret; +} + static void make_pack_full_files(struct packdata *pack) { GList *item; @@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack) char *fullfrom, *fullto; /* hardlink each file that is in but not in */ - string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); + string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, pack->module, pack->from, pack->to, file->hash); string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash); string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, pack->module, pack->from, pack->to, file->hash); - ret = -1; - errno = 0; - - /* Prefer to hardlink uncompressed files (excluding - * directories) first, and fall back to the compressed - * versions if the hardlink fails. + /* Prefer to hardlink uncompressed files or replicate + * directories first, and fall back to the compressed + * versions if that failed. */ - if (!file->is_dir) { - ret = link(fullfrom, fullto); - if (ret && errno != EEXIST) { - LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno); - } - } - if (ret) { - ret = link(from, to); - if (ret && errno != EEXIST) { - LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno); - } - } + ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile"); if (ret == 0) { pack->fullcount++; @@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack) return files; } -static void create_delta(gpointer data, __unused__ gpointer user_data) +static void create_delta(gpointer data, gpointer user_data) { struct file *file = data; + int *to_version = user_data; /* if the file was not found in the from version, skip delta creation */ if (file->peer) { - __create_delta(file, file->peer->last_change); + __create_delta(file, file->peer->last_change, *to_version); } } -static void make_pack_deltas(GList *files) +static void make_pack_deltas(GList *files, int to_version) { GThreadPool *threadpool; GList *item; @@ -292,7 +325,7 @@ static void make_pack_deltas(GList *files) sysconf(_SC_NPROCESSORS_ONLN); LOG(NULL, "pack deltas threadpool", "%d threads", numthreads); - threadpool = g_thread_pool_new(create_delta, NULL, + threadpool = g_thread_pool_new(create_delta, &to_version, numthreads, FALSE, NULL); item = g_list_first(files); @@ -367,7 +400,7 @@ static int make_final_pack(struct packdata *pack) file->last_change, file->hash); string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir, pack->module, pack->from, pack->to, file->hash); - string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename); + string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename); string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir, pack->module, pack->from, pack->to, file->hash); @@ -401,27 +434,7 @@ static int make_final_pack(struct packdata *pack) } } } else { - ret = -1; - errno = 0; - - /* Prefer to hardlink uncompressed files (excluding - * directories) first, and fall back to the compressed - * versions if the hardlink fails. - */ - if (!file->is_dir) { - ret = link(fullfrom, fullto); - if (ret && errno != EEXIST) { - LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno); - } - } - - if (ret) { - ret = link(tarfrom, tarto); - if (ret && errno != EEXIST) { - LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno); - } - } - + ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final"); if (ret == 0) { pack->fullcount++; } @@ -539,7 +552,7 @@ int make_pack(struct packdata *pack) /* step 2: consolidate delta list & create all delta files*/ delta_list = consolidate_packs_delta_files(delta_list, pack); - make_pack_deltas(delta_list); + make_pack_deltas(delta_list, pack->to); g_list_free(delta_list); /* step 3: complete pack creation */ -- 2.1.4