aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-09-30 09:11:09 +0200
committerPatrick Ohly <patrick.ohly@intel.com>2016-12-08 14:12:55 +0100
commitbcede1b94951641536d3e5318406bcec7dab2213 (patch)
tree785551140b539e3fbc58943b4908c281a600d5e8
parent1b020a3e28f9527e32a8a06a55a908ba1038d92f (diff)
downloadmeta-swupd-bcede1b94951641536d3e5318406bcec7dab2213.tar.gz
meta-swupd-bcede1b94951641536d3e5318406bcec7dab2213.tar.bz2
meta-swupd-bcede1b94951641536d3e5318406bcec7dab2213.zip
swupd-server: enable support for single input rootfs
Splitting up the "mega" image just so that the original swupd-create-update can be used unmodified creates lots of redundant file operations, which are noticably slow under pseudo. This path is meant to go upstream. For now it is included here as POC. Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
-rw-r--r--recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch347
-rw-r--r--recipes-core/swupd-server/swupd-server_git.bb1
2 files changed, 348 insertions, 0 deletions
diff --git a/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch b/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch
new file mode 100644
index 0000000..7151c5c
--- /dev/null
+++ b/recipes-core/swupd-server/swupd-server/0001-swupd-create-update-alternative-input-layout.patch
@@ -0,0 +1,347 @@
+From e1f0d54a940eb7d04e2fbd59bd995a819331425f Mon Sep 17 00:00:00 2001
+From: Patrick Ohly <patrick.ohly@intel.com>
+Date: Fri, 30 Sep 2016 08:42:08 +0200
+Subject: [PATCH 1/2] swupd-create-update: alternative input layout
+
+In Ostro OS, we already have a "full" directory with all files.
+Splitting it up into bundles just so that swupd-create-update can
+reconstruct the "full" directory is a waste of IO, and noticably slow
+when run under pseudo.
+
+To streamline the required work, a new layout for the "image" input
+directory gets introduced:
+- The "full" directory gets created by the caller before invoking
+ swupd-create-update.
+- For each bundle, instead of a <bundle> directory, there is a
+ <bundle>.content.txt file, listing all entries (including directories)
+ of the bundle.
+
+The traditional mode of operation still works as before because each operation
+which normally works with a bundle directory checks whether there is such a
+directory and if not, switches to the new mode.
+
+That way it is even possible to mix the two modes, i.e. replacing only
+some bundles with a content list, although that's probably not all
+that useful.
+
+Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>
+---
+ src/analyze_fs.c | 171 ++++++++++++++++++++++++++++++++++++++--------------
+ src/chroot.c | 24 +++++---
+ src/create_update.c | 6 +-
+ src/fullfiles.c | 2 +
+ 4 files changed, 150 insertions(+), 53 deletions(-)
+
+diff --git a/src/analyze_fs.c b/src/analyze_fs.c
+index 0f16343..d534587 100644
+--- a/src/analyze_fs.c
++++ b/src/analyze_fs.c
+@@ -275,7 +275,7 @@ static void get_hash(gpointer data, gpointer user_data)
+ /* disallow characters which can do unexpected things when the filename is
+ * used on a tar command line via system("tar [args] filename [more args]");
+ */
+-static bool illegal_characters(char *filename)
++static bool illegal_characters(const char *filename)
+ {
+ char c;
+ int i;
+@@ -301,25 +301,145 @@ static bool illegal_characters(char *filename)
+ return false;
+ }
+
++static void add_file(struct manifest *manifest,
++ const char *entry_name,
++ char *sub_filename,
++ char *fullname,
++ bool do_hash)
++{
++ GError *err = NULL;
++ struct file *file;
++
++ if (illegal_characters(entry_name)) {
++ printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
++ free(sub_filename);
++ free(fullname);
++ return;
++ }
++
++ file = calloc(1, sizeof(struct file));
++ assert(file);
++
++ file->last_change = manifest->version;
++ file->filename = sub_filename;
++
++ populate_file_struct(file, fullname);
++ if (file->is_deleted) {
++ /*
++ * populate_file_struct() logs a stat() failure, but
++ * does not abort. When adding files that should
++ * exist, this case is an error.
++ */
++ LOG(NULL, "file not found", "%s", fullname);
++ assert(0);
++ }
++
++
++ /* if for some reason there is a file in the official build
++ * which should not be included in the Manifest, then open a bug
++ * to get it removed, and work around its presence by
++ * excluding it here, eg:
++ if (strncmp(file->filename, "/dev/", 5) == 0) {
++ continue;
++ }
++ */
++
++ if (do_hash) {
++ /* compute the hash from a thread */
++ int ret;
++ ret = g_thread_pool_push(threadpool, file, &err);
++ if (ret == FALSE) {
++ printf("GThread hash computation push error\n");
++ printf("%s\n", err->message);
++ assert(0);
++ }
++ }
++ manifest->files = g_list_prepend(manifest->files, file);
++ manifest->count++;
++}
++
++
+ static void iterate_directory(struct manifest *manifest, char *pathprefix,
+ char *subpath, bool do_hash)
+ {
+ DIR *dir;
+ struct dirent *entry;
+ char *fullpath;
+- int ret;
+- GError *err = NULL;
+
+ string_or_die(&fullpath, "%s/%s", pathprefix, subpath);
+
+ dir = opendir(fullpath);
+ if (!dir) {
++ FILE *content;
++ int len;
++ free(fullpath);
++ if (errno != ENOENT) {
++ return;
++ }
++ /*
++ * If there is a <dir>.content.txt instead of
++ * the actual directory, then read that
++ * file. It has a list of path names,
++ * including all directories. The
++ * corresponding file system entry is then
++ * expected to be in a pre-populated "full"
++ * directory.
++ */
++ if (subpath[0]) {
++ string_or_die(&fullpath, "%s/%s.content.txt", pathprefix, len, subpath);
++ } else {
++ string_or_die(&fullpath, "%s.content.txt", pathprefix);
++ }
++ content = fopen(fullpath, "r");
+ free(fullpath);
++ fullpath = NULL;
++ if (content) {
++ char *line = NULL;
++ size_t len = 0;
++ ssize_t read;
++ const char *full;
++ int full_len;
++ /*
++ * determine path to "full" directory: it is assumed to be alongside
++ * "pathprefix", i.e. pathprefix/../full. But pathprefix does not exit,
++ * so we have to strip the last path component.
++ */
++ full = strrchr(pathprefix, '/');
++ if (full) {
++ full_len = full - pathprefix + 1;
++ full = pathprefix;
++ } else {
++ full = "";
++ full_len = 0;
++ }
++ while ((read = getline(&line, &len, content)) != -1) {
++ if (read) {
++ const char *entry_name = strrchr(line, '/');
++ if (entry_name) {
++ entry_name++;
++ } else {
++ entry_name = line;
++ }
++ if (line[read - 1] == '\n') {
++ line[read - 1] = 0;
++ }
++ string_or_die(&fullpath, "%.*sfull/%s", full_len, full, line);
++ add_file(manifest,
++ entry_name,
++ strdup(line),
++ fullpath,
++ do_hash);
++ }
++ }
++ free(line);
++ }
++
++ // If both directory and content file are missing, silently (?)
++ // don't add anything to the manifest.
+ return;
+ }
+
+ while (dir) {
+- struct file *file;
+ char *sub_filename;
+ char *fullname;
+
+@@ -334,50 +454,13 @@ static void iterate_directory(struct manifest *manifest, char *pathprefix,
+ }
+
+ string_or_die(&sub_filename, "%s/%s", subpath, entry->d_name);
+-
+- if (illegal_characters(entry->d_name)) {
+- printf("WARNING: Filename %s includes illegal character(s) ...skipping.\n", sub_filename);
+- free(sub_filename);
+- continue;
+- }
+-
+- file = calloc(1, sizeof(struct file));
+- if (!file) {
+- break;
+- }
+-
+- file->last_change = manifest->version;
+- file->filename = sub_filename;
+-
+ string_or_die(&fullname, "%s/%s", fullpath, entry->d_name);
+- populate_file_struct(file, fullname);
+- free(fullname);
+-
+ if (entry->d_type == DT_DIR) {
+- iterate_directory(manifest, pathprefix, file->filename, do_hash);
++ iterate_directory(manifest, pathprefix, sub_filename, do_hash);
+ }
++ /* takes ownership of the strings */
++ add_file(manifest, entry->d_name, sub_filename, fullname, do_hash);
+
+- /* if for some reason there is a file in the official build
+- * which should not be included in the Manifest, then open a bug
+- * to get it removed, and work around its presence by
+- * excluding it here, eg:
+- if (strncmp(file->filename, "/dev/", 5) == 0) {
+- continue;
+- }
+- */
+-
+- if (do_hash) {
+- /* compute the hash from a thread */
+- ret = g_thread_pool_push(threadpool, file, &err);
+- if (ret == FALSE) {
+- printf("GThread hash computation push error\n");
+- printf("%s\n", err->message);
+- closedir(dir);
+- return;
+- }
+- }
+- manifest->files = g_list_prepend(manifest->files, file);
+- manifest->count++;
+ }
+ closedir(dir);
+ free(fullpath);
+diff --git a/src/chroot.c b/src/chroot.c
+index 32ed997..f3832e1 100644
+--- a/src/chroot.c
++++ b/src/chroot.c
+@@ -39,15 +39,21 @@ void chroot_create_full(int newversion)
+ char *full_dir;
+
+ string_or_die(&full_dir, "%s/%i/full/", image_dir, newversion);
++ if (!access(full_dir, R_OK|X_OK)) {
++ free(full_dir);
++ return;
++ }
+
+ g_mkdir_with_parents(full_dir, S_IRWXU);
+
+ /* start with base */
+- LOG(NULL, "Copying chroot os-core to full", "");
+ string_or_die(&param, "%s/%i/os-core/", image_dir, newversion);
+- char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
+- if (system_argv(rsynccmd) != 0) {
+- assert(0);
++ if (!access(param, F_OK)) {
++ LOG(NULL, "Copying chroot os-core to full", "");
++ char *const rsynccmd[] = { "rsync", "-aAX", param, full_dir, NULL };
++ if (system_argv(rsynccmd) != 0) {
++ assert(0);
++ }
+ }
+ free(param);
+
+@@ -58,11 +64,13 @@ void chroot_create_full(int newversion)
+ break;
+ }
+
+- LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
+ string_or_die(&param, "%s/%i/%s/", image_dir, newversion, group);
+- char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
+- if (system_argv(rsynccmd) != 0) {
+- assert(0);
++ if (!access(param, F_OK)) {
++ LOG(NULL, "Overlaying bundle chroot onto full", "%s", group);
++ char *const rsynccmd[] = { "rsync", "-aAX", "--ignore-existing", param, full_dir, NULL };
++ if (system_argv(rsynccmd) != 0) {
++ assert(0);
++ }
+ }
+ free(param);
+ }
+diff --git a/src/create_update.c b/src/create_update.c
+index 766609b..74d5376 100644
+--- a/src/create_update.c
++++ b/src/create_update.c
+@@ -141,6 +141,7 @@ static bool parse_options(int argc, char **argv)
+ static void populate_dirs(int version)
+ {
+ char *newversiondir;
++ char *newversiondircontent = NULL;
+
+ string_or_die(&newversiondir, "%s/%d", image_dir, version);
+
+@@ -182,9 +183,11 @@ static void populate_dirs(int version)
+ }
+
+ string_or_die(&newversiondir, "%s/%d/%s", image_dir, version, group);
++ string_or_die(&newversiondircontent, "%s/%d/%s.content.txt", image_dir, version, group);
+
+ /* Create the bundle directory(s) as needed */
+- if (access(newversiondir, F_OK | R_OK) != 0) {
++ if (access(newversiondir, F_OK | R_OK) != 0 &&
++ access(newversiondircontent, F_OK | R_OK) != 0) {
+ printf("%s does not exist...creating\n", group);
+ if (mkdir(newversiondir, 0755) != 0) {
+ printf("Failed to create %s subdirectory\n", group);
+@@ -193,6 +196,7 @@ static void populate_dirs(int version)
+ }
+ }
+ free(newversiondir);
++ free(newversiondircontent);
+ }
+
+ static int check_build_env(void)
+diff --git a/src/fullfiles.c b/src/fullfiles.c
+index 9fdfc2f..214b7b4 100644
+--- a/src/fullfiles.c
++++ b/src/fullfiles.c
+@@ -136,6 +136,8 @@ static void create_fullfile(struct file *file)
+ }
+ done += curr;
+ }
++ close(fd);
++ fd = -1;
+ }
+
+ for (int i = 0; compression_filters[i]; i++) {
+--
+2.1.4
+
diff --git a/recipes-core/swupd-server/swupd-server_git.bb b/recipes-core/swupd-server/swupd-server_git.bb
index d4757e3..e5cc3d8 100644
--- a/recipes-core/swupd-server/swupd-server_git.bb
+++ b/recipes-core/swupd-server/swupd-server_git.bb
@@ -13,6 +13,7 @@ SRC_URI = "git://github.com/clearlinux/swupd-server.git;protocol=https \
file://0027-update-control-over-parallelism.patch \
file://0028-enable-locales-in-all-programs.patch \
file://0029-fullfiles-use-libarchive-directly.patch \
+ file://0001-swupd-create-update-alternative-input-layout.patch \
"
SRCREV = "ddca171dad32229ceeff8b8527a179610b88ce55"