aboutsummaryrefslogtreecommitdiffstats
path: root/recipes-core/swupd-server/swupd-server/0001-create_pack-rely-less-on-previous-builds.patch
blob: f27b11149ebe147c8db297ab10c465651927faee (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
From 4be3d38364cc1bead38c32465cb7e2f3a0be4c88 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 16 Nov 2016 14:26:30 +0100
Subject: [PATCH 11/13] create_pack: rely less on previous builds

When a file has not been modified in the current build, then by
definition the current copy of the file is the same as in the build
were it was last changed and thus it does not matter whether we use
<current build>/full/<file> or <last change>/full/<file>. But using
the current copy is better for a CI system which starts without local
access to older rootfs directories. It might also be a bit more
efficient (file access less scattered between different "full"
directories).

Staging directories is better than staging .tar archives containing
those directories for the same reason (the .tar archive might not be
available in the CI system) and probably also improves efficiency (no
need to invoke bsdtar just to create a directory; impact not
measured).

Also fix a slight flaw in the "target file exists already" handling:
when that occured for whatever reason (likely only during manual
debugging), the code would add the original fullfile .tar although it
isn't needed. Clearing the "ret" variable in that particular error
case avoids that.

make_pack_full_files() and make_final_pack() used the exact same code
for populating the "staged" directory. Now that common code is in
stage_entry().

Upstream-Status: Submitted [https://github.com/clearlinux/swupd-server/pull/47]

Signed-off-by: Patrick Ohly <patrick.ohly@intel.com>

---
 include/swupd.h |   2 +-
 src/delta.c     |   4 +--
 src/pack.c      | 105 +++++++++++++++++++++++++++++++-------------------------
 3 files changed, 62 insertions(+), 49 deletions(-)

diff --git a/include/swupd.h b/include/swupd.h
index 2263018..1b3a046 100644
--- a/include/swupd.h
+++ b/include/swupd.h
@@ -247,7 +247,7 @@ extern void type_change_detection(struct manifest *manifest);
 
 extern void rename_detection(struct manifest *manifest, int last_change, GList *last_versions_list);
 extern void link_renames(GList *newfiles, struct manifest *from_manifest);
-extern void __create_delta(struct file *file, int from_version, char *from_hash);
+extern void __create_delta(struct file *file, int from_version, int to_version, char *from_hash);
 
 extern void account_delta_hit(void);
 extern void account_delta_miss(void);
diff --git a/src/delta.c b/src/delta.c
index 563abb9..8c1bc64 100644
--- a/src/delta.c
+++ b/src/delta.c
@@ -35,7 +35,7 @@
 #include "swupd.h"
 #include "xattrs.h"
 
-void __create_delta(struct file *file, int from_version, char *from_hash)
+void __create_delta(struct file *file, int from_version, int to_version, char *from_hash)
 {
 	char *original = NULL, *newfile = NULL, *outfile = NULL, *dotfile = NULL, *testnewfile = NULL, *conf = NULL;
 	char *tmpdir = NULL;
@@ -59,7 +59,7 @@ void __create_delta(struct file *file, int from_version, char *from_hash)
 	}
 
 	conf = config_image_base();
-	string_or_die(&newfile, "%s/%i/full/%s", conf, file->last_change, file->filename);
+	string_or_die(&newfile, "%s/%i/full/%s", conf, to_version, file->filename);
 
 	string_or_die(&original, "%s/%i/full/%s", conf, from_version, file->peer->filename);
 
diff --git a/src/pack.c b/src/pack.c
index a7094b8..f7770b2 100644
--- a/src/pack.c
+++ b/src/pack.c
@@ -37,6 +37,7 @@
 #include <unistd.h>
 
 #include "swupd.h"
+#include "xattrs.h"
 
 static void empty_pack_stage(int full, int from_version, int to_version, char *module)
 {
@@ -149,6 +150,51 @@ static void prepare_pack(struct packdata *pack)
 	link_renames(pack->end_manifest->files, manifest);
 }
 
+static int stage_entry(struct file *file,
+		       const char *fullfrom, const char *fullto,
+		       const char *tarfrom, const char *tarto,
+		       const char *packname)
+{
+	int ret;
+
+	/* Prefer to hardlink uncompressed files or replicate
+	 * directories first, and fall back to the compressed
+	 * versions if that failed.
+	 */
+	if (!file->is_dir) {
+		ret = link(fullfrom, fullto);
+		if (ret && errno == EEXIST) {
+			ret = 0;
+		} else if (ret) {
+			LOG(NULL, "Failure to link for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno);
+		}
+	} else {
+		/* Replicate directory. */
+		struct stat st;
+		if ((stat(fullfrom, &st) ||
+		     mkdir(fullto, 0) ||
+		     chmod(fullto, st.st_mode) ||
+		     chown(fullto, st.st_uid, st.st_gid) ||
+		     (xattrs_copy(fullfrom, fullto), false)) &&
+		    errno != EEXIST) {
+			LOG(NULL, "Failure to replicate dir for pack", "%s: %s to %s (%s) %i", packname, fullfrom, fullto, strerror(errno), errno);
+			rmdir(fullto);
+			ret = -1;
+		} else {
+			ret = 0;
+		}
+	}
+
+	if (ret) {
+		ret = link(tarfrom, tarto);
+		if (ret && errno != EEXIST) {
+			LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", tarfrom, tarto, strerror(errno), errno);
+		}
+	}
+
+	return ret;
+}
+
 static void make_pack_full_files(struct packdata *pack)
 {
 	GList *item;
@@ -168,32 +214,18 @@ static void make_pack_full_files(struct packdata *pack)
 			char *fullfrom, *fullto;
 
 			/* hardlink each file that is in <end> but not in <X> */
-			string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
+			string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename);
 			string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
 				      pack->module, pack->from, pack->to, file->hash);
 			string_or_die(&from, "%s/%i/files/%s.tar", staging_dir, file->last_change, file->hash);
 			string_or_die(&to, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
 				      pack->module, pack->from, pack->to, file->hash);
 
-			ret = -1;
-			errno = 0;
-
-			/* Prefer to hardlink uncompressed files (excluding
-			 * directories) first, and fall back to the compressed
-			 * versions if the hardlink fails.
+			/* Prefer to hardlink uncompressed files or replicate
+			 * directories first, and fall back to the compressed
+			 * versions if that failed.
 			 */
-			if (!file->is_dir) {
-				ret = link(fullfrom, fullto);
-				if (ret && errno != EEXIST) {
-					LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", fullfrom, fullto, strerror(errno), errno);
-				}
-			}
-			if (ret) {
-				ret = link(from, to);
-				if (ret && errno != EEXIST) {
-					LOG(NULL, "Failure to link for fullfile pack", "%s to %s (%s) %i", from, to, strerror(errno), errno);
-				}
-			}
+			ret = stage_entry(file, fullfrom, fullto, from, to, "fullfile");
 
 			if (ret == 0) {
 				pack->fullcount++;
@@ -270,17 +302,18 @@ static GList *consolidate_packs_delta_files(GList *files, struct packdata *pack)
 	return files;
 }
 
-static void create_delta(gpointer data, __unused__ gpointer user_data)
+static void create_delta(gpointer data, gpointer user_data)
 {
 	struct file *file = data;
+	int *to_version = user_data;
 
 	/* if the file was not found in the from version, skip delta creation */
 	if (file->peer) {
-		__create_delta(file, file->peer->last_change, file->peer->hash);
+		__create_delta(file, file->peer->last_change, *to_version, file->peer->hash);
 	}
 }
 
-static void make_pack_deltas(GList *files)
+static void make_pack_deltas(GList *files, int to_version)
 {
 	GThreadPool *threadpool;
 	GList *item;
@@ -290,7 +323,7 @@ static void make_pack_deltas(GList *files)
 	int numthreads = num_threads(1.0);
 
 	LOG(NULL, "pack deltas threadpool", "%d threads", numthreads);
-	threadpool = g_thread_pool_new(create_delta, NULL,
+	threadpool = g_thread_pool_new(create_delta, &to_version,
 				       numthreads, FALSE, NULL);
 
 	item = g_list_first(files);
@@ -365,7 +398,7 @@ static int make_final_pack(struct packdata *pack)
 			      file->last_change, file->hash);
 		string_or_die(&tarto, "%s/%s/%i_to_%i/staged/%s.tar", packstage_dir,
 			      pack->module, pack->from, pack->to, file->hash);
-		string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, file->last_change, file->filename);
+		string_or_die(&fullfrom, "%s/%i/full/%s", image_dir, pack->to, file->filename);
 		string_or_die(&fullto, "%s/%s/%i_to_%i/staged/%s", packstage_dir,
 			      pack->module, pack->from, pack->to, file->hash);
 
@@ -399,27 +432,7 @@ static int make_final_pack(struct packdata *pack)
 				}
 			}
 		} else {
-			ret = -1;
-			errno = 0;
-
-			/* Prefer to hardlink uncompressed files (excluding
-			 * directories) first, and fall back to the compressed
-			 * versions if the hardlink fails.
-			 */
-			if (!file->is_dir) {
-				ret = link(fullfrom, fullto);
-				if (ret && errno != EEXIST) {
-					LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", fullfrom, fullto, strerror(errno), errno);
-				}
-			}
-
-			if (ret) {
-				ret = link(tarfrom, tarto);
-				if (ret && errno != EEXIST) {
-					LOG(NULL, "Failure to link for final pack", "%s to %s (%s) %i\n", tarfrom, tarto, strerror(errno), errno);
-				}
-			}
-
+			ret = stage_entry(file, fullfrom, fullto, tarfrom, tarto, "final");
 			if (ret == 0) {
 				pack->fullcount++;
 			}
@@ -537,7 +550,7 @@ int make_pack(struct packdata *pack)
 
 	/* step 2: consolidate delta list & create all delta files*/
 	delta_list = consolidate_packs_delta_files(delta_list, pack);
-	make_pack_deltas(delta_list);
+	make_pack_deltas(delta_list, pack->to);
 	g_list_free(delta_list);
 
 	/* step 3: complete pack creation */
-- 
2.1.4