1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
|
#!/bin/bash -e
# (C) 2018 GENIVI Alliance
# LICENSE: MPLv2
# ---- General settings ----
# Fixed cache location on all agents that support this.
AGENT_STANDARD_DL_DIR="/var/cache/yocto/downloads"
AGENT_STANDARD_SSTATE_DIR="/var/cache/yocto/sstate"
AGENT_STANDARD_SGX_LOCATION="/var/go/sgx_bin"
AGENT_STANDARD_SGX_GEN3_LOCATION="/var/go/sgx_bin_gen3"
# ---- Helper functions ----
fail=false
# This is some trickery, but quite useful.
# Return the value of a variable whose name is stored in another variable
deref() { eval echo \$$1 ; }
ensure_var_is_defined() {
var_name=$1
current_value=$(deref $var_name)
if [[ -z "$current_value" ]]; then
echo "Please define environment var_name \$$var_name (found empty value)"
fail=true # We want all the errors reported, so don't halt yet.
fi
}
define_with_default() {
var_name=$1
default="$2"
current_value=$(deref $var_name)
if [[ -z "$current_value" ]]; then
eval $var_name="$default"
fi
}
stop_if_failure() {
if [[ "$fail" == "true" ]] ; then
exit 1
fi
}
stop_immediately() {
echo "Fatal error occurred - stopping ci-build script"
exit 2
}
append_local_conf() {
LOCAL_CONF="$BASEDIR/build/conf/local.conf"
if [[ -f "$LOCAL_CONF" ]]; then
echo -n "Appending to local.conf: "
cat <<EOT | tee -a "$LOCAL_CONF"
$1
EOT
else
echo "Fatal: Did not find local.conf where expected"
stop_immediately
fi
}
stage_artifact() {
mkdir -p staging
cmd=$1
shift
if [[ -z "$1" ]] ; then
echo "Skipped an artifact pattern that matched nothing"
return
fi
for f in $@ ; do # <- could be empty glob list
flag=
if [[ -e "$f" ]] ; then
if [[ $cmd == cp && -d "$f" ]] ; then
flag=-a
elif [[ $cmd == cp ]] ; then
flag=--preserve=all
fi
$cmd $flag "$f" staging/
else
echo "Skipping non-existing (or already moved) artifact: $f"
fi
done
}
# Layer version overrides:
# Find and evaluate environment variables of type
# LAYER_<layername>_FORK
# LAYER_<layername>_COMMIT
# LAYER_<layername>_BRANCH
# LAYER_<layername>_TAG
# *NOTE* Variables cannot have dashes in name therefore, underscores must be
# used! For example: LAYER_meta_openembedded for meta-openembedded layer
# If an override variable is defined, build with this version of the layer
# instead. If multiple conflicting values, then the most specific override
# (commit) wins
layer_override() {
local name=$1 suffix var_name value d
# Variables cannot have dashes in name therefore, underscores must be
# used. For example: LAYER_meta_browser_COMMIT for meta-browser layer
clean_name=$(echo "$name" | sed 's/-/_/g')
# First FORK...
var_name="LAYER_${clean_name}_FORK"
value=$(deref "$var_name")
if [ -n "$value" ] ; then
echo "***** NOTE: OVERRIDING LAYER $name USING \$${var_name} = $value *****"
d="$PWD"
cd $name || echo "Layer name wrong? Can't cd to dir : $name"
echo "Fetching from $value"
git remote add temp_fork "$value"
git fetch temp_fork
# Master is assumed, but it is likely overridden by a COMMIT/TAG/BRANCH
# value below. but this could fail.
echo "NOTE: Assuming master branch for now (might be overridden later)"
echo "NOTE: This will fail if no master branch exists in $value"
echo "+ git reset temp_fork/master --hard"
git reset temp_fork/master --hard
cd "$d"
fi
# ...then the others
for suffix in COMMIT TAG BRANCH ; do
var_name="LAYER_${clean_name}_${suffix}"
value=$(deref "$var_name")
if [ -n "$value" ] ; then
echo "***** NOTE: OVERRIDING LAYER $name USING \$${var_name} = $value *****"
d="$PWD"
cd $name || echo "Layer name wrong? Can't cd to dir : $name"
git fetch --all
# If forked, make sure to use the right remote, otherwise origin
# For branches, prefer the explicit: remote/branchname
if git remote | grep -q temp_fork ; then
git checkout "temp_fork/$value" 2>/dev/null || git checkout $value
else
git checkout "origin/$value" 2>/dev/null || git checkout $value
fi
cd "$d"
break # First one wins, priority order: COMMIT >= TAG >= BRANCH
fi
done
}
# Because of the order they are called we can't use output from the
# layer_override function above, so we have some repeated code here.
print_layer_overrides() {
local names="$1" name clean_name suffix var_name value d
for name in $names ; do
clean_name=$(echo "$name" | sed 's/-/_/g')
for suffix in COMMIT TAG BRANCH ; do
var_name="LAYER_${clean_name}_${suffix}"
value=$(deref "$var_name")
if [ -n "$value" ] ; then
echo "$var_name = $value"
fi
done
done
}
LAYERS="
meta-ivi
poky
meta-gplv2
meta-openembedded
renesas-rcar-gen3
"
# Clean up function called at end of script, or if interrupted
cleanup() {
# Restore git config user - if this was not defined locally before then it is
# unset (which might mean a global setting is used)
if [ -z "$olduser" ] ; then
git config --unset user.name
else
git config user.name "$olduser"
fi
if [ -z "$oldemail" ] ; then
git config --unset user.email
else
git config user.email "$oldemail"
fi
}
# ---- Main program ----
trap cleanup SIGINT SIGTERM
D=$(dirname "$0")
cd "$D"
BASEDIR=$(readlink -f "$PWD/..")
# The user must define this
ensure_var_is_defined TARGET
stop_if_failure
# The values can be overridden by defining environment variables
# If no value given, use this default:
define_with_default BUILD_SDK false
define_with_default BUILD_TEST_IMAGE false
define_with_default COPY_LICENSES false
define_with_default LAYER_ARCHIVE false
define_with_default CREATE_RELEASE_DIR false
define_with_default MIRROR "https://docs.projects.genivi.org/releases/yocto_mirror"
define_with_default PREMIRROR "" # By default none (but we have the shared DL_DIR)
define_with_default RM_WORK false
define_with_default REUSE_STANDARD_DL_DIR true
define_with_default REUSE_STANDARD_SSTATE_DIR true
define_with_default SGX_DRIVERS $AGENT_STANDARD_SGX_LOCATION
define_with_default SGX_GEN_3_DRIVERS $AGENT_STANDARD_SGX_GEN3_LOCATION
define_with_default SOURCE_ARCHIVE false
define_with_default STANDARD_RELEASE_BUILD false
# The following only apply to temporary (local) dirs. If any of
# REUSE_STANDARD_{DL,SSTATE}_DIR is defined then those directories will be
# used no matter what. Those reusable DL/SSTATE dirs are never cleared by
# this script.
define_with_default KEEP_DOWNLOADS false
define_with_default KEEP_SSTATE false
define_with_default KEEP_TMP false
stop_if_failure
#git_gdp="https://github.com/GENIVI/meta-ivi"
branch="master"
# cd workingdir
MACHINE="$TARGET" # For most boards - exceptions handled below
if [[ "$TARGET" == "r-car-m3-starter-kit" ]]; then
MACHINE="m3ulcb"
fi
ensure_var_is_defined MACHINE
export MACHINE
# OVERRIDING VARIABLES
if [[ "$REUSE_STANDARD_DL_DIR" == "true" ]]; then
DL_DIR="$AGENT_STANDARD_DL_DIR"
fi
if [[ "$REUSE_STANDARD_SSTATE_DIR" == "true" ]]; then
SSTATE_DIR="$AGENT_STANDARD_SSTATE_DIR"
fi
if [[ "$STANDARD_RELEASE_BUILD" == "true" ]]; then
SOURCE_ARCHIVE=true # NOTE: overriding env settings
COPY_LICENSES=true
LAYER_ARCHIVE=true
CREATE_RELEASE_DIR=true
fi
echo Configuration:
echo
echo "TARGET = $TARGET"
echo "FORK = $FORK"
echo "BRANCH = $BRANCH"
echo "TAG = $TAG"
echo "COMMIT = $COMMIT"
echo "RELEASE = $RELEASE (currently unused)"
print_layer_overrides "$LAYERS"
echo "BUILD_SDK = $BUILD_SDK"
echo "BUILD_TEST_IMAGE = $BUILD_TEST_IMAGE"
echo "COPY_LICENSES" = "$COPY_LICENSES"
echo "CREATE_RELEASE_DIR" = "$CREATE_RELEASE_DIR"
echo "DL_DIR = $DL_DIR"
echo "MIRROR" = "$MIRROR"
echo "PREMIRROR" = "$PREMIRROR"
echo "REUSE_STANDARD_DL_DIR = $REUSE_STANDARD_DL_DIR"
echo "REUSE_STANDARD_SSTATE_DIR = $REUSE_STANDARD_SSTATE_DIR"
echo "RM_WORK = $RM_WORK"
echo "SGX_DRIVERS = $SGX_DRIVERS"
echo "SGX_GEN_3_DRIVERS = $SGX_GEN_3_DRIVERS"
echo "SOURCE_ARCHIVE" = "$SOURCE_ARCHIVE"
echo "SSTATE_DIR = $DL_DIR"
echo "STANDARD_RELEASE_BUILD" = "$STANDARD_RELEASE_BUILD"
# INIT
cd "$BASEDIR"
echo "*** Initializing layers"
scripts/checkout
echo "*** Initializing conf"
export TEMPLATECONF=$PWD/meta-ivi/conf
source ../poky/oe-init-build-env build
# build steps
# We are now in build/
# If DL/SSTATE are to be reused it is normally by the use of
# REUSE_STANDARD_DL_DIR and REUSE_STANDARD_SSTATE_DIR. Local dirs left in the
# build directory are therefore normally wiped, unless these environment
# variables say otherwise.
if [[ "$KEEP_DOWNLOADS" != "true" ]] ; then
rm -rf downloads
fi
if [[ "$KEEP_SSTATE" != "true" ]]; then
rm -rf cache sstate-cache
fi
if [[ "$KEEP_TMP" != "true" ]]; then
rm -rf tmp
fi
cd "$BASEDIR"
# Need to set an identity for some git patching done by recipes
set +e # The following two commands can fail if value is unset
olduser="$(git config user.name)"
oldemail="$(git config user.email)"
set -e
git config user.name "CI build -- ignore"
git config user.email no_email
# Normally the material (source code) is defined in the pipeline itself in the
# CIAT system but there are multiple ways to override it provided here.
if [[ -n "$FORK" ]]; then
echo "***** NOTE: OVERRIDING REPOSITORY WITH \$FORK = $FORK *****"
git remote set-url origin "$FORK"
git fetch
# V *danger* V. One reason why you should not use this script if it's not in CI
git reset origin/master --hard
git checkout $BRANCH # <- note this should be ok even if $BRANCH is an empty value
fi
if [[ -n "$BRANCH" ]]; then
echo "***** NOTE: OVERRIDING CHOSEN COMMIT USING \$BRANCH = $BRANCH *****"
git checkout $BRANCH
fi
if [[ -n "$TAG" ]]; then
echo "***** NOTE: OVERRIDING CHOSEN COMMIT USING \$TAG = $TAG *****"
git checkout $TAG
fi
if [[ -n "$COMMIT" ]]; then
echo "***** NOTE: OVERRIDING CHOSEN COMMIT USING \$COMMIT = $COMMIT *****"
git checkout $COMMIT
fi
## FIXME echo 'BBLAYERS += "##your source code root##/poky/../meta-ivi /meta-ivi-test"'
# Do version override on sublayers (if any such overrides defined)
cd ..
for l in $LAYERS ; do
layer_override $l
done
# LOCAL CONF MODIFICATIONS
if [[ "$RM_WORK" == "true" ]]; then
append_local_conf 'INHERIT += "rm_work"'
fi
if [[ -n "$DL_DIR" ]]; then
append_local_conf "DL_DIR = \"$DL_DIR\""
fi
if [[ -n "$SSTATE_DIR" ]]; then
append_local_conf "SSTATE_DIR = \"$SSTATE_DIR\""
fi
if [[ -n "$BB_NUMBER_THREADS" ]]; then
append_local_conf "BB_NUMBER_THREADS = \"$BB_NUMBER_THREADS\""
fi
if [[ -n "$PARALLEL_MAKE" ]]; then
echo $PARALLEL_MAKE | egrep -q '^-j' || PARALLEL_MAKE="-j$PARALLEL_MAKE"
append_local_conf "PARALLEL_MAKE = \"$PARALLEL_MAKE\""
fi
if [[ "$SOURCE_ARCHIVE" == "true" ]]; then
append_local_conf 'INHERIT += "archiver"'
append_local_conf 'ARCHIVER_MODE[src] = "original"'
fi
if [[ "$COPY_LICENSES" == "true" ]]; then
append_local_conf 'COPY_LIC_DIRS = "1"'
append_local_conf 'COPY_LIC_MANIFEST = "1"'
fi
# The own-mirrors bbclass is generally more convenient for PREMIRRORS, but
# this format makes it similar to the $MIRROR setup below, which needs to be
# explicit anyhow.
if [[ -n "$PREMIRROR" ]]; then
append_local_conf "
PREMIRRORS_prepend = \"\\
git://.*/.* $PREMIRROR \\n \\
ftp://.*/.* $PREMIRROR \\n \\
http://.*/.* $PREMIRROR \\n \\
https://.*/.* $PREMIRROR \\n \\
\"
"
fi
# This is the "post" mirror (i.e. checked last).
if [[ -n "$MIRROR" ]]; then
append_local_conf "
MIRRORS_append = \"\\
git://.*/.* $MIRROR \\n \\
ftp://.*/.* $MIRROR \\n \\
http://.*/.* $MIRROR \\n \\
https://.*/.* $MIRROR \\n \\
\"
"
fi
if [[ "$BUILD_SDK" != "true" ]]; then
bitbake pulsar-image
fi
if [[ "$BUILD_TEST_IMAGE" == "true" ]]; then
# FIXME
bitbake pulsar-image-test
fi
if [[ "$BUILD_SDK" == "true" ]]; then
# same?
bitbake pulsar-image-sdk
fi
cd "$BASEDIR"
rm -f logs.tar logs.tar.gz
find build/tmp/work \( -name "*.log" -o -name "log.*" -o -name "run.*" \) -print0 | xargs -0 tar uf logs.tar || true
gzip logs.tar || true
# Soften up the failure requirements here. Maybe sometimes
# some things can't be staged, which is probably ok.
set +e
rm -rf staging
shopt -s nullglob
stage_artifact mv build/tmp/deploy/licenses
stage_artifact mv build/tmp/deploy/licenses/genivi-dev-platform*/license.manifest
stage_artifact mv build/tmp/deploy/sdk*
stage_artifact cp build/tmp/deploy/images/*
stage_artifact mv build/tmp/deploy/sources
stage_artifact cp build/conf/*.conf
stage_artifact mv logs.tar.gz
stage_artifact cp build/buildhistory/images/*/glibc/genivi-dev-platform/files-in-image.txt
stage_artifact mv build/buildhistory
stage_artifact mv build/tmp/buildstats
if [[ "$LAYER_ARCHIVE" == "true" ]]; then
tar cfj staging/meta-layers-snapshot.tar.bz2 meta-* poky renesas* build/conf
fi
set -e
# META-IVI note:
# usually we don't release built (binary) images of baseline -- however to keep
# the script consistent with GDP version of the script (and make releases
# possible in theory), the support is kept intact here:
# Environment contains alot of variables from Go.CD that specify the built
# version/hash, and other metadata. Let's store them for future reference.
# and other relevant info
cd "$BASEDIR"
build_info_file=staging/build_info.txt
env >$build_info_file
echo 'For conf , see files *.conf, and any diff below' >>$build_info_file
git diff build/conf/templates/*.inc >>$build_info_file
mkdir -p staging/images
mv staging/*/{*201*ext*,*201*rootfs*,*sdimg*,*qemuboot.conf*,modules*.tgz,*hddimg*,bzImage*201*,*201*.iso,*201*.wic,*.efi,*.dtd} staging/images/ 2>/dev/null || true
cd staging && rm -rf "$TARGET"
cd "$BASEDIR"
if [[ "$CREATE_RELEASE_DIR" == "true" ]]; then
set +e
mkdir -p release
echo "Moving images to release/"
mv staging/images release/ 2>/dev/null || true
echo "Moving staging/sources to release/"
cp -a staging/sources release/ 2>/dev/null
echo "Moving staging/licenses into release/"
mv staging/licenses release/
echo "Copying various metadata to release"
cp staging/files-in-image.txt release/ 2>/dev/null
cp staging/build_info.txt release/ 2>/dev/null
cp staging/license.manifest release/ 2>/dev/null
set -e
fi
set +e
echo "Artifacts in staging/ and release/"
ls -al staging/ release/
echo
echo "...in release/images/ :"
ls -al release/images/
set -e
cleanup
|