aboutsummaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorGunnar Andersson <gandersson@genivi.org>2018-02-21 14:58:30 +0100
committerOscar Andreasson <oan@frozentux.net>2018-05-18 08:42:50 +0200
commit7e0029da49eed081e78a6d483a23ab8d8ef2b026 (patch)
tree128fb7b134f6e599fbd2c1093ec0ae0718782a98 /scripts
parent2c11b36653a3c5c0657dedfdfcf78278a9920f22 (diff)
downloadmeta-ivi-7e0029da49eed081e78a6d483a23ab8d8ef2b026.tar.gz
meta-ivi-7e0029da49eed081e78a6d483a23ab8d8ef2b026.tar.bz2
meta-ivi-7e0029da49eed081e78a6d483a23ab8d8ef2b026.zip
Add initial ci-build.sh
This is a copy of ci-build.sh from genivi-dev-platform which is *not yet adjusted* for meta-ivi build. This is to facilitate diffing against the next commit to see what was changed. Signed-off-by: Gunnar Andersson <gandersson@genivi.org>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/ci-build.sh575
1 files changed, 575 insertions, 0 deletions
diff --git a/scripts/ci-build.sh b/scripts/ci-build.sh
new file mode 100755
index 0000000..dae533e
--- /dev/null
+++ b/scripts/ci-build.sh
@@ -0,0 +1,575 @@
+#!/bin/bash -e
+
+# (C) 2017 Gunnar Andersson
+# LICENSE: MPLv2
+
+# User warning!
+
+# A casual user will need to read the script to understand what the first
+# parameter shall be to make it actually run:
+#
+
+# Fail script unless you give the magic value:
+if [[ "$1" != CI_FLAG ]]; then
+
+ cat <<EOT
+
+READ THIS: This script was not designed for personal use.
+
+If you still use it, you hereby agree to any applicable End-User License
+Agreement (EULA) of the corresponding BSPs that are used. You must read
+and understand the obligations by studying the script.
+
+It is still a useful script but a few things could be destructive, e.g.
+"git reset --hard" if you use this in your development environment instead of
+a throw-away environment like CI.
+
+Please make sure you commit/backup local changes and read the script to
+understand what it does!
+
+After that I hope you enjoy using it.
+
+EOT
+
+ exit 2
+fi
+
+# ---- General settings ----
+
+# Fixed cache location on all agents that support this.
+AGENT_STANDARD_DL_DIR="/var/cache/yocto/downloads"
+AGENT_STANDARD_SSTATE_DIR="/var/cache/yocto/sstate"
+AGENT_STANDARD_SGX_LOCATION="/var/go/sgx_bin"
+AGENT_STANDARD_SGX_GEN3_LOCATION="/var/go/sgx_bin_gen3"
+
+# ---- Helper functions ----
+
+fail=false
+
+# This is some trickery, but quite useful.
+# Return the value of a variable whose name is stored in another variable
+deref() { eval echo \$$1 ; }
+
+ensure_var_is_defined() {
+ var_name=$1
+ current_value=$(deref $var_name)
+ if [[ -z "$current_value" ]]; then
+ echo "Please define environment var_name \$$var_name (found empty value)"
+ fail=true # We want all the errors reported, so don't halt yet.
+ fi
+}
+
+define_with_default() {
+ var_name=$1
+ default="$2"
+ current_value=$(deref $var_name)
+ if [[ -z "$current_value" ]]; then
+ eval $var_name="$default"
+ fi
+}
+
+stop_if_failure() {
+ if [[ "$fail" == "true" ]] ; then
+ exit 1
+ fi
+}
+
+stop_immediately() {
+ echo "Fatal error occurred - stopping ci-build script"
+ exit 2
+}
+
+append_local_conf() {
+ LOCAL_CONF="$BASEDIR/gdp-src-build/conf/local.conf"
+ if [[ -f "$LOCAL_CONF" ]]; then
+ echo -n "Appending to local.conf: "
+ cat <<EOT | tee -a "$LOCAL_CONF"
+$1
+EOT
+ else
+ echo "Fatal: Did not find local.conf where expected"
+ stop_immediately
+ fi
+}
+
+stage_artifact() {
+ mkdir -p staging
+ cmd=$1
+ shift
+
+ if [[ -z "$1" ]] ; then
+ echo "Skipped an artifact pattern that matched nothing"
+ return
+ fi
+
+ for f in $@ ; do # <- could be empty glob list
+ flag=
+ if [[ -e "$f" ]] ; then
+ if [[ $cmd == cp && -d "$f" ]] ; then
+ flag=-a
+ elif [[ $cmd == cp ]] ; then
+ flag=--preserve=all
+ fi
+ $cmd $flag "$f" staging/
+ else
+ echo "Skipping non-existing (or already moved) artifact: $f"
+ fi
+ done
+}
+
+# Layer version overrides:
+# Find and evaluate environment variables of type
+
+# LAYER_<layername>_FORK
+# LAYER_<layername>_COMMIT
+# LAYER_<layername>_BRANCH
+# LAYER_<layername>_TAG
+
+# *NOTE* Variables cannot have dashes in name therefore, underscores must be
+# used! For example: LAYER_meta_browser_COMMIT for meta-browser layer
+
+# If an override variable is defined, build with this version of the layer
+# instead. If multiple conflicting values, then the most specific override
+# (commit) wins
+
+layer_override() {
+ local name=$1 suffix var_name value d
+
+ # Variables cannot have dashes in name therefore, underscores must be
+ # used. For example: LAYER_meta_browser_COMMIT for meta-browser layer
+ clean_name=$(echo "$name" | sed 's/-/_/g')
+
+ # First FORK...
+ var_name="LAYER_${clean_name}_FORK"
+ value=$(deref "$var_name")
+ if [ -n "$value" ] ; then
+ echo "***** NOTE: OVERRIDING LAYER $name USING \$${var_name} = $value *****"
+ d="$PWD"
+ cd $name || echo "Layer name wrong? Can't cd to dir : $name"
+ echo "Fetching from $value"
+ git remote add temp_fork "$value"
+ git fetch temp_fork
+
+ # Master is assumed, but it is likely overridden by a COMMIT/TAG/BRANCH
+ # value below. but this could fail.
+ echo "NOTE: Assuming master branch for now (might be overridden later)"
+ echo "NOTE: This will fail if no master branch exists in $value"
+ echo "+ git reset temp_fork/master --hard"
+ git reset temp_fork/master --hard
+ cd "$d"
+ fi
+
+ # ...then the others
+ for suffix in COMMIT TAG BRANCH ; do
+ var_name="LAYER_${clean_name}_${suffix}"
+ value=$(deref "$var_name")
+ if [ -n "$value" ] ; then
+ echo "***** NOTE: OVERRIDING LAYER $name USING \$${var_name} = $value *****"
+ d="$PWD"
+ cd $name || echo "Layer name wrong? Can't cd to dir : $name"
+ git fetch --all
+
+ # If forked, make sure to use the right remote, otherwise origin
+ # For branches, prefer the explicit: remote/branchname
+ if git remote | grep -q temp_fork ; then
+ git checkout "temp_fork/$value" 2>/dev/null || git checkout $value
+ else
+ git checkout "origin/$value" 2>/dev/null || git checkout $value
+ fi
+
+ cd "$d"
+ break # First one wins, priority order: COMMIT >= TAG >= BRANCH
+ fi
+ done
+}
+
+# Because of the order they are called we can't use output from the
+# layer_override function above, so we have some repeated code here.
+print_layer_overrides() {
+ local names="$1" name clean_name suffix var_name value d
+
+ for name in $names ; do
+ clean_name=$(echo "$name" | sed 's/-/_/g')
+ for suffix in COMMIT TAG BRANCH ; do
+ var_name="LAYER_${clean_name}_${suffix}"
+ value=$(deref "$var_name")
+ if [ -n "$value" ] ; then
+ echo "$var_name = $value"
+ fi
+ done
+ done
+}
+
+# FIXME: Asking git to list submodules would be better
+LAYERS="
+meta-browser
+meta-erlang
+meta-genivi-dev
+meta-intel
+meta-iot-web
+meta-ivi
+meta-linaro
+meta-oic
+meta-openembedded
+meta-qcom
+meta-qt5
+meta-raspberrypi
+meta-renesas
+meta-rvi
+poky
+renesas-rcar-gen3
+"
+
+# Clean up function called at end of script, or if interrupted
+cleanup() {
+ # Restore git config user - if this was not defined locally before then it is
+ # unset (which might mean a global setting is used)
+ if [ -z "$olduser" ] ; then
+ git config --unset user.name
+ else
+ git config user.name "$olduser"
+ fi
+ if [ -z "$oldemail" ] ; then
+ git config --unset user.email
+ else
+ git config user.email "$oldemail"
+ fi
+}
+
+# ---- Main program ----
+
+trap cleanup SIGINT SIGTERM
+
+D=$(dirname "$0")
+cd "$D"
+BASEDIR=$(readlink -f "$PWD/..")
+
+# The user must define this
+ensure_var_is_defined TARGET
+
+stop_if_failure
+
+# The values can be overridden by defining environment variables
+# If no value given, use this default:
+define_with_default BUILD_SDK false
+define_with_default COPY_LICENSES false
+define_with_default LAYER_ARCHIVE false
+define_with_default CREATE_RELEASE_DIR false
+define_with_default MIRROR "https://docs.projects.genivi.org/releases/yocto_mirror"
+define_with_default PREMIRROR "" # By default none (but we have the shared DL_DIR)
+define_with_default RM_WORK false
+define_with_default REUSE_STANDARD_DL_DIR true
+define_with_default REUSE_STANDARD_SSTATE_DIR true
+define_with_default SGX_DRIVERS $AGENT_STANDARD_SGX_LOCATION
+define_with_default SGX_GEN_3_DRIVERS $AGENT_STANDARD_SGX_GEN3_LOCATION
+define_with_default SOURCE_ARCHIVE false
+define_with_default STANDARD_RELEASE_BUILD false
+
+# The following only apply to temporary (local) dirs. If any of
+# REUSE_STANDARD_{DL,SSTATE}_DIR is defined then those directories will be
+# used no matter what. Those reusable DL/SSTATE dirs are never cleared by
+# this script.
+define_with_default KEEP_DOWNLOADS false
+define_with_default KEEP_SSTATE false
+
+define_with_default KEEP_TMP false
+
+stop_if_failure
+
+git_gdp="https://github.com/GENIVI/genivi-dev-platform"
+branch="master"
+
+# Special case: Use porter script to copy graphics drivers for koelsch
+if [[ "$TARGET" == "koelsch" ]]; then
+ GFX_MACHINE=porter
+else
+ GFX_MACHINE=$TARGET
+fi
+
+# cd workingdir
+MACHINE="$TARGET" # For most boards - exceptions handled below
+
+if [[ "$TARGET" == "r-car-m3-starter-kit" ]]; then
+ MACHINE="m3ulcb"
+fi
+
+ensure_var_is_defined MACHINE
+export MACHINE
+
+# OVERRIDING VARIABLES
+if [[ "$REUSE_STANDARD_DL_DIR" == "true" ]]; then
+ DL_DIR="$AGENT_STANDARD_DL_DIR"
+fi
+
+if [[ "$REUSE_STANDARD_SSTATE_DIR" == "true" ]]; then
+ SSTATE_DIR="$AGENT_STANDARD_SSTATE_DIR"
+fi
+
+if [[ "$STANDARD_RELEASE_BUILD" == "true" ]]; then
+ SOURCE_ARCHIVE=true # NOTE: overriding env settings
+ COPY_LICENSES=true
+ LAYER_ARCHIVE=true
+ CREATE_RELEASE_DIR=true
+fi
+
+echo Configuration:
+echo
+echo "TARGET = $TARGET"
+echo "FORK = $FORK"
+echo "BRANCH = $BRANCH"
+echo "TAG = $TAG"
+echo "COMMIT = $COMMIT"
+echo "RELEASE = $RELEASE (currently unused)"
+print_layer_overrides "$LAYERS"
+echo "BUILD_SDK = $BUILD_SDK"
+echo "COPY_LICENSES" = "$COPY_LICENSES"
+echo "CREATE_RELEASE_DIR" = "$CREATE_RELEASE_DIR"
+echo "DL_DIR = $DL_DIR"
+echo "MIRROR" = "$MIRROR"
+echo "PREMIRROR" = "$PREMIRROR"
+echo "REUSE_STANDARD_DL_DIR = $REUSE_STANDARD_DL_DIR"
+echo "REUSE_STANDARD_SSTATE_DIR = $REUSE_STANDARD_SSTATE_DIR"
+echo "RM_WORK = $RM_WORK"
+echo "SGX_DRIVERS = $SGX_DRIVERS"
+echo "SGX_GEN_3_DRIVERS = $SGX_GEN_3_DRIVERS"
+echo "SOURCE_ARCHIVE" = "$SOURCE_ARCHIVE"
+echo "SSTATE_DIR = $DL_DIR"
+echo "STANDARD_RELEASE_BUILD" = "$STANDARD_RELEASE_BUILD"
+
+# build steps
+cd "$BASEDIR/gdp-src-build"
+
+# If DL/SSTATE are to be reused it is normally by the use of
+# REUSE_STANDARD_DL_DIR and REUSE_STANDARD_SSTATE_DIR. Local dirs left in the
+# build directory are therefore normally wiped, unless these environment
+# variables say otherwise.
+
+if [[ "$KEEP_DOWNLOADS" != "true" ]] ; then
+ rm -rf downloads
+fi
+
+if [[ "$KEEP_SSTATE" != "true" ]]; then
+ rm -rf cache sstate-cache
+fi
+
+if [[ "$KEEP_TMP" != "true" ]]; then
+ rm -rf tmp
+fi
+
+cd "$BASEDIR"
+
+# Need to set an identity for some git patching done by recipes
+set +e # The following two commands can fail if value is unset
+olduser="$(git config user.name)"
+oldemail="$(git config user.email)"
+set -e
+git config user.name "CI build -- ignore"
+git config user.email no_email
+
+# Normally the material (source code) is defined in the pipeline itself in the
+# CIAT system but there are multiple ways to override it provided here.
+
+if [[ -n "$FORK" ]]; then
+ echo "***** NOTE: OVERRIDING REPOSITORY WITH \$FORK = $FORK *****"
+ git remote set-url origin "$FORK"
+ git fetch
+ # V *danger* V. One reason why you should not use this script if it's not in CI
+ git reset origin/master --hard
+ # After this, submodule sync/update will be done by init.sh
+ # this may need some further testing!
+ git checkout $BRANCH # <- note this should be ok even if $BRANCH is an empty value
+fi
+
+if [[ -n "$BRANCH" ]]; then
+ echo "***** NOTE: OVERRIDING CHOSEN COMMIT USING \$BRANCH = $BRANCH *****"
+ git checkout $BRANCH
+fi
+
+if [[ -n "$TAG" ]]; then
+ echo "***** NOTE: OVERRIDING CHOSEN COMMIT USING \$TAG = $TAG *****"
+ git checkout $TAG
+fi
+
+if [[ -n "$COMMIT" ]]; then
+ echo "***** NOTE: OVERRIDING CHOSEN COMMIT USING \$COMMIT = $COMMIT *****"
+ git checkout $COMMIT
+fi
+
+# Deal with special setup, copy binary drivers etc.
+if [[ "$TARGET" == "r-car-m3-starter-kit" || "$TARGET" == "r-car-h3-starter-kit" ]]; then
+ cd meta-renesas
+ meta-rcar-gen3/docs/sample/copyscript/copy_evaproprietary_softwares.sh /var/go/sgx_bin_gen3/
+ cd -
+fi
+
+if [[ "$GFX_MACHINE" == "porter" || "$GFX_MACHINE" == silk ]]; then
+ echo "Copying binary graphics drivers for $GFX_MACHINE"
+ cd meta-renesas/meta-rcar-gen2
+ ./copy_gfx_software_$GFX_MACHINE.sh /var/go/sgx_bin
+ ./copy_mm_software_lcb.sh /var/go/sgx_bin/
+ cd -
+fi
+
+# INIT
+cd "$BASEDIR"
+echo "Running init.sh"
+
+# By using this script you accept the EULA (see top of script)
+if [[ "$TARGET" == "dragonboard-410c" ]]; then
+ source ./init.sh $TARGET accept-eula -f
+else
+ source ./init.sh $TARGET -f
+fi
+
+# Do version override on sublayers (if any such overrides defined)
+cd ..
+for l in $LAYERS ; do
+ layer_override $l
+done
+cd gdp-src-build
+
+# Remind us exactly what submodules hashes are used (this is already stated by
+# go.cd when fetching materials, but materials can be overriden by FORK /
+# BRANCH / TAG / COMMIT environment variables
+echo "Submodules (after any updates or overrides):"
+git submodule status
+
+# LOCAL CONF MODIFICATIONS
+
+if [[ "$RM_WORK" == "true" ]]; then
+ append_local_conf 'INHERIT += "rm_work"'
+fi
+
+if [[ -n "$DL_DIR" ]]; then
+ append_local_conf "DL_DIR = \"$DL_DIR\""
+fi
+
+if [[ -n "$SSTATE_DIR" ]]; then
+ append_local_conf "SSTATE_DIR = \"$SSTATE_DIR\""
+fi
+
+if [[ -n "$BB_NUMBER_THREADS" ]]; then
+ append_local_conf "BB_NUMBER_THREADS = \"$BB_NUMBER_THREADS\""
+fi
+
+if [[ -n "$PARALLEL_MAKE" ]]; then
+ echo $PARALLEL_MAKE | egrep -q '^-j' || PARALLEL_MAKE="-j$PARALLEL_MAKE"
+ append_local_conf "PARALLEL_MAKE = \"$PARALLEL_MAKE\""
+fi
+
+if [[ "$SOURCE_ARCHIVE" == "true" ]]; then
+ append_local_conf 'INHERIT += "archiver"'
+ append_local_conf 'ARCHIVER_MODE[src] = "original"'
+fi
+
+if [[ "$COPY_LICENSES" == "true" ]]; then
+ append_local_conf 'COPY_LIC_DIRS = "1"'
+ append_local_conf 'COPY_LIC_MANIFEST = "1"'
+fi
+
+# The own-mirrors bbclass is generally more convenient for PREMIRRORS, but
+# this format makes it similar to the $MIRROR setup below, which needs to be
+# explicit anyhow.
+if [[ -n "$PREMIRROR" ]]; then
+ append_local_conf "
+PREMIRRORS_prepend = \"\\
+ git://.*/.* $PREMIRROR \\n \\
+ ftp://.*/.* $PREMIRROR \\n \\
+ http://.*/.* $PREMIRROR \\n \\
+ https://.*/.* $PREMIRROR \\n \\
+ \"
+"
+fi
+
+# This is the "post" mirror (i.e. checked last).
+if [[ -n "$MIRROR" ]]; then
+ append_local_conf "
+MIRRORS_append = \"\\
+ git://.*/.* $MIRROR \\n \\
+ ftp://.*/.* $MIRROR \\n \\
+ http://.*/.* $MIRROR \\n \\
+ https://.*/.* $MIRROR \\n \\
+ \"
+"
+fi
+
+if [[ "$BUILD_SDK" != "true" ]]; then
+ bitbake genivi-dev-platform
+fi
+
+if [[ "$BUILD_SDK" == "true" ]]; then
+ bitbake genivi-dev-platform-sdk
+fi
+
+if [[ "$BUILD_SDK" == "true" ]]; then
+ bitbake meta-ide-support
+fi
+
+cd "$BASEDIR"
+rm -f logs.tar logs.tar.gz
+find gdp-src-build/tmp/work \( -name "*.log" -o -name "log.*" -o -name "run.*" \) -print0 | xargs -0 tar uf logs.tar || true
+gzip logs.tar || true
+
+# Soften up the failure requirements here. Maybe sometimes
+# some things can't be staged, which is probably ok.
+set +e
+rm -rf staging
+shopt -s nullglob
+stage_artifact mv gdp-src-build/tmp/deploy/licenses
+stage_artifact mv gdp-src-build/tmp/deploy/licenses/genivi-dev-platform*/license.manifest
+stage_artifact mv gdp-src-build/tmp/deploy/sdk*
+stage_artifact cp gdp-src-build/tmp/deploy/images/*
+stage_artifact mv gdp-src-build/tmp/deploy/sources
+stage_artifact cp gdp-src-build/conf/*.conf
+stage_artifact mv logs.tar.gz
+stage_artifact cp gdp-src-build/buildhistory/images/*/glibc/genivi-dev-platform/files-in-image.txt
+stage_artifact mv gdp-src-build/buildhistory
+stage_artifact mv gdp-src-build/tmp/buildstats
+
+if [[ "$LAYER_ARCHIVE" == "true" ]]; then
+ tar cfj staging/meta-layers-snapshot.tar.bz2 meta-* poky renesas* gdp-src-build/conf
+fi
+
+set -e
+
+# Environment contains alot of variables from Go.CD that specify the built
+# version/hash, and other metadata. Let's store them for future reference.
+# and other relevant info
+cd "$BASEDIR"
+build_info_file=staging/build_info.txt
+
+env >$build_info_file
+git submodule status >>$build_info_file
+echo 'For conf , see files *.conf, and any diff below' >>$build_info_file
+git diff gdp-src-build/conf/templates/*.inc >>$build_info_file
+
+mkdir -p staging/images
+mv staging/*/{*201*ext*,*201*rootfs*,*sdimg*,*qemuboot.conf*,modules*.tgz,*hddimg*,bzImage*201*,*201*.iso,*201*.wic,*.efi,*.dtd} staging/images/ 2>/dev/null || true
+cd staging && rm -rf "$TARGET"
+cd "$BASEDIR"
+
+if [[ "$CREATE_RELEASE_DIR" == "true" ]]; then
+ set +e
+ mkdir -p release
+ echo "Moving images to release/"
+ mv staging/images release/ 2>/dev/null || true
+ echo "Moving staging/sources to release/"
+ cp -a staging/sources release/ 2>/dev/null
+ echo "Moving staging/licenses into release/"
+ mv staging/licenses release/
+ echo "Copying various metadata to release"
+ cp staging/files-in-image.txt release/ 2>/dev/null
+ cp staging/build_info.txt release/ 2>/dev/null
+ cp staging/license.manifest release/ 2>/dev/null
+ set -e
+fi
+
+set +e
+echo "Artifacts in staging/ and release/"
+ls -al staging/ release/
+echo
+echo "...in release/images/ :"
+ls -al release/images/
+set -e
+
+cleanup