diff options
Diffstat (limited to 'scripts/sstate-cache-management.sh')
| -rwxr-xr-x | scripts/sstate-cache-management.sh | 458 |
1 files changed, 0 insertions, 458 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh deleted file mode 100755 index f1706a2229..0000000000 --- a/scripts/sstate-cache-management.sh +++ /dev/null @@ -1,458 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2012 Wind River Systems, Inc. -# -# SPDX-License-Identifier: GPL-2.0-only -# - -# Global vars -cache_dir= -confirm= -fsym= -total_deleted=0 -verbose= -debug=0 - -usage () { - cat << EOF -Welcome to sstate cache management utilities. -sstate-cache-management.sh <OPTION> - -Options: - -h, --help - Display this help and exit. - - --cache-dir=<sstate cache dir> - Specify sstate cache directory, will use the environment - variable SSTATE_CACHE_DIR if it is not specified. - - --extra-archs=<arch1>,<arch2>...<archn> - Specify list of architectures which should be tested, this list - will be extended with native arch, allarch and empty arch. The - script won't be trying to generate list of available archs from - AVAILTUNES in tune files. - - --extra-layer=<layer1>,<layer2>...<layern> - Specify the layer which will be used for searching the archs, - it will search the meta and meta-* layers in the top dir by - default, and will search meta, meta-*, <layer1>, <layer2>, - ...<layern> when specified. Use "," as the separator. - - This is useless for --stamps-dir or when --extra-archs is used. - - -d, --remove-duplicated - Remove the duplicated sstate cache files of one package, only - the newest one will be kept. The duplicated sstate cache files - of one package must have the same arch, which means sstate cache - files with multiple archs are not considered duplicate. - - Conflicts with --stamps-dir. - - --stamps-dir=<dir1>,<dir2>...<dirn> - Specify the build directory's stamps directories, the sstate - cache file which IS USED by these build diretories will be KEPT, - other sstate cache files in cache-dir will be removed. Use "," - as the separator. For example: - --stamps-dir=build1/tmp/stamps,build2/tmp/stamps - - Conflicts with --remove-duplicated. - - -L, --follow-symlink - Remove both the symbol link and the destination file, default: no. - - -y, --yes - Automatic yes to prompts; assume "yes" as answer to all prompts - and run non-interactively. - - -v, --verbose - Explain what is being done. - - -D, --debug - Show debug info, repeat for more debug info. - -EOF -} - -if [ $# -lt 1 ]; then - usage - exit 0 -fi - -# Echo no files to remove -no_files () { - echo No files to remove -} - -# Echo nothing to do -do_nothing () { - echo Nothing to do -} - -# Read the input "y" -read_confirm () { - echo "$total_deleted out of $total_files files will be removed! " - if [ "$confirm" != "y" ]; then - echo "Do you want to continue (y/n)? " - while read confirm; do - [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \ - -o "$confirm" = "N" ] && break - echo "Invalid input \"$confirm\", please input 'y' or 'n': " - done - else - echo - fi -} - -# Print error information and exit. -echo_error () { - echo "ERROR: $1" >&2 - exit 1 -} - -# Generate the remove list: -# -# * Add .done/.siginfo to the remove list -# * Add destination of symlink to the remove list -# -# $1: output file, others: sstate cache file (.tgz) -gen_rmlist (){ - local rmlist_file="$1" - shift - local files="$@" - for i in $files; do - echo $i >> $rmlist_file - # Add the ".siginfo" - if [ -e $i.siginfo ]; then - echo $i.siginfo >> $rmlist_file - fi - # Add the destination of symlink - if [ -L "$i" ]; then - if [ "$fsym" = "y" ]; then - dest="`readlink -e $i`" - if [ -n "$dest" ]; then - echo $dest >> $rmlist_file - # Remove the .siginfo when .tgz is removed - if [ -f "$dest.siginfo" ]; then - echo $dest.siginfo >> $rmlist_file - fi - fi - fi - # Add the ".tgz.done" and ".siginfo.done" (may exist in the future) - base_fn="${i##/*/}" - t_fn="$base_fn.done" - s_fn="$base_fn.siginfo.done" - for d in $t_fn $s_fn; do - if [ -f $cache_dir/$d ]; then - echo $cache_dir/$d >> $rmlist_file - fi - done - fi - done -} - -# Remove the duplicated cache files for the pkg, keep the newest one -remove_duplicated () { - - local topdir - local oe_core_dir - local tunedirs - local all_archs - local all_machines - local ava_archs - local arch - local file_names - local sstate_files_list - local fn_tmp - local list_suffix=`mktemp` || exit 1 - - if [ -z "$extra_archs" ] ; then - # Find out the archs in all the layers - echo "Figuring out the archs in the layers ... " - oe_core_dir=$(dirname $(dirname $(readlink -e $0))) - topdir=$(dirname $oe_core_dir) - tunedirs="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/include' 2>/dev/null`" - [ -n "$tunedirs" ] || echo_error "Can't find the tune directory" - all_machines="`find $topdir/meta* ${oe_core_dir}/meta* $layers -path '*/meta*/conf/machine/*' -name '*.conf' 2>/dev/null | sed -e 's/.*\///' -e 's/.conf$//'`" - all_archs=`grep -r -h "^AVAILTUNES .*=" $tunedirs | sed -e 's/.*=//' -e 's/\"//g'` - fi - - # Use the "_" to substitute "-", e.g., x86-64 to x86_64, but not for extra_archs which can be something like cortexa9t2-vfp-neon - # Sort to remove the duplicated ones - # Add allarch and builder arch (native) - builder_arch=$(uname -m) - all_archs="$(echo allarch $all_archs $all_machines $builder_arch \ - | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs" - echo "Done" - - # Total number of files including sstate-, .siginfo and .done files - total_files=`find $cache_dir -name 'sstate*' | wc -l` - # Save all the sstate files in a file - sstate_files_list=`mktemp` || exit 1 - find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list - - echo "Figuring out the suffixes in the sstate cache dir ... " - sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`" - echo "Done" - echo "The following suffixes have been found in the cache dir:" - echo $sstate_suffixes - - echo "Figuring out the archs in the sstate cache dir ... " - # Using this SSTATE_PKGSPEC definition it's 6th colon separated field - # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" - for arch in $all_archs; do - grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list - [ $? -eq 0 ] && ava_archs="$ava_archs $arch" - # ${builder_arch}_$arch used by toolchain sstate - grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list - [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch" - done - echo "Done" - echo "The following archs have been found in the cache dir:" - echo $ava_archs - echo "" - - # Save the file list which needs to be removed - local remove_listdir=`mktemp -d` || exit 1 - for suffix in $sstate_suffixes; do - if [ "$suffix" = "populate_lic" ] ; then - echo "Skipping populate_lic, because removing duplicates doesn't work correctly for them (use --stamps-dir instead)" - continue - fi - # Total number of files including .siginfo and .done files - total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list | wc -l 2>/dev/null` - total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz$" $sstate_files_list | wc -l 2>/dev/null` - # Save the file list to a file, some suffix's file may not exist - grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix\.tgz.*" $sstate_files_list >$list_suffix 2>/dev/null - local deleted_tgz=0 - local deleted_files=0 - for ext in tgz tgz.siginfo tgz.done; do - echo "Figuring out the sstate:xxx_$suffix.$ext ... " - # Uniq BPNs - file_names=`for arch in $ava_archs ""; do - sed -ne "s%.*/sstate:\([^:]*\):[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$%\1%p" $list_suffix - done | sort -u` - - fn_tmp=`mktemp` || exit 1 - rm_list="$remove_listdir/sstate:xxx_$suffix" - for fn in $file_names; do - [ -z "$verbose" ] || echo "Analyzing sstate:$fn-xxx_$suffix.${ext}" - for arch in $ava_archs ""; do - grep -h ".*/sstate:$fn:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.${ext}$" $list_suffix >$fn_tmp - if [ -s $fn_tmp ] ; then - [ $debug -gt 1 ] && echo "Available files for $fn-$arch- with suffix $suffix.${ext}:" && cat $fn_tmp - # Use the modification time - to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p') - [ $debug -gt 2 ] && echo "Considering to delete: $to_del" - # The sstate file which is downloaded from the SSTATE_MIRROR is - # put in SSTATE_DIR, and there is a symlink in SSTATE_DIR/??/ to - # it, so filter it out from the remove list if it should not be - # removed. - to_keep=$(ls -t $(cat $fn_tmp) | sed -n '1p') - [ $debug -gt 2 ] && echo "Considering to keep: $to_keep" - for k in $to_keep; do - if [ -L "$k" ]; then - # The symlink's destination - k_dest="`readlink -e $k`" - # Maybe it is the one in cache_dir - k_maybe="$cache_dir/${k##/*/}" - # Remove it from the remove list if they are the same. - if [ "$k_dest" = "$k_maybe" ]; then - to_del="`echo $to_del | sed 's#'\"$k_maybe\"'##g'`" - fi - fi - done - rm -f $fn_tmp - [ $debug -gt 2 ] && echo "Decided to delete: $to_del" - gen_rmlist $rm_list.$ext "$to_del" - fi - done - done - done - deleted_tgz=`cat $rm_list.* 2>/dev/null | grep ".tgz$" | wc -l` - deleted_files=`cat $rm_list.* 2>/dev/null | wc -l` - [ "$deleted_files" -gt 0 -a $debug -gt 0 ] && cat $rm_list.* - echo "($deleted_tgz out of $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files out of $total_files_suffix when counting also .siginfo and .done files)" - let total_deleted=$total_deleted+$deleted_files - done - deleted_tgz=0 - rm_old_list=$remove_listdir/sstate-old-filenames - find $cache_dir -name 'sstate-*.tgz' >$rm_old_list - [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l` - [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l` - [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list - echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)" - let total_deleted=$total_deleted+$deleted_files - - rm -f $list_suffix - rm -f $sstate_files_list - if [ $total_deleted -gt 0 ]; then - read_confirm - if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then - for list in `ls $remove_listdir/`; do - echo "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... " - # Remove them one by one to avoid the argument list too long error - for i in `cat $remove_listdir/$list`; do - rm -f $verbose $i - done - echo "Done" - done - echo "$total_deleted files have been removed!" - else - do_nothing - fi - else - no_files - fi - [ -d $remove_listdir ] && rm -fr $remove_listdir -} - -# Remove the sstate file by stamps dir, the file not used by the stamps dir -# will be removed. -rm_by_stamps (){ - - local cache_list=`mktemp` || exit 1 - local keep_list=`mktemp` || exit 1 - local rm_list=`mktemp` || exit 1 - local sums - local all_sums - - # Total number of files including sstate-, .siginfo and .done files - total_files=`find $cache_dir -type f -name 'sstate*' | wc -l` - # Save all the state file list to a file - find $cache_dir -type f -name 'sstate*' | sort -u -o $cache_list - - echo "Figuring out the suffixes in the sstate cache dir ... " - local sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $cache_list | sort -u`" - echo "Done" - echo "The following suffixes have been found in the cache dir:" - echo $sstate_suffixes - - # Figure out all the md5sums in the stamps dir. - echo "Figuring out all the md5sums in stamps dir ... " - for i in $sstate_suffixes; do - # There is no "\.sigdata" but "_setcene" when it is mirrored - # from the SSTATE_MIRRORS, use them to figure out the sum. - sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \ - -o -name "*.do_${i}_setscene.*" | \ - sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \ - sed -e 's#\..*##' | sort -u` - all_sums="$all_sums $sums" - done - echo "Done" - - echo "Figuring out the files which will be removed ... " - for i in $all_sums; do - grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:${i}_.*" $cache_list >>$keep_list - done - echo "Done" - - if [ -s $keep_list ]; then - sort -u $keep_list -o $keep_list - to_del=`comm -1 -3 $keep_list $cache_list` - gen_rmlist $rm_list "$to_del" - let total_deleted=`cat $rm_list | sort -u | wc -w` - if [ $total_deleted -gt 0 ]; then - [ $debug -gt 0 ] && cat $rm_list | sort -u - read_confirm - if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then - echo "Removing sstate cache files ... ($total_deleted files)" - # Remove them one by one to avoid the argument list too long error - for i in `cat $rm_list | sort -u`; do - rm -f $verbose $i - done - echo "$total_deleted files have been removed" - else - do_nothing - fi - else - no_files - fi - else - echo_error "All files in cache dir will be removed! Abort!" - fi - - rm -f $cache_list - rm -f $keep_list - rm -f $rm_list -} - -# Parse arguments -while [ -n "$1" ]; do - case $1 in - --cache-dir=*) - cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e` - [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir" - shift - ;; - --remove-duplicated|-d) - rm_duplicated="y" - shift - ;; - --yes|-y) - confirm="y" - shift - ;; - --follow-symlink|-L) - fsym="y" - shift - ;; - --extra-archs=*) - extra_archs=`echo $1 | sed -e 's#^--extra-archs=##' -e 's#,# #g'` - [ -n "$extra_archs" ] || echo_error "Invalid extra arch parameter" - shift - ;; - --extra-layer=*) - extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'` - [ -n "$extra_layers" ] || echo_error "Invalid extra layer parameter" - for i in $extra_layers; do - l=`readlink -e $i` - if [ -d "$l" ]; then - layers="$layers $l" - else - echo_error "Can't find layer $i" - fi - done - shift - ;; - --stamps-dir=*) - stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'` - [ -n "$stamps" ] || echo_error "Invalid stamps dir $i" - for i in $stamps; do - [ -d "$i" ] || echo_error "Invalid stamps dir $i" - done - shift - ;; - --verbose|-v) - verbose="-v" - shift - ;; - --debug|-D) - debug=`expr $debug + 1` - echo "Debug level $debug" - shift - ;; - --help|-h) - usage - exit 0 - ;; - *) - echo "Invalid arguments $*" - echo_error "Try 'sstate-cache-management.sh -h' for more information." - ;; - esac -done - -# sstate cache directory, use environment variable SSTATE_CACHE_DIR -# if it was not specified, otherwise, error. -[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR -[ -n "$cache_dir" ] || echo_error "No cache dir found!" -[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\"" - -[ -n "$rm_duplicated" -a -n "$stamps" ] && \ - echo_error "Can not use both --remove-duplicated and --stamps-dir" - -[ "$rm_duplicated" = "y" ] && remove_duplicated -[ -n "$stamps" ] && rm_by_stamps -[ -z "$rm_duplicated" -a -z "$stamps" ] && \ - echo "What do you want to do?" -exit 0 |