summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/sstate-cache-management.sh289
1 files changed, 240 insertions, 49 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index d0e3abc820..be185bbb46 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -16,22 +16,54 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
+# Global vars
+cache_dir=
+confirm=
+total_deleted=0
+verbose=
+
usage () {
cat << EOF
Welcome to sstate cache management utilities.
sstate-cache-management.sh <OPTION>
Options:
- --help, -h
+ -h, --help
Display this help and exit.
--cache-dir=<sstate cache dir>
Specify sstate cache directory, will use the environment
variable SSTATE_CACHE_DIR if it is not specified.
- --remove-duplicated
+ --extra-layer=<layer1>,<layer2>...<layern>
+ Specify the layer which will be used for searching the archs,
+ it will search the meta and meta-* layers in the top dir by
+ default, and will search meta, meta-*, <layer1>, <layer2>,
+ ...<layern> when specified. Use "," as the separator.
+
+ This is useless for --stamps-dir.
+
+ -d, --remove-duplicated
Remove the duplicated sstate cache files of one package, only
- the newest one would be kept.
+ the newest one will be kept.
+
+ Conflicts with --stamps-dir.
+
+ --stamps-dir=<dir1>,<dir2>...<dirn>
+ Specify the build directory's stamps directories, the sstate
+ cache file which IS USED by these build diretories will be KEPT,
+ other sstate cache files in cache-dir will be removed. Use ","
+ as the separator. For example:
+ --stamps-dir=build1/tmp/stamps,build2/tmp/stamps
+
+ Conflicts with --remove-duplicated.
+
+ -y, --yes
+ Automatic yes to prompts; assume "yes" as answer to all prompts
+ and run non-interactively.
+
+ -v, --verbose
+ explain what is being done
EOF
}
@@ -41,6 +73,31 @@ if [ $# -lt 1 ]; then
exit 0
fi
+# Echo no files to remove
+no_files () {
+ echo No files to remove
+}
+
+# Echo nothing to do
+do_nothing () {
+ echo Nothing to do
+}
+
+# Read the input "y"
+read_confirm () {
+ echo -n "$total_deleted files will be removed! "
+ if [ "$confirm" != "y" ]; then
+ echo -n "Do you want to continue (y/n)? "
+ while read confirm; do
+ [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
+ -o "$confirm" = "N" ] && break
+ echo -n "Invalid input \"$confirm\", please input 'y' or 'n': "
+ done
+ else
+ echo
+ fi
+}
+
# Print error information and exit.
echo_error () {
echo "ERROR: $1" >&2
@@ -49,45 +106,197 @@ echo_error () {
# Remove the duplicated cache files for the pkg, keep the newest one
remove_duplicated () {
- local all_suffixes="$1"
- local ava_archs="$2"
- local total_deleted=0
- for suffix in $all_suffixes; do
+
+ local topdir
+ local tunedirs
+ local all_archs
+ local ava_archs
+ local arch
+ local file_names
+ local sstate_list
+ local fn_tmp
+ local list_suffix=`mktemp` || exit 1
+
+ # Find out the archs in all the layers
+ echo -n "Figuring out the archs in the layers ... "
+ topdir=$(dirname $(dirname $(readlink -e $0)))
+ tunedirs="`find $topdir/meta* $layers -path '*/meta*/conf/machine/include'`"
+ [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
+ all_archs=`grep -r -h "^AVAILTUNES " $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
+ # Add the qemu and native archs
+ # Use the "_" to substitute "-", e.g., x86-64 to x86_64
+ # Sort to remove the duplicated ones
+ all_archs=$(echo $all_archs qemuarm qemux86 qemumips qemuppc qemux86_64 $(uname -m) \
+ | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
+ echo "Done"
+
+ sstate_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot"
+
+ cd $cache_dir || exit 1
+ # Save all the sstate files in a file
+ sstate_list=`mktemp` || exit 1
+ ls sstate-*.tgz >$sstate_list
+ echo -n "Figuring out the archs in the sstate cache dir ... "
+ for arch in $all_archs; do
+ grep -q -w $arch $sstate_list
+ [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
+ done
+ echo "Done"
+ echo "The following archs have been found in the cache dir:"
+ echo $ava_archs
+ echo ""
+
+ # Save the file list which needs to be removed
+ local remove_listdir=`mktemp -d` || exit 1
+
+ for suffix in $sstate_suffixes; do
+ # Save the file list to a file, some suffix's file may not exist
+ ls *_$suffix.tgz >$list_suffix 2>/dev/null
local deleted=0
- echo -n "Removing the sstate-xxx_$suffix.tgz ... "
- # sed twice to avoid the greedy match
+ echo -n "Figuring out the sstate-xxx_$suffix.tgz ... "
+ # There are at list 6 dashes (-) after arch, use this to avoid the
+ # greedy match of sed.
file_names=`for arch in $ava_archs; do
- ls sstate-*-$arch-*_$suffix.tgz 2>/dev/null | \
- sed -e 's/\(.*\)-'"$arch"'-.*/\1/' \
- -e 's/\(.*\)-'"$arch"'-.*/\1/'
+ sed -ne 's/^\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*/\1/p' $list_suffix
done | sort -u`
+ fn_tmp=`mktemp` || exit 1
for fn in $file_names; do
+ [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
for arch in $ava_archs; do
- ls $fn-$arch-*_$suffix.tgz 2>/dev/null >>/tmp/$fn
+ grep -h "^$fn-$arch-" $list_suffix >>$fn_tmp
done
- # Also delete the .siginfo file
- to_del=$(ls -t $(cat /tmp/$fn) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/')
- rm -f $to_del
+ # Use the access time, also delete the .siginfo file
+ to_del=$(ls -u $(cat $fn_tmp) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/')
+ echo $to_del >>$remove_listdir/sstate-xxx_$suffix
let deleted=$deleted+`echo $to_del | wc -w`
- rm -f /tmp/$fn
+ rm -f $fn_tmp
done
- echo "($deleted files)"
+ echo "($deleted files will be removed)"
let total_deleted=$total_deleted+$deleted
done
- echo "$total_deleted files have been removed"
+ rm -f $list_suffix
+ if [ $total_deleted -gt 0 ]; then
+ read_confirm
+ if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
+ for list in `ls $remove_listdir/`; do
+ echo -n "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
+ rm -f $verbose `cat $remove_listdir/$list`
+ echo "Done"
+ done
+ echo "$total_deleted files have been removed!"
+ else
+ do_nothing
+ fi
+ else
+ no_files
+ fi
+ [ -d $remove_listdir ] && rm -fr $remove_listdir
+}
+
+# Remove the sstate file by stamps dir, the file not used by the stamps dir
+# will be removed.
+rm_by_stamps (){
+
+ local cache_list=`mktemp` || exit 1
+ local keep_list=`mktemp` || exit 1
+ local mv_to_dir=`mktemp -d -p $cache_dir` || exit 1
+ local suffixes
+ local sums
+ local all_sums
+
+ suffixes="populate_sysroot populate_lic package_write_ipk \
+ package_write_rpm package_write_deb package deploy"
+
+ # Figure out all the md5sums in the stamps dir.
+ echo -n "Figuring out all the md5sums in stamps dir ... "
+ for i in $suffixes; do
+ sums=`find $stamps -maxdepth 2 -name "*\.do_$i\.sigdata.*" | \
+ sed 's#.*\.sigdata\.##' | sort -u`
+ all_sums="$all_sums $sums"
+ done
+ echo "Done"
+
+ # Save all the state file list to a file
+ ls $cache_dir/sstate-*.tgz >$cache_list
+
+ echo -n "Figuring out the files which will be removed ... "
+ for i in $all_sums; do
+ grep ".*-$i.*" $cache_list >>$keep_list
+ done
+ echo "Done"
+
+ if [ -s $keep_list ]; then
+ let total_deleted=(`cat $cache_list | wc -w` - `cat $keep_list | wc -l`)*2
+
+ if [ $total_deleted -gt 0 ]; then
+ read_confirm
+ if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
+ echo "Removing sstate cache files ... ($total_deleted files)"
+ # Save the file which needs to be kept, remove the others,
+ # then move it back
+ for i in `cat $keep_list`; do
+ mv $i $mv_to_dir
+ mv $i.siginfo $mv_to_dir || true
+ done
+ rm -f $verbose $cache_dir/sstate-*.tgz
+ rm -f $verbose $cache_dir/sstate-*.tgz.siginfo
+ mv $mv_to_dir/* $cache_dir/
+ echo "$total_deleted files have been removed"
+ else
+ do_nothing
+ fi
+ else
+ no_files
+ fi
+ else
+ echo_error "All files in cache dir will be removed! Abort!"
+ fi
+
+ rm -f $cache_list
+ rm -f $keep_list
+ rmdir $mv_to_dir
}
# Parse arguments
while [ -n "$1" ]; do
case $1 in
--cache-dir=*)
- cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' -e 's#/*$##' | xargs readlink -f`
+ cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
[ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
shift
;;
- --remove-duplicated)
- rm_duplicated="yes"
+ --remove-duplicated|-d)
+ rm_duplicated="y"
+ shift
+ ;;
+ --yes|-y)
+ confirm="y"
+ shift
+ ;;
+ --extra-layer=*)
+ extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
+ [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i"
+ for i in $extra_layers; do
+ l=`readlink -e $i`
+ if [ -d "$l" ]; then
+ layers="$layers $l"
+ else
+ echo_error "Can't find layer $i"
+ fi
+ done
+ shift
+ ;;
+ --stamps-dir=*)
+ stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
+ [ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
+ for i in $stamps; do
+ [ -d "$i" ] || echo_error "Invalid stamps dir $i"
+ done
+ shift
+ ;;
+ --verbose|-v)
+ verbose="-v"
shift
;;
--help|-h)
@@ -95,8 +304,8 @@ while [ -n "$1" ]; do
exit 0
;;
*)
- echo "Invalid arguments $*"
- echo_error "Try 'sstate-cache-management.sh -h' for more information."
+ echo "Invalid arguments $*"
+ echo_error "Try 'sstate-cache-management.sh -h' for more information."
;;
esac
done
@@ -104,32 +313,14 @@ done
# sstate cache directory, use environment variable SSTATE_CACHE_DIR
# if it was not specified, otherwise, error.
[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
+[ -n "$cache_dir" ] || echo_error "No cache dir found!"
[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
-cache_dir=`readlink -f $cache_dir`
+[ -n "$rm_duplicated" -a -n "$stamps" ] && \
+ echo_error "Can not use both --remove-duplicated and --stamps-dir"
-topdir=$(dirname $(dirname $(readlink -f $0)))
-tunedir=$topdir/meta/conf/machine/include
-[ -d $tunedir ] || echo_error "Can't find the tune directory"
+[ "$rm_duplicated" = "y" ] && remove_duplicated
+[ -n "$stamps" ] && rm_by_stamps
+[ -z "$rm_duplicated" -a -z "$stamps" ] && \
+ echo "What do you want to do?"
-# Use the "_" to substitute "-", e.g., x86-64 to x86_64
-all_archs=`grep -r DEFAULTTUNE $tunedir | \
- sed -e 's/.*\"\(.*\)\"/\1/' -e 's/-/_/g' | sort -u`
-# Add the qemu archs
-all_archs="$all_archs qemuarm qemux86 qemumips qemuppc"
-
-all_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot"
-
-cd $cache_dir
-
-echo "Figuring out the archs in the sstate cache dir ..."
-for arch in $all_archs; do
- ls | grep -q -w $arch
- [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
-done
-echo "The following archs have been found in the sstate cache dir:"
-echo $ava_archs
-
-if [ "$rm_duplicated" == "yes" -a -n "$ava_archs" ]; then
- remove_duplicated "$all_suffixes" "$ava_archs"
-fi