#!/bin/bash
#
# This script used to update Yocto Packages Report System database.
#
# This script will also send two reminding mails about recipe upgrade to mail list after database update successful, otherwise, send mail to system maint# ainer.
#
# We can decide whether we need to draw a barchart of history record or send report mail to a specific mail address.
#
#/***************************************************************************
# * This software is licensed as described in the file COPYING, which
# * you should have received as part of this distribution. 
# *
# * You may opt to use, copy, modify, merge, publish, distribute and/or sell
# * copies of the Software, and permit persons to whom the Software is
# * furnished to do so, under the terms of the COPYING file.
# *
# ***************************************************************************/



function usage() {
	echo "Usage: $0 <dailycheck/weeklycheck> <mailaddr> <drawchart> <milestone_start> <milestone_end>"
	echo ""
	echo "dailycheck/weeklycheck: dailycheck only check the local recipe version change and record the change time, weeklycheck will update package"
	echo "information database and send out check reports"
	echo ""
	echo "mailaddr: Report mail or Warning mail will send to mailaddr instead of admin mailbox"
	echo ""
	echo "drawchart: Barchart will be generated if you add the argument \"drawchart\" "
	echo ""
	echo "Without arguments, this script will send mail to default maillist and admin and execute without generate barchart"
	echo ""
	exit 1
}

if [ $# -gt 5 ]; then
	echo " Number of argument isn't right!"
	usage
fi

if [ -n "$1" ]; then
	if [ "$1" != "dailycheck" -a "$1" != "weeklycheck" ]; then
		echo "First augument must be dailycheck or weeklycheck!"
		usage
	else
		checkfrq="$1"
	fi
fi


if [ -n "$2" ]; then
	if ! [ `echo $2 | grep "@"` ]; then
		echo "Mail address isn't right!"
		usage
	else
		sendemailto="$2"
	fi
fi

if [ -n "$3" ]; then
	if [ "$3" != "drawchart" ]; then
		echo "The second argument must be "drawchart"!"
		usage
	else
		drawchart="$3"
	fi
fi

#
#format check
#

milestone_start="$4"
milestone_end="$5"

function drawchart () {
	#
	#This function will draw a barchart contains recipe upgrade history number and manual check history number
	#
	cd $FunctionHome
	python showchart.py >> $Log 2>&1
	python history_upgrade_number_chart.py >> $Log 2>&1
	drawchart_result=$?
	if [ $drawchart_result -eq 1 ]; then
		echo "REPORT-SYSTEM-UPDATE ERROR: Draw barchart faild!" >> $Log
	fi
}

function check_dirstro_format () {
	#
	#This function will check distro_tracking_fields.inc format issue
	# 
	cd $FunctionHome
	python distro_tracking_format_check.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: distro_tracking_format_check.py faild!" >> $Log
}


function collect_history_data_for_chart() {
	cd $LogRestore/
	echo $date >> HistoryDataPre.log
	grep "Upgradable Count:" upgradecontent.log | awk -F: '{print $2}' >> HistoryDataPre.log
	grep "Manual Need Check Count:" lastchktime.log | awk -F: '{print $2}' >> HistoryDataPre.log
	tail -30 HistoryDataPre.log > HistoryData.log
}

function bitbake_engine () {
	bitbake universe -c $1 -k >> $Log 2>&1
	cp $LogHome/$2 $LogRestore
}

function python_engine () {
	cd $FunctionHome
	python $1 1>$LogRestore/$2 2>>$Log || echo "REPORT-SYSTEM-UPDATE ERROR: Run $1 faild!" >> $Log
}

function sendreportmail_upgrade () {
	cd $LogRestore
	if [ -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -a weekly_upgrde_history_chart.png potential_format_problem.log  -- $2
	else
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -a weekly_upgrde_history_chart.png  -- $2
	fi
}

function sendreportmail_upgrade_no_chart () {
	cd $LogRestore
	if [ -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -a potential_format_problem.log -- $2
	else
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -- $2
	fi
}


function sendreportmail_manualcheck () {
	cd $LogRestore
	echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Manual check recipes name list" -- $2
}

function sendlogmail () {
	cd $LogRestore
	if [ -s block_update_process_format_problem.log -a -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -a block_update_process_format_problem.log potential_format_problem.log -- $2
	elif [ -s block_update_process_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -a block_update_process_format_problem.log -- $2
	elif [ -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -a potential_format_problem.log -- $2
	else
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -- $2
	fi
}

function check_local_version_change () {
	#
	#This function will record the local version change and change time into table pkgrecord in pkginfobackup.sqlite
	#
	#Collect all the recipes local version information via bitbake
	#
	cd $PokyHome
	. ./oe-init-build-env build
	bitbake -s | sed '/^WARN/d;/^Load/d;/^NOTE:/d;/^Parsing/d;/^Pack/d;/^====/d;/^[Dd]one/d;/^$/d' | awk '{printf("%s,%s\n", $1, $2)}' | sed -e 's/,.*:/,/;s/-r[0-9].*$//' > $LogRestore/recipe_name_version.log.$date
	ln -fs $LogRestore/recipe_name_version.log.$date $LogRestore/recipe_name_version.log

	#
	#Use dailycheck_to_db.py to collect all the recipes information to daily_check.sqlite daily
	#
	cd $FunctionHome
	python dailycheck_to_db.py >> $Log || echo "REPORT-SYSTEM-UPDATE ERROR: Run dailycheck_to_db.py faild!" >> $Log
	cd $WebHome/newserver/webserver
	cp daily_check.sqlite $LogRestore/daily_check.sqlite.$date


}


#
#you can set your own environment variable
#
date="`date +%Y-%m-%d`"
export WebHome=$(cd `dirname $0` && pwd)
export PokyHome="$WebHome/poky/"
export LogHome="$PokyHome/build/tmp/log/"
export ConfigFile="$WebHome/pokyconfig/"
export LogRestore="$WebHome/log/"
export FunctionHome="$WebHome/python-function/"
export Log="$LogRestore/run_getpkginfoall.log.$date"



proxyenv=`env | grep proxy | xargs`
for proxyname in $proxyenv
do
        export $proxyname
        cmd="export $proxyname"
        echo $cmd >> $Log
done

echo "$milestone_start" > $LogRestore/milestone_date
echo "$milestone_end" >> $LogRestore/milestone_date
#For index.py can load it
cp $LogRestore/milestone_date $WebHome/newserver/webserver/

#
#generate log file
#
if [ -e $LogRestore/run_getpkginfoall.log ]; then
	rm -f $LogRestore/run_getpkginfoall.log
fi
if [ -e $Log ]; then
	rm -f $Log
fi
touch $LogRestore/run_getpkginfoall.log.$date
ln -sf $Log $LogRestore/run_getpkginfoall.log

#
#Get poky tree
#
cd $WebHome
if [ ! -d poky ]; then
	git clone git://git.yoctoproject.org/poky.git >> $Log 2>&1
	if [ $? -eq 1 ]; then
		echo "REPORT-SYSTEM-UPDATE ERROR: Get a new poky tree failed!" >> $Log
	fi
fi

cd $PokyHome
#Checkout to master
#USE_MASTER="master"
if  [ -n "$USE_MASTER" ]; then
        git checkout master
        git pull -f git://git.yoctoproject.org/poky.git master:master >> $Log 2>&1 
        if [ $? -eq 1 ]; then
                echo "REPORT-SYSTEM-UPDATE ERROR: Get the latest poky tree failed" >> $Log
        fi
else
	git checkout master
	git branch -D stage
        git fetch git://git.yoctoproject.org/poky-contrib.git stage/master_under_test:stage >> $Log 2>&1
        git checkout stage
        if [ $? -eq 1 ]; then
                echo "REPORT-SYSTEM-UPDATE ERROR: Get the latest poky tree failed" >> $Log
        fi
fi

cd $PokyHome
rm -rf build/
. ./oe-init-build-env build
cp  $ConfigFile/auto.conf ./conf/

#
#get original data from poky tasks
#
rm -rf $PokyHome/build/tmp/log/*
bitbake_engine "distro_check" "distrocheck.csv"
bitbake_engine "checkpkg" "checkpkg.csv"
bitbake_engine "checklicense" "missinglicense.csv"

#
#handle data so that system can use them smoothly
#
python_engine "upgrade_to_log.py" "upgradeinfo.log"
python_engine "lastchktime_to_log.py" "lastchktimeinfo.log"
python_engine "noupgreason.py" "noupgreasoninfo.log"
python_engine "checkdistro.py" "distroexist.log"
python_engine "checklicense.py" "licensecheck.log"
check_dirstro_format

#
#Also need to check local version change, do not worry about repeat issue with daily check task, upgrade_history_to_db.py will abandon repeat data
#
check_local_version_change

#
#run recipe_repo.py to get recipe repo address
#
cd $PokyHome
find meta* -iname *.bb>$LogRestore/recipelist.log  
find meta* -iname *.bb | awk -F/ '{print $NF}'| awk -F ".bb" '{print $1}'>$LogRestore/recipename.log
find meta* -iname *.bb | awk -F/ '{print $NF}'| awk -F ".bb" '{print $1}' | awk -F_ '{print $1}'>$LogRestore/pkgPureName.log
cd $FunctionHome
python recipe_repo.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Run recipe_repo.py faild!" >> $Log

#
#run info_to_db.py to update the database
#
cd $WebHome/newserver/webserver
if [ -e pkginfo.sqlite ]; then
	rm -f pkginfo.sqlite
fi
if [ -e $LogRestore/pkginfo.sqlite.$date ]; then
	rm -f $LogRestore/pkginfo.sqlite.$date
fi
touch $LogRestore/pkginfo.sqlite.$date
ln -sf $LogRestore/pkginfo.sqlite.$date $WebHome/newserver/webserver/pkginfo.sqlite
cd $FunctionHome
python info_to_db.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Update database faild!" >> $Log

#
#Use upgrade_history_to_db.py to get the latest version change of recipe to pkgrecord
#
cd $FunctionHome
python upgrade_history_to_db.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Run upgrade_history_to_db.py faild!" >> $Log

#
#run milestone_target_to_db.py to create table milestone_target for milestone target track
#
python milestone_target_to_db.py $milestone_start >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: run milestone_target_to_db.py faild! Please Check target format! " >> $Log


#
#put the sqlite data to a csv spreadsheet
#
cd $WebHome/newserver/webserver
if [ -e report.csv ]; then
	rm -f report.csv
fi
if [ -e report.csv.$date ]; then
	rm -f report.csv.$date
fi
touch $LogRestore/report.csv.$date
ln -sf $LogRestore/report.csv.$date report.csv
cd $FunctionHome
python pkginfo_to_csv.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Run pkginfo_to_csv.py faild!" >> $Log

#
#check recipes version and give out the list mail which need to be updated
#
cd $FunctionHome
rm $LogRestore/upgradecontent.log $LogRestore/lastchktime.log
python mailaboutupgrade.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Mail content of upgrade info failed!" >> $Log
python mailaboutchktime.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Mail content of check time failed!" >> $Log

#
#get based commit info
#
cd $PokyHome
echo "The based commit merge time is:" >> $LogRestore/upgradecontent.log
git log --format=%cd -1 >> $LogRestore/upgradecontent.log
echo "The based commit is:" >> $LogRestore/upgradecontent.log
git log -1 | head -n 3 >> $LogRestore/upgradecontent.log

##
##Collect history date data for select date in history page 
##
#cd $LogRestore/
#ls pkginfo.sqlite.* | awk -F . '{print $(NF)}' > databaseHistoryDate.log || echo "REPORT-SYSTEM-UPDATE ERROR: Collect history date faild!" >> $Log

#
#Collect data for bar chart then draw bar chart
#
if [ "$drawchart" = "drawchart" ]; then
	cd $FunctionHome
	python checklog.py
	if [ $? -eq 0 ]; then
		collect_history_data_for_chart
		drawchart
		if [ $drawchart_result -eq 0 ]; then
			cd $LogRestore
			cp $WebHome/newserver/webserver/images/weekly_upgrde_history_chart.png weekly_upgrde_history_chart.png.$date
			ln -sf weekly_upgrde_history_chart.png.$date weekly_upgrde_history_chart.png
		fi
	else
		echo "REPORT-SYSTEM-UPDATE ERROR: Didn't draw chart because some failures happend before" >> $Log
	fi
fi


#Replace the old database with latest one if no ERROR reported in the log
cd $FunctionHome
python checklog.py
if [ $? -eq 0 ]; then
	cd $WebHome/newserver/webserver
	if [ -e pkginfobackup.sqlite ]; then
		rm -f pkginfobackup.sqlite
	fi
	cp pkginfo.sqlite pkginfobackup.sqlite 
	cp report.csv $WebHome
else
	if [ -n "$sendemailto" ]; then
		sendlogmail "$Log" "$sendemailto"
	else
		sendlogmail "$Log" "saul.wold@intel.com"
	fi
fi

#If the checkfrq is dailycheck, we won't send a report, we will exit this script 
if [ "$checkfrq" = "dailycheck" ]; then
	exit 0
fi

#
#use mutt to send list of recipes which need to be updated 
#
cd $FunctionHome
python checklog.py
if [ $? -eq 0 ]; then
	#Send report mail, divide into two conditions, with chart as an attachment or without chart 
	if [ -n "$sendemailto" ]; then
		if [ $drawchart_result -eq 0 ]; then
			sendreportmail_upgrade "upgradecontent.log" "$sendemailto"
		else
			sendreportmail_upgrade_no_chart "upgradecontent.log" "$sendemailto"
		fi
		sendreportmail_manualcheck "lastchktime.log" "$sendemailto"
		if [ $? -eq 1 ]; then
			echo "REPORT-SYSTEM-UPDATE ERROR: Send reportmail faild!" >> $Log
		fi
	else
		if [ $drawchart_result -eq 0 ]; then
			sendreportmail_upgrade "upgradecontent.log" "yocto@linux.intel.com"
		else
			sendreportmail_upgrade_no_chart "upgradecontent.log" "yocto@linux.intel.com"
		sendreportmail_manualcheck "lastchktime.log" "yocto@linux.intel.com"
		fi
		if [ $? -eq 1 ]; then
			echo "REPORT-SYSTEM-UPDATE ERROR: Send reportmail faild!" >> $Log
		fi
	fi
else
	if [ -n "$sendemailto" ]; then
		sendlogmail "$Log" "$sendemailto"
	else
		sendlogmail "$Log" "saul.wold@intel.com"
	fi
	if [ $? -eq 1 ]; then
		echo "REPORT-SYSTEM-UPDATE ERROR: Send logmail faild!" >> $Log
	fi
fi
echo "All tasks finished! You can check the log file $Log" >> $Log