aboutsummaryrefslogtreecommitdiffstats
path: root/getallpkginfo.bash
blob: 73de567537bc5fb1db36130e25c5d5ddbab23fe7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
#!/bin/bash
#
# This script used to update Yocto Packages Report System database.
#
# This script will also send two reminding mails about recipe upgrade to mail list after database update successful, otherwise, send mail to system maint# ainer.
#
# We can decide whether we need to draw a barchart of history record or send report mail to a specific mail address.
#
#/***************************************************************************
# * This software is licensed as described in the file COPYING, which
# * you should have received as part of this distribution. 
# *
# * You may opt to use, copy, modify, merge, publish, distribute and/or sell
# * copies of the Software, and permit persons to whom the Software is
# * furnished to do so, under the terms of the COPYING file.
# *
# ***************************************************************************/



function usage() {
	echo "Usage: $0 <dailycheck/weeklycheck> <mailaddr> <drawchart> <milestone_start> <milestone_end>"
	echo ""
	echo "dailycheck/weeklycheck: dailycheck only check the local recipe version change and record the change time, weeklycheck will update package"
	echo "information database and send out check reports"
	echo ""
	echo "mailaddr: Report mail or Warning mail will send to mailaddr instead of admin mailbox"
	echo ""
	echo "drawchart: Barchart will be generated if you add the argument \"drawchart\" "
	echo ""
	echo "Without arguments, this script will send mail to default maillist and admin and execute without generate barchart"
	echo ""
	exit 1
}

if [ $# -gt 5 ]; then
	echo " Number of argument isn't right!"
	usage
fi

if [ -n "$1" ]; then
	if [ "$1" != "dailycheck" -a "$1" != "weeklycheck" ]; then
		echo "First augument must be dailycheck or weeklycheck!"
		usage
	else
		checkfrq="$1"
	fi
fi


if [ -n "$2" ]; then
	if ! [ `echo $2 | grep "@"` ]; then
		echo "Mail address isn't right!"
		usage
	else
		sendemailto="$2"
	fi
fi

if [ -n "$3" ]; then
	if [ "$3" != "drawchart" ]; then
		echo "The second argument must be "drawchart"!"
		usage
	else
		drawchart="$3"
	fi
fi

#
#format check
#

milestone_start="$4"
milestone_end="$5"

function drawchart () {
	#
	#This function will draw a barchart contains recipe upgrade history number and manual check history number
	#
	cd $FunctionHome
	python showchart.py >> $Log 2>&1
	python history_upgrade_number_chart.py >> $Log 2>&1
	drawchart_result=$?
	if [ $drawchart_result -eq 1 ]; then
		echo "REPORT-SYSTEM-UPDATE ERROR: Draw barchart faild!" >> $Log
	fi
}

function check_dirstro_format () {
	#
	#This function will check distro_tracking_fields.inc format issue
	# 
	cd $FunctionHome
	python distro_tracking_format_check.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: distro_tracking_format_check.py faild!" >> $Log
}


function collect_history_data_for_chart() {
	cd $LogRestore/
	echo $date >> HistoryDataPre.log
	grep "Upgradable Count:" upgradecontent.log | awk -F: '{print $2}' >> HistoryDataPre.log
	grep "Manual Need Check Count:" lastchktime.log | awk -F: '{print $2}' >> HistoryDataPre.log
	tail -30 HistoryDataPre.log > HistoryData.log
}

function bitbake_engine () {
	bitbake universe -c $1 -k >> $Log 2>&1
	cp $LogHome/$2 $LogRestore
}

function python_engine () {
	cd $FunctionHome
	python $1 1>$LogRestore/$2 2>>$Log || echo "REPORT-SYSTEM-UPDATE ERROR: Run $1 faild!" >> $Log
}

function sendreportmail_upgrade () {
	cd $LogRestore
	if [ -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -a weekly_upgrde_history_chart.png potential_format_problem.log  -- $2
	else
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -a weekly_upgrde_history_chart.png  -- $2
	fi
}

function sendreportmail_upgrade_no_chart () {
	cd $LogRestore
	if [ -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -a potential_format_problem.log -- $2
	else
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Upgrade recipes name list" -- $2
	fi
}


function sendreportmail_manualcheck () {
	cd $LogRestore
	echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "[Package Report System]Manual check recipes name list" -- $2
}

function sendlogmail () {
	cd $LogRestore
	if [ -s block_update_process_format_problem.log -a -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -a block_update_process_format_problem.log potential_format_problem.log -- $2
	elif [ -s block_update_process_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -a block_update_process_format_problem.log -- $2
	elif [ -s potential_format_problem.log ]; then
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -a potential_format_problem.log -- $2
	else
		echo 'Any problem, please contact Saul Wold <sgw@linux.intel.com> ' | mutt -i $1 -s "Warning! Some failures happened on pkg-report system! " -- $2
	fi
}

function check_local_version_change () {
	#
	#This function will record the local version change and change time into table pkgrecord in pkginfobackup.sqlite
	#
	#Collect all the recipes local version information via bitbake
	#
	cd $PokyHome
	. ./oe-init-build-env build
	bitbake -s | sed '/^WARN/d;/^Load/d;/^NOTE:/d;/^Parsing/d;/^Pack/d;/^====/d;/^[Dd]one/d;/^$/d' | awk '{printf("%s,%s\n", $1, $2)}' | sed -e 's/,.*:/,/;s/-r[0-9].*$//' > $LogRestore/recipe_name_version.log.$date
	ln -fs $LogRestore/recipe_name_version.log.$date $LogRestore/recipe_name_version.log

	#
	#Use dailycheck_to_db.py to collect all the recipes information to daily_check.sqlite daily
	#
	cd $FunctionHome
	python dailycheck_to_db.py >> $Log || echo "REPORT-SYSTEM-UPDATE ERROR: Run dailycheck_to_db.py faild!" >> $Log
	cd $WebHome/newserver/webserver
	cp daily_check.sqlite $LogRestore/daily_check.sqlite.$date


}


#
#you can set your own environment variable
#
date="`date +%Y-%m-%d`"
export WebHome=$(cd `dirname $0` && pwd)
export PokyHome="$WebHome/poky/"
export LogHome="$PokyHome/build/tmp/log/"
export ConfigFile="$WebHome/pokyconfig/"
export LogRestore="$WebHome/log/"
export FunctionHome="$WebHome/python-function/"
export Log="$LogRestore/run_getpkginfoall.log.$date"



proxyenv=`env | grep proxy | xargs`
for proxyname in $proxyenv
do
        export $proxyname
        cmd="export $proxyname"
        echo $cmd >> $Log
done

echo "$milestone_start" > $LogRestore/milestone_date
echo "$milestone_end" >> $LogRestore/milestone_date
#For index.py can load it
cp $LogRestore/milestone_date $WebHome/newserver/webserver/

#
#generate log file
#
if [ -e $LogRestore/run_getpkginfoall.log ]; then
	rm -f $LogRestore/run_getpkginfoall.log
fi
if [ -e $Log ]; then
	rm -f $Log
fi
touch $LogRestore/run_getpkginfoall.log.$date
ln -sf $Log $LogRestore/run_getpkginfoall.log

#
#Get poky tree
#
cd $WebHome
if [ ! -d poky ]; then
	git clone git://git.yoctoproject.org/poky.git >> $Log 2>&1
	if [ $? -eq 1 ]; then
		echo "REPORT-SYSTEM-UPDATE ERROR: Get a new poky tree failed!" >> $Log
	fi
fi

cd $PokyHome
#Checkout to master
#USE_MASTER="master"
if  [ -n "$USE_MASTER" ]; then
        git checkout master
        git pull -f git://git.yoctoproject.org/poky.git master:master >> $Log 2>&1 
        if [ $? -eq 1 ]; then
                echo "REPORT-SYSTEM-UPDATE ERROR: Get the latest poky tree failed" >> $Log
        fi
else
	git checkout master
	git branch -D stage
        git fetch git://git.yoctoproject.org/poky-contrib.git stage/master_under_test:stage >> $Log 2>&1
        git checkout stage
        if [ $? -eq 1 ]; then
                echo "REPORT-SYSTEM-UPDATE ERROR: Get the latest poky tree failed" >> $Log
        fi
fi

cd $PokyHome
rm -rf build/
. ./oe-init-build-env build
cp  $ConfigFile/auto.conf ./conf/

#
#get original data from poky tasks
#
rm -rf $PokyHome/build/tmp/log/*
bitbake_engine "distro_check" "distrocheck.csv"
bitbake_engine "checkpkg" "checkpkg.csv"
bitbake_engine "checklicense" "missinglicense.csv"

#
#handle data so that system can use them smoothly
#
python_engine "upgrade_to_log.py" "upgradeinfo.log"
python_engine "lastchktime_to_log.py" "lastchktimeinfo.log"
python_engine "noupgreason.py" "noupgreasoninfo.log"
python_engine "checkdistro.py" "distroexist.log"
python_engine "checklicense.py" "licensecheck.log"
check_dirstro_format

#
#Also need to check local version change, do not worry about repeat issue with daily check task, upgrade_history_to_db.py will abandon repeat data
#
check_local_version_change

#
#run recipe_repo.py to get recipe repo address
#
cd $PokyHome
find meta* -iname *.bb>$LogRestore/recipelist.log  
find meta* -iname *.bb | awk -F/ '{print $NF}'| awk -F ".bb" '{print $1}'>$LogRestore/recipename.log
find meta* -iname *.bb | awk -F/ '{print $NF}'| awk -F ".bb" '{print $1}' | awk -F_ '{print $1}'>$LogRestore/pkgPureName.log
cd $FunctionHome
python recipe_repo.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Run recipe_repo.py faild!" >> $Log

#
#run info_to_db.py to update the database
#
cd $WebHome/newserver/webserver
if [ -e pkginfo.sqlite ]; then
	rm -f pkginfo.sqlite
fi
if [ -e $LogRestore/pkginfo.sqlite.$date ]; then
	rm -f $LogRestore/pkginfo.sqlite.$date
fi
touch $LogRestore/pkginfo.sqlite.$date
ln -sf $LogRestore/pkginfo.sqlite.$date $WebHome/newserver/webserver/pkginfo.sqlite
cd $FunctionHome
python info_to_db.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Update database faild!" >> $Log

#
#Use upgrade_history_to_db.py to get the latest version change of recipe to pkgrecord
#
cd $FunctionHome
python upgrade_history_to_db.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Run upgrade_history_to_db.py faild!" >> $Log

#
#run milestone_target_to_db.py to create table milestone_target for milestone target track
#
python milestone_target_to_db.py $milestone_start >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: run milestone_target_to_db.py faild! Please Check target format! " >> $Log


#
#put the sqlite data to a csv spreadsheet
#
cd $WebHome/newserver/webserver
if [ -e report.csv ]; then
	rm -f report.csv
fi
if [ -e report.csv.$date ]; then
	rm -f report.csv.$date
fi
touch $LogRestore/report.csv.$date
ln -sf $LogRestore/report.csv.$date report.csv
cd $FunctionHome
python pkginfo_to_csv.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Run pkginfo_to_csv.py faild!" >> $Log

#
#check recipes version and give out the list mail which need to be updated
#
cd $FunctionHome
rm $LogRestore/upgradecontent.log $LogRestore/lastchktime.log
python mailaboutupgrade.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Mail content of upgrade info failed!" >> $Log
python mailaboutchktime.py >> $Log 2>&1 || echo "REPORT-SYSTEM-UPDATE ERROR: Mail content of check time failed!" >> $Log

#
#get based commit info
#
cd $PokyHome
echo "The based commit merge time is:" >> $LogRestore/upgradecontent.log
git log --format=%cd -1 >> $LogRestore/upgradecontent.log
echo "The based commit is:" >> $LogRestore/upgradecontent.log
git log -1 | head -n 3 >> $LogRestore/upgradecontent.log

##
##Collect history date data for select date in history page 
##
#cd $LogRestore/
#ls pkginfo.sqlite.* | awk -F . '{print $(NF)}' > databaseHistoryDate.log || echo "REPORT-SYSTEM-UPDATE ERROR: Collect history date faild!" >> $Log

#
#Collect data for bar chart then draw bar chart
#
if [ "$drawchart" = "drawchart" ]; then
	cd $FunctionHome
	python checklog.py
	if [ $? -eq 0 ]; then
		collect_history_data_for_chart
		drawchart
		if [ $drawchart_result -eq 0 ]; then
			cd $LogRestore
			cp $WebHome/newserver/webserver/images/weekly_upgrde_history_chart.png weekly_upgrde_history_chart.png.$date
			ln -sf weekly_upgrde_history_chart.png.$date weekly_upgrde_history_chart.png
		fi
	else
		echo "REPORT-SYSTEM-UPDATE ERROR: Didn't draw chart because some failures happend before" >> $Log
	fi
fi


#Replace the old database with latest one if no ERROR reported in the log
cd $FunctionHome
python checklog.py
if [ $? -eq 0 ]; then
	cd $WebHome/newserver/webserver
	if [ -e pkginfobackup.sqlite ]; then
		rm -f pkginfobackup.sqlite
	fi
	cp pkginfo.sqlite pkginfobackup.sqlite 
	cp report.csv $WebHome
else
	if [ -n "$sendemailto" ]; then
		sendlogmail "$Log" "$sendemailto"
	else
		sendlogmail "$Log" "saul.wold@intel.com"
	fi
fi

#If the checkfrq is dailycheck, we won't send a report, we will exit this script 
if [ "$checkfrq" = "dailycheck" ]; then
	exit 0
fi

#
#use mutt to send list of recipes which need to be updated 
#
cd $FunctionHome
python checklog.py
if [ $? -eq 0 ]; then
	#Send report mail, divide into two conditions, with chart as an attachment or without chart 
	if [ -n "$sendemailto" ]; then
		if [ $drawchart_result -eq 0 ]; then
			sendreportmail_upgrade "upgradecontent.log" "$sendemailto"
		else
			sendreportmail_upgrade_no_chart "upgradecontent.log" "$sendemailto"
		fi
		sendreportmail_manualcheck "lastchktime.log" "$sendemailto"
		if [ $? -eq 1 ]; then
			echo "REPORT-SYSTEM-UPDATE ERROR: Send reportmail faild!" >> $Log
		fi
	else
		if [ $drawchart_result -eq 0 ]; then
			sendreportmail_upgrade "upgradecontent.log" "yocto@linux.intel.com"
		else
			sendreportmail_upgrade_no_chart "upgradecontent.log" "yocto@linux.intel.com"
		sendreportmail_manualcheck "lastchktime.log" "yocto@linux.intel.com"
		fi
		if [ $? -eq 1 ]; then
			echo "REPORT-SYSTEM-UPDATE ERROR: Send reportmail faild!" >> $Log
		fi
	fi
else
	if [ -n "$sendemailto" ]; then
		sendlogmail "$Log" "$sendemailto"
	else
		sendlogmail "$Log" "saul.wold@intel.com"
	fi
	if [ $? -eq 1 ]; then
		echo "REPORT-SYSTEM-UPDATE ERROR: Send logmail faild!" >> $Log
	fi
fi
echo "All tasks finished! You can check the log file $Log" >> $Log