diff options
Diffstat (limited to 'bin/common/srtool_update.py')
| -rwxr-xr-x | bin/common/srtool_update.py | 123 |
1 files changed, 102 insertions, 21 deletions
diff --git a/bin/common/srtool_update.py b/bin/common/srtool_update.py index f73d6800..a093a7b6 100755 --- a/bin/common/srtool_update.py +++ b/bin/common/srtool_update.py @@ -22,12 +22,10 @@ import os import sys -import re import argparse import sqlite3 -import subprocess import json -import urllib +import time # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -43,12 +41,18 @@ from urllib.parse import urlparse is_verbose = False srtDbName = 'srt.sqlite' -UPDATE_STATUS_LOG = 'update_status.log' +UPDATE_STATUS_LOG = 'update_logs/update_status.log' +SRT_UPDATE_PID_FILE = '.srtupdate.pid' ################################# # Common routines # +# Safe write even when in cron backgroup mode +def master_write(msg): + master_log.write(msg) + master_log.flush() + # quick development/debugging support def _log(msg): DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2 @@ -67,8 +71,8 @@ def get_tag_key(tag,key,default=''): return d[key] else: return default - except: - print("ERROR TAG FORMAT:get_tag_key(%s,%s)" % (tag,key)) + except Exception as e: + print("ERROR TAG FORMAT:get_tag_key(%s,%s)=%s" % (tag,key,e)) return default ################################# @@ -88,8 +92,8 @@ def run_updates(force_all,name_filter,is_trial): cur = conn.cursor() cur_write = conn.cursor() - time_now = datetime.now() #datetime.now(pytz.utc) - print("time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S')) + time_now = datetime.now() #datetime.now(pytz.utc) + print("SRTool Update: time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S')) status_str = "====================\n" status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all) @@ -149,6 +153,8 @@ def run_updates(force_all,name_filter,is_trial): testdiff = timedelta(months=1) elif ORM.DATASOURCE_ONDEMAND == update_frequency: continue + elif ORM.DATASOURCE_ONSTARTUP == update_frequency: + continue testdate = last_modified_date + testdiff # Adjust for update presets @@ -180,7 +186,11 @@ def run_updates(force_all,name_filter,is_trial): else: print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE])) status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE]) - os.system(os.path.join(script_pathname, source[ORM.DATASOURCE_UPDATE])) + master_write("SRTOOL_UPDATE:%s:%s:%s\n" %(time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_DESCRIPTION],source[ORM.DATASOURCE_UPDATE])) + update_command = source[ORM.DATASOURCE_UPDATE] + if force_all: + update_command += " --force" + os.system(os.path.join(script_pathname, update_command)) # Reset datasource's last_modified_date sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?" @@ -189,9 +199,8 @@ def run_updates(force_all,name_filter,is_trial): conn.close() # Status summary - fd=open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') - fd.write(status_str) - fd.close() + with open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') as status_file: + status_file.write(status_str) if verbose: print(status_str) @@ -208,25 +217,92 @@ def configure_ds_update(datasource_description, frequency, time): ################################# +# List update data sources +# + +def list(): + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + cur_write = conn.cursor() + + format_str = "%14s %7s %14s %10s %28s %s" + + print("SRTool Update List:") + status_str = "====================\n" + print(format_str % ('Data','Source','Name','Frequency','Offset','Description')) + #get sources that have update command + sources = cur.execute("SELECT * FROM orm_datasource").fetchall() + for source in sources: + # Only process datasoures with update command + if not source[ORM.DATASOURCE_UPDATE]: + continue + frequency_str = ORM.DATASOURCE_FREQUENCY_STR.split(',')[source[ORM.DATASOURCE_UPDATE_FREQUENCY]] + print(format_str % (source[ORM.DATASOURCE_DATA],source[ORM.DATASOURCE_SOURCE],source[ORM.DATASOURCE_NAME],frequency_str,source[ORM.DATASOURCE_UPDATE_TIME],source[ORM.DATASOURCE_DESCRIPTION])) + +################################# +# Start 'cron' job for updates +# + +def cron_start(): + pid = os.getpid() + master_write("SRTOOL_UPDATE:%s:Starting -v update cron job, pid=%s\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),pid)) + + # Preserve this app's pid + srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE) + with open(srt_update_pid_file, 'w') as pidfile: + pidfile.write("%s" % pid) + + # Loop until app is killed + extra_line = False + while True: + run_updates(False,'all',False) + # Toggle an extra line in the log to make updates obvious + if extra_line: + extra_line = False + os.system("echo '' >> %s" % os.path.join(script_pathname,UPDATE_STATUS_LOG)) + else: + extra_line = True + # Default to 5 minute loop + time.sleep(5 * 60) + +def cron_stop(): + # Fetch the stored update app's pid + srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE) + if os.path.isfile(srt_update_pid_file): + with open(srt_update_pid_file, 'r') as pidfile: + pid = pidfile.read() + print("KILL UPDATE:%s" % pid) + # Kill the update app + os.system("kill %s" % pid) + os.system("rm %s" % srt_update_pid_file) + master_write("SRTOOL_UPDATE:%s:Stopping -^ update cron job, pid=%s\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),pid)) + else: + print("No running update task file found") + +################################# # main loop # + def main(argv): global verbose + global master_log # setup parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database') - parser.add_argument('--cron-start', action='store_const', const='cron-start', dest='command', help='Start the SRTool backgroud updater') - parser.add_argument('--cron-stop', action='store_const', const='cron-stop', dest='command', help='Stop the SRTool backgroud updater') + parser.add_argument('--cron-start', action='store_const', const='cron_start', dest='command', help='Start the SRTool backgroud updater') + parser.add_argument('--cron-stop', action='store_const', const='cron_stop', dest='command', help='Stop the SRTool backgroud updater') - parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources') - parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update') + parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List data sources') + parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='Update scheduled data sources') parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name') - parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info') + parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update') parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run') + parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info') + args = parser.parse_args() master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a") @@ -236,13 +312,15 @@ def main(argv): if args.name_filter: name_filter = args.name_filter - if 'run-updates' == args.command: - if True: #try: + if 'list' == args.command: + list() + elif 'run-updates' == args.command: + try: print("BEGINNING UPDATING DATASOURCES... this MAY take a long time") run_updates(args.force,name_filter,args.is_trial) master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today())) print("FINISHED UPDATING ALL DATASOURCES\n") - if False: #except Exception as e: + except Exception as e: print("FAILED UPDATING ALL DATASOURCES (%s)" % e) master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) elif args.configure_ds_update: @@ -253,7 +331,10 @@ def main(argv): except Exception as e: print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0]) master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e)) - + elif 'cron_start' == args.command: + cron_start() + elif 'cron_stop' == args.command: + cron_stop() else: print("Command not found") master_log.close() |