#!/usr/bin/env python3 # # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # # Security Response Tool Commandline Tool # # Copyright (C) 2018 Wind River Systems # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import sys import re import argparse import sqlite3 import subprocess import json import urllib # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from srt_schema import ORM from datetime import datetime, timedelta, date from pprint import pprint from urllib.request import urlopen, URLError from urllib.parse import urlparse # setup is_verbose = False srtDbName = 'srt.sqlite' UPDATE_STATUS_LOG = 'update_status.log' ################################# # Common routines # # quick development/debugging support def _log(msg): DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2 DBG_LOG = os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log' if 1 == DBG_LVL: print(msg) elif 2 == DBG_LVL: f1=open(DBG_LOG, 'a') f1.write("|" + msg + "|\n" ) f1.close() def get_tag_key(tag,key,default=''): try: d = json.loads(tag) if key in d: return d[key] else: return default except: print("ERROR TAG FORMAT:get_tag_key(%s,%s)" % (tag,key)) return default ################################# # Update routines # # Example 'update_time' filters: # MINUTELY = 0 "{\"minutes\":\"10\"}" # every ten minutes # HOURLY = 1 "{\"minute\":\"10\"}" # at ten minutes past the hour # DAILY = 2 "{\"hour\":\"2\"}" # at 2 hours after midnight # WEEKLY = 3 "{\"weekday\":\"6\",\"hour\":\"2\"}" # day of week, hour # MONTHLY = 4 "{\"day\":\"1\"\"hour\":\"2\"}" # day of month # ONDEMAND = 5 "{}" # only on demand def run_updates(force_all,name_filter,is_trial): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() time_now = datetime.now() #datetime.now(pytz.utc) print("time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S')) status_str = "====================\n" status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all) #get sources that have update command sources = cur.execute("SELECT * FROM orm_datasource").fetchall() for source in sources: # Only process datasoures with update command if not source[ORM.DATASOURCE_UPDATE]: continue # Test filter if 'all' != name_filter: is_match = \ (name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \ (name_filter == source[ORM.DATASOURCE_NAME]) or \ (name_filter == source[ORM.DATASOURCE_SOURCE]) or \ (name_filter == source[ORM.DATASOURCE_DATA]) if not is_match: status_str += " Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION] continue # Test the update time if not force_all: # testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, # testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) #print("Update datasource:'%s'" % source[ORM.DATASOURCE_DESCRIPTION]) # Get the datasource values update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY] if not source[ORM.DATASOURCE_LASTMODIFIEDDATE]: # Force update if no registed modified date for datasource (e.g. Init) last_modified_date = time_now-timedelta(days=365) else: last_modified_date = datetime.strptime(source[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S') # Get the update presets update_time = source[ORM.DATASOURCE_UPDATE_TIME] delta_minutes = get_tag_key(update_time,'minutes',None) delta_minute = get_tag_key(update_time,'minute',None) delta_hour = get_tag_key(update_time,'hour',None) delta_weekday = get_tag_key(update_time,'weekday',None) delta_day = get_tag_key(update_time,'day',None) # Calulate the next update datetime if ORM.DATASOURCE_MINUTELY == update_frequency: if not delta_minutes: print("ERROR:Missing minutes in '%s' for '%s'" % (source[ORM.DATASOURCE_DESCRIPTION],update_time)) delta_minutes = 10 testdiff = timedelta(minutes=int(delta_minutes)) elif ORM.DATASOURCE_HOURLY == update_frequency: testdiff = timedelta(hours=1) elif ORM.DATASOURCE_DAILY == update_frequency: testdiff = timedelta(days=1) elif ORM.DATASOURCE_WEEKLY == update_frequency: testdiff = timedelta(weeks=1) elif ORM.DATASOURCE_MONTHLY == update_frequency: testdiff = timedelta(months=1) elif ORM.DATASOURCE_ONDEMAND == update_frequency: continue testdate = last_modified_date + testdiff # Adjust for update presets if None != delta_minute: # Force to selected day of month testdate = datetime(testdate.year, testdate.month, testdate.day, testdate.hour, int(delta_minute), testdate.second) if None != delta_day: # Force to selected day of month testdate = datetime(testdate.year, testdate.month, testdate.day, int(delta_hour), testdate.minute, testdate.second) if None != delta_day: # Force to selected day of month testdate = datetime(testdate.year, testdate.month, int(delta_day), testdate.hour, testdate.minute, testdate.second) if None != delta_weekday: # Force to selected day of week testdiff = timedelta( days=(int(delta_weekday) - testdate.weekday()) ) testdate += testdiff # Not yet? if testdate > time_now: status_str += " Skip '%s': update time not reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S')) continue else: status_str += " UPDATE '%s': update time reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S')) # Execute the update if is_trial: print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE])) status_str += " > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE]) else: print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE])) status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE]) os.system(os.path.join(script_pathname, source[ORM.DATASOURCE_UPDATE])) # Reset datasource's last_modified_date sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?" cur_write.execute(sql, (time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_ID],) ) conn.commit() conn.close() # Status summary fd=open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') fd.write(status_str) fd.close() if verbose: print(status_str) #time must be in '%H:%M:%S' format def configure_ds_update(datasource_description, frequency, time): conn = sqlite3.connect(srtDbName) cur = conn.cursor() sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?" cur.execute(sql, (frequency, time, datasource_description)) conn.commit() conn.close() ################################# # main loop # def main(argv): global verbose # setup parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database') parser.add_argument('--cron-start', action='store_const', const='cron-start', dest='command', help='Start the SRTool backgroud updater') parser.add_argument('--cron-stop', action='store_const', const='cron-stop', dest='command', help='Stop the SRTool backgroud updater') parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources') parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update') parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name') parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info') parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run') args = parser.parse_args() master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a") verbose = args.verbose name_filter = 'all' if args.name_filter: name_filter = args.name_filter if 'run-updates' == args.command: if True: #try: print("BEGINNING UPDATING DATASOURCES... this MAY take a long time") run_updates(args.force,name_filter,args.is_trial) master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today())) print("FINISHED UPDATING ALL DATASOURCES\n") if False: #except Exception as e: print("FAILED UPDATING ALL DATASOURCES (%s)" % e) master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) elif args.configure_ds_update: try: print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0]) configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2]) master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0])) except Exception as e: print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0]) master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e)) else: print("Command not found") master_log.close() if __name__ == '__main__': global script_pathname from os.path import abspath script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:])