diff options
Diffstat (limited to 'bin/common')
-rwxr-xr-x | bin/common/datasource.json | 32 | ||||
-rwxr-xr-x | bin/common/srtool_update.py | 265 |
2 files changed, 280 insertions, 17 deletions
diff --git a/bin/common/datasource.json b/bin/common/datasource.json index 2625abca..789c67cc 100755 --- a/bin/common/datasource.json +++ b/bin/common/datasource.json @@ -24,9 +24,8 @@ "init" : "bin/common/srtool_common.py --generate-schema-header", "update" : "", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", "update_frequency" : "5", - "update_time" : "02:00:00" + "update_time" : "{}" }, { "key" : "0001-common-keywords", @@ -38,9 +37,9 @@ "init" : "bin/common/srtool_common.py --init-package-keywords", "update" : "", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", "update_frequency" : "3", - "update_time" : "02:00:00" + "_comment_" : "Update on Saturdays at 2:00 am", + "update_time" : "{\"weekday\":\"5\",\"hour\":\"2\"}" }, { "key" : "0001-common-notify", @@ -52,9 +51,9 @@ "init" : "bin/common/srtool_common.py --init-notify-categories", "update" : "", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", "update_frequency" : "3", - "update_time" : "02:00:00" + "_comment_" : "Update on Saturdays at 2:00 am", + "update_time" : "{\"weekday\":\"5\",\"hour\":\"2\"}" }, { @@ -67,9 +66,8 @@ "init" : "", "update" : "", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", - "update_frequency" : "0", - "update_time" : "02:00:00" + "update_frequency" : "5", + "update_time" : "" }, { "key" : "0910-common-weekly", @@ -79,11 +77,11 @@ "description" : "Weekly archive database backup", "cve_filter" : "", "init" : "", - "update" : "bin/common/srtool_utils.py --backup-db-json", + "update" : "bin/common/srtool_backup.py --backup-db-json", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", "update_frequency" : "3", - "update_time" : "02:00:00" + "_comment_" : "Update on Saturdays at 2:00 am", + "update_time" : "{\"weekday\":\"5\",\"hour\":\"2\"}" }, { "key" : "0911-common-daily", @@ -93,11 +91,11 @@ "description" : "Daily archive database backup", "cve_filter" : "", "init" : "", - "update" : "bin/common/srtool_utils.py --backup-db-json-daily", + "update" : "bin/common/srtool_backup.py --backup-db-json-daily", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", "update_frequency" : "2", - "update_time" : "02:00:00" + "_comment_" : "Update at 7:00 am", + "update_time" : "{\"hour\":\"7\"}" }, { @@ -110,9 +108,9 @@ "init" : "bin/common/srtool_common.py --score-new-cves NEW", "update" : "bin/common/srtool_common.py --score-new-cves NEW", "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", "update_frequency" : "0", - "update_time" : "02:00:00" + "_comment_" : "Update every 10 minutes", + "update_time" : "{\"minutes\":\"10\"}" } ], diff --git a/bin/common/srtool_update.py b/bin/common/srtool_update.py new file mode 100755 index 00000000..f73d6800 --- /dev/null +++ b/bin/common/srtool_update.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Security Response Tool Commandline Tool +# +# Copyright (C) 2018 Wind River Systems +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import sys +import re +import argparse +import sqlite3 +import subprocess +import json +import urllib + +# load the srt.sqlite schema indexes +dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +sys.path.insert(0, dir_path) +from srt_schema import ORM + +from datetime import datetime, timedelta, date +from pprint import pprint +from urllib.request import urlopen, URLError +from urllib.parse import urlparse + +# setup +is_verbose = False + +srtDbName = 'srt.sqlite' +UPDATE_STATUS_LOG = 'update_status.log' + +################################# +# Common routines +# + +# quick development/debugging support +def _log(msg): + DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2 + DBG_LOG = os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log' + if 1 == DBG_LVL: + print(msg) + elif 2 == DBG_LVL: + f1=open(DBG_LOG, 'a') + f1.write("|" + msg + "|\n" ) + f1.close() + +def get_tag_key(tag,key,default=''): + try: + d = json.loads(tag) + if key in d: + return d[key] + else: + return default + except: + print("ERROR TAG FORMAT:get_tag_key(%s,%s)" % (tag,key)) + return default + +################################# +# Update routines +# +# Example 'update_time' filters: +# MINUTELY = 0 "{\"minutes\":\"10\"}" # every ten minutes +# HOURLY = 1 "{\"minute\":\"10\"}" # at ten minutes past the hour +# DAILY = 2 "{\"hour\":\"2\"}" # at 2 hours after midnight +# WEEKLY = 3 "{\"weekday\":\"6\",\"hour\":\"2\"}" # day of week, hour +# MONTHLY = 4 "{\"day\":\"1\"\"hour\":\"2\"}" # day of month +# ONDEMAND = 5 "{}" # only on demand + +def run_updates(force_all,name_filter,is_trial): + + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + cur_write = conn.cursor() + + time_now = datetime.now() #datetime.now(pytz.utc) + print("time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S')) + status_str = "====================\n" + status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all) + + #get sources that have update command + sources = cur.execute("SELECT * FROM orm_datasource").fetchall() + for source in sources: + # Only process datasoures with update command + if not source[ORM.DATASOURCE_UPDATE]: + continue + + # Test filter + if 'all' != name_filter: + is_match = \ + (name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \ + (name_filter == source[ORM.DATASOURCE_NAME]) or \ + (name_filter == source[ORM.DATASOURCE_SOURCE]) or \ + (name_filter == source[ORM.DATASOURCE_DATA]) + if not is_match: + status_str += " Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION] + continue + + # Test the update time + if not force_all: + # testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, + # testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) + + #print("Update datasource:'%s'" % source[ORM.DATASOURCE_DESCRIPTION]) + + # Get the datasource values + update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY] + if not source[ORM.DATASOURCE_LASTMODIFIEDDATE]: + # Force update if no registed modified date for datasource (e.g. Init) + last_modified_date = time_now-timedelta(days=365) + else: + last_modified_date = datetime.strptime(source[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S') + # Get the update presets + update_time = source[ORM.DATASOURCE_UPDATE_TIME] + delta_minutes = get_tag_key(update_time,'minutes',None) + delta_minute = get_tag_key(update_time,'minute',None) + delta_hour = get_tag_key(update_time,'hour',None) + delta_weekday = get_tag_key(update_time,'weekday',None) + delta_day = get_tag_key(update_time,'day',None) + + # Calulate the next update datetime + if ORM.DATASOURCE_MINUTELY == update_frequency: + if not delta_minutes: + print("ERROR:Missing minutes in '%s' for '%s'" % (source[ORM.DATASOURCE_DESCRIPTION],update_time)) + delta_minutes = 10 + testdiff = timedelta(minutes=int(delta_minutes)) + elif ORM.DATASOURCE_HOURLY == update_frequency: + testdiff = timedelta(hours=1) + elif ORM.DATASOURCE_DAILY == update_frequency: + testdiff = timedelta(days=1) + elif ORM.DATASOURCE_WEEKLY == update_frequency: + testdiff = timedelta(weeks=1) + elif ORM.DATASOURCE_MONTHLY == update_frequency: + testdiff = timedelta(months=1) + elif ORM.DATASOURCE_ONDEMAND == update_frequency: + continue + testdate = last_modified_date + testdiff + + # Adjust for update presets + if None != delta_minute: + # Force to selected day of month + testdate = datetime(testdate.year, testdate.month, testdate.day, testdate.hour, int(delta_minute), testdate.second) + if None != delta_day: + # Force to selected day of month + testdate = datetime(testdate.year, testdate.month, testdate.day, int(delta_hour), testdate.minute, testdate.second) + if None != delta_day: + # Force to selected day of month + testdate = datetime(testdate.year, testdate.month, int(delta_day), testdate.hour, testdate.minute, testdate.second) + if None != delta_weekday: + # Force to selected day of week + testdiff = timedelta( days=(int(delta_weekday) - testdate.weekday()) ) + testdate += testdiff + + # Not yet? + if testdate > time_now: + status_str += " Skip '%s': update time not reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S')) + continue + else: + status_str += " UPDATE '%s': update time reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S')) + + # Execute the update + if is_trial: + print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE])) + status_str += " > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE]) + else: + print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE])) + status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE]) + os.system(os.path.join(script_pathname, source[ORM.DATASOURCE_UPDATE])) + + # Reset datasource's last_modified_date + sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?" + cur_write.execute(sql, (time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_ID],) ) + conn.commit() + conn.close() + + # Status summary + fd=open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') + fd.write(status_str) + fd.close() + if verbose: + print(status_str) + +#time must be in '%H:%M:%S' format +def configure_ds_update(datasource_description, frequency, time): + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + + sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?" + cur.execute(sql, (frequency, time, datasource_description)) + + conn.commit() + conn.close() + + +################################# +# main loop +# +def main(argv): + global verbose + + # setup + parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database') + + parser.add_argument('--cron-start', action='store_const', const='cron-start', dest='command', help='Start the SRTool backgroud updater') + parser.add_argument('--cron-stop', action='store_const', const='cron-stop', dest='command', help='Stop the SRTool backgroud updater') + + parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources') + parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update') + parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name') + + parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info') + parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') + parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run') + + args = parser.parse_args() + + master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a") + + verbose = args.verbose + name_filter = 'all' + if args.name_filter: + name_filter = args.name_filter + + if 'run-updates' == args.command: + if True: #try: + print("BEGINNING UPDATING DATASOURCES... this MAY take a long time") + run_updates(args.force,name_filter,args.is_trial) + master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today())) + print("FINISHED UPDATING ALL DATASOURCES\n") + if False: #except Exception as e: + print("FAILED UPDATING ALL DATASOURCES (%s)" % e) + master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) + elif args.configure_ds_update: + try: + print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0]) + configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2]) + master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0])) + except Exception as e: + print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0]) + master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e)) + + else: + print("Command not found") + master_log.close() + +if __name__ == '__main__': + global script_pathname + from os.path import abspath + script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) + main(sys.argv[1:]) |