aboutsummaryrefslogtreecommitdiffstats
path: root/bin/common/srtool_update.py
diff options
context:
space:
mode:
Diffstat (limited to 'bin/common/srtool_update.py')
-rwxr-xr-xbin/common/srtool_update.py265
1 files changed, 265 insertions, 0 deletions
diff --git a/bin/common/srtool_update.py b/bin/common/srtool_update.py
new file mode 100755
index 00000000..f73d6800
--- /dev/null
+++ b/bin/common/srtool_update.py
@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Security Response Tool Commandline Tool
+#
+# Copyright (C) 2018 Wind River Systems
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+import re
+import argparse
+import sqlite3
+import subprocess
+import json
+import urllib
+
+# load the srt.sqlite schema indexes
+dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+sys.path.insert(0, dir_path)
+from srt_schema import ORM
+
+from datetime import datetime, timedelta, date
+from pprint import pprint
+from urllib.request import urlopen, URLError
+from urllib.parse import urlparse
+
+# setup
+is_verbose = False
+
+srtDbName = 'srt.sqlite'
+UPDATE_STATUS_LOG = 'update_status.log'
+
+#################################
+# Common routines
+#
+
+# quick development/debugging support
+def _log(msg):
+ DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2
+ DBG_LOG = os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log'
+ if 1 == DBG_LVL:
+ print(msg)
+ elif 2 == DBG_LVL:
+ f1=open(DBG_LOG, 'a')
+ f1.write("|" + msg + "|\n" )
+ f1.close()
+
+def get_tag_key(tag,key,default=''):
+ try:
+ d = json.loads(tag)
+ if key in d:
+ return d[key]
+ else:
+ return default
+ except:
+ print("ERROR TAG FORMAT:get_tag_key(%s,%s)" % (tag,key))
+ return default
+
+#################################
+# Update routines
+#
+# Example 'update_time' filters:
+# MINUTELY = 0 "{\"minutes\":\"10\"}" # every ten minutes
+# HOURLY = 1 "{\"minute\":\"10\"}" # at ten minutes past the hour
+# DAILY = 2 "{\"hour\":\"2\"}" # at 2 hours after midnight
+# WEEKLY = 3 "{\"weekday\":\"6\",\"hour\":\"2\"}" # day of week, hour
+# MONTHLY = 4 "{\"day\":\"1\"\"hour\":\"2\"}" # day of month
+# ONDEMAND = 5 "{}" # only on demand
+
+def run_updates(force_all,name_filter,is_trial):
+
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+ cur_write = conn.cursor()
+
+ time_now = datetime.now() #datetime.now(pytz.utc)
+ print("time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S'))
+ status_str = "====================\n"
+ status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all)
+
+ #get sources that have update command
+ sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
+ for source in sources:
+ # Only process datasoures with update command
+ if not source[ORM.DATASOURCE_UPDATE]:
+ continue
+
+ # Test filter
+ if 'all' != name_filter:
+ is_match = \
+ (name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \
+ (name_filter == source[ORM.DATASOURCE_NAME]) or \
+ (name_filter == source[ORM.DATASOURCE_SOURCE]) or \
+ (name_filter == source[ORM.DATASOURCE_DATA])
+ if not is_match:
+ status_str += " Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION]
+ continue
+
+ # Test the update time
+ if not force_all:
+ # testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *,
+ # testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)
+
+ #print("Update datasource:'%s'" % source[ORM.DATASOURCE_DESCRIPTION])
+
+ # Get the datasource values
+ update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY]
+ if not source[ORM.DATASOURCE_LASTMODIFIEDDATE]:
+ # Force update if no registed modified date for datasource (e.g. Init)
+ last_modified_date = time_now-timedelta(days=365)
+ else:
+ last_modified_date = datetime.strptime(source[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S')
+ # Get the update presets
+ update_time = source[ORM.DATASOURCE_UPDATE_TIME]
+ delta_minutes = get_tag_key(update_time,'minutes',None)
+ delta_minute = get_tag_key(update_time,'minute',None)
+ delta_hour = get_tag_key(update_time,'hour',None)
+ delta_weekday = get_tag_key(update_time,'weekday',None)
+ delta_day = get_tag_key(update_time,'day',None)
+
+ # Calulate the next update datetime
+ if ORM.DATASOURCE_MINUTELY == update_frequency:
+ if not delta_minutes:
+ print("ERROR:Missing minutes in '%s' for '%s'" % (source[ORM.DATASOURCE_DESCRIPTION],update_time))
+ delta_minutes = 10
+ testdiff = timedelta(minutes=int(delta_minutes))
+ elif ORM.DATASOURCE_HOURLY == update_frequency:
+ testdiff = timedelta(hours=1)
+ elif ORM.DATASOURCE_DAILY == update_frequency:
+ testdiff = timedelta(days=1)
+ elif ORM.DATASOURCE_WEEKLY == update_frequency:
+ testdiff = timedelta(weeks=1)
+ elif ORM.DATASOURCE_MONTHLY == update_frequency:
+ testdiff = timedelta(months=1)
+ elif ORM.DATASOURCE_ONDEMAND == update_frequency:
+ continue
+ testdate = last_modified_date + testdiff
+
+ # Adjust for update presets
+ if None != delta_minute:
+ # Force to selected day of month
+ testdate = datetime(testdate.year, testdate.month, testdate.day, testdate.hour, int(delta_minute), testdate.second)
+ if None != delta_day:
+ # Force to selected day of month
+ testdate = datetime(testdate.year, testdate.month, testdate.day, int(delta_hour), testdate.minute, testdate.second)
+ if None != delta_day:
+ # Force to selected day of month
+ testdate = datetime(testdate.year, testdate.month, int(delta_day), testdate.hour, testdate.minute, testdate.second)
+ if None != delta_weekday:
+ # Force to selected day of week
+ testdiff = timedelta( days=(int(delta_weekday) - testdate.weekday()) )
+ testdate += testdiff
+
+ # Not yet?
+ if testdate > time_now:
+ status_str += " Skip '%s': update time not reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S'))
+ continue
+ else:
+ status_str += " UPDATE '%s': update time reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S'))
+
+ # Execute the update
+ if is_trial:
+ print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
+ status_str += " > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
+ else:
+ print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
+ status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
+ os.system(os.path.join(script_pathname, source[ORM.DATASOURCE_UPDATE]))
+
+ # Reset datasource's last_modified_date
+ sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?"
+ cur_write.execute(sql, (time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_ID],) )
+ conn.commit()
+ conn.close()
+
+ # Status summary
+ fd=open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w')
+ fd.write(status_str)
+ fd.close()
+ if verbose:
+ print(status_str)
+
+#time must be in '%H:%M:%S' format
+def configure_ds_update(datasource_description, frequency, time):
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+
+ sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?"
+ cur.execute(sql, (frequency, time, datasource_description))
+
+ conn.commit()
+ conn.close()
+
+
+#################################
+# main loop
+#
+def main(argv):
+ global verbose
+
+ # setup
+ parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database')
+
+ parser.add_argument('--cron-start', action='store_const', const='cron-start', dest='command', help='Start the SRTool backgroud updater')
+ parser.add_argument('--cron-stop', action='store_const', const='cron-stop', dest='command', help='Stop the SRTool backgroud updater')
+
+ parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources')
+ parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
+ parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name')
+
+ parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')
+ parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
+ parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run')
+
+ args = parser.parse_args()
+
+ master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a")
+
+ verbose = args.verbose
+ name_filter = 'all'
+ if args.name_filter:
+ name_filter = args.name_filter
+
+ if 'run-updates' == args.command:
+ if True: #try:
+ print("BEGINNING UPDATING DATASOURCES... this MAY take a long time")
+ run_updates(args.force,name_filter,args.is_trial)
+ master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today()))
+ print("FINISHED UPDATING ALL DATASOURCES\n")
+ if False: #except Exception as e:
+ print("FAILED UPDATING ALL DATASOURCES (%s)" % e)
+ master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
+ elif args.configure_ds_update:
+ try:
+ print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0])
+ configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2])
+ master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0]))
+ except Exception as e:
+ print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0])
+ master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e))
+
+ else:
+ print("Command not found")
+ master_log.close()
+
+if __name__ == '__main__':
+ global script_pathname
+ from os.path import abspath
+ script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
+ main(sys.argv[1:])