path: root/bin/common/srtool_update.py

#!/usr/bin/env python3
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Security Response Tool Commandline Tool
#
# Copyright (C) 2018-2023  Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import os
import sys
import argparse
import json
import time
from datetime import datetime, timedelta
import pytz
import traceback

# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, dir_path)
from common.srt_schema import ORM
from common.srtool_sql import *
# Setup:
verbose = False
master_log = ''

srtDbName = 'srt.sqlite'
UPDATE_STATUS_LOG = 'update_logs/update_status.log'
SRT_UPDATE_PID_FILE = '.srtupdate.pid'
SRT_UPDATE_TASK_FILE = '.srtupdate.task'

#################################
# Common routines
#

# Safe write even when in cron backgroup mode
def master_write(msg):
    master_log.write(msg)
    master_log.flush()

# quick development/debugging support
def _log(msg):
    DBG_LVL =  os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2
    DBG_LOG =  os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log'
    if 1 == DBG_LVL:
        print(msg)
    elif 2 == DBG_LVL:
        f1=open(DBG_LOG, 'a')
        f1.write("|" + msg + "|\n" )
        f1.close()

def get_tag_key(tag,key,default=''):
    try:
        d = json.loads(tag)
        if key in d:
            return d[key]
        else:
            return default
    except Exception as e:
        print("ERROR TAG FORMAT:get_tag_key(%s,%s)=%s" % (tag,key,e))
        return default

#################################
# Update routines
#
# Example 'update_time' filters:
#    MINUTELY = 0   "{\"minutes\":\"10\"}"  # every ten minutes
#    HOURLY = 1     "{\"minute\":\"10\"}"   # at ten minutes past the hour
#    DAILY = 2      "{\"hour\":\"2\"}"   # at 2 hours after midnight
#    WEEKLY = 3     "{\"weekday\":\"6\",\"hour\":\"2\"}"  # day of week, hour
#    MONTHLY = 4    "{\"day\":\"1\"\"hour\":\"2\"}"  # day of month
#    ONDEMAND = 5   "{}"                    # only on demand
#    ONSTARTUP = 6  "{}"                    # on every SRTool start up

def next_refresh_date(update_frequency,update_time_keys,last_updated_date,display=False):
    # Get the update keys
    delta_minutes = int(get_tag_key(update_time_keys,'minutes','10'))
    delta_months = int(get_tag_key(update_time_keys,'months','1'))
    minute_of_day = int(get_tag_key(update_time_keys,'minute','10'))
    hour_of_day = int(get_tag_key(update_time_keys,'hour','12'))
    weekday_of_week = int(get_tag_key(update_time_keys,'weekday','4'))
    day_of_week = int(get_tag_key(update_time_keys,'day','4'))

    # Calulate the next update datetime
    if ORM.DATASOURCE_MINUTELY == update_frequency:
        # Time relative to last
        test_date = last_updated_date + timedelta(minutes=delta_minutes)
    else:
        # Time relative to a time_of_day
        test_date = last_updated_date
        if ORM.DATASOURCE_HOURLY == update_frequency:
            test_date = test_date.replace(minute = minute_of_day)
            if test_date < last_updated_date:
                test_date += timedelta(hours=1)
        elif ORM.DATASOURCE_DAILY == update_frequency:
            test_date = test_date.replace(hour = hour_of_day)
            test_date = test_date.replace(minute = minute_of_day)
            if test_date < last_updated_date:
                test_date += timedelta(days=1)
        elif ORM.DATASOURCE_WEEKLY == update_frequency:
            test_date = test_date.replace(hour = hour_of_day)
            test_date = test_date.replace(minute = minute_of_day)
            weekday = test_date.weekday()
            if weekday >= weekday_of_week:
                test_date += timedelta(days=(7 + weekday_of_week - weekday))
            elif weekday < weekday_of_week:
                test_date += timedelta(days=(weekday_of_week - weekday))
        elif ORM.DATASOURCE_MONTHLY == update_frequency:
            test_date = test_date.replace(day = day_of_week)
            test_date = test_date.replace(hour = hour_of_day)
            test_date = test_date.replace(minute = minute_of_day)
            if test_date < last_updated_date:
                test_date += timedelta(days=31)
        else:
            print("ERROR:unknown update '%s'" % update_frequency)
            exit(1)

    if display:
        # ORM.DATASOURCE_DATETIME_FORMAT
        print("%s <= %s,%s,%s" % (test_date.strftime("%c"),last_updated_date.strftime("%c"),update_frequency,update_time_keys))
    return(test_date)

def update_unit_test():
    # datetime(year, month, day, hour=0, minute=0, tzinfo=None)
    date_now = datetime.now(pytz.utc)
    print("Unit test the update differentials and modes")

    next_refresh_date(ORM.DATASOURCE_MINUTELY,"{\"minutes\":\"10\"}",date_now,True)
    next_refresh_date(ORM.DATASOURCE_MINUTELY,"{\"minutes\":\"10\"}",date_now.replace(minute=59),True)

    next_refresh_date(ORM.DATASOURCE_HOURLY,"{\"minute\":\"10\"}",date_now.replace(minute=11),True)
    next_refresh_date(ORM.DATASOURCE_HOURLY,"{\"minutes\":\"10\"}",date_now.replace(minute=9),True)

    next_refresh_date(ORM.DATASOURCE_DAILY,"{\"hour\":\"2\"}",date_now.replace(hour=1),True)
    next_refresh_date(ORM.DATASOURCE_DAILY,"{\"hour\":\"2\"}",date_now.replace(hour=3),True)

    # May need to adjust this relative to today's test day of week
    next_refresh_date(ORM.DATASOURCE_WEEKLY,"{\"weekday\":\"5\",\"hour\":\"2\"}",date_now.replace(day=3,hour=1),True)
    next_refresh_date(ORM.DATASOURCE_WEEKLY,"{\"weekday\":\"5\",\"hour\":\"2\"}",date_now.replace(day=4,hour=1),True)
    next_refresh_date(ORM.DATASOURCE_WEEKLY,"{\"weekday\":\"5\",\"hour\":\"2\"}",date_now.replace(day=6,hour=3),True)

    next_refresh_date(ORM.DATASOURCE_MONTHLY,"{\"day\":\"5\",\"hour\":\"2\"}",date_now.replace(day=4,hour=1),True)
    next_refresh_date(ORM.DATASOURCE_MONTHLY,"{\"day\":\"25\",\"hour\":\"2\"}",date_now.replace(day=24,hour=1),True)
    next_refresh_date(ORM.DATASOURCE_MONTHLY,"{\"day\":\"25\",\"hour\":\"2\"}",date_now.replace(day=26,hour=1),True)

def time_delta_to_dhms(time_to_go):
    days = time_to_go.days
    clicks_to_go = time_to_go.seconds
    seconds = clicks_to_go % 60
    clicks_to_go //= 60
    minutes = clicks_to_go % 60
    hours = clicks_to_go // 60
    return [days,hours,minutes,seconds]

def run_updates(force_all,name_filter,update_skip_history,is_trial):
    conn = SQL_CONNECT()
    cur = SQL_CURSOR(conn)
    cur_write = SQL_CURSOR(conn)

    # get local timezone
#    local_tz = datetime.now().astimezone().tzinfo
#    time_now = datetime.now(local_tz)  #datetime.now(pytz.utc)
    time_now = datetime.now()  #datetime.now(pytz.utc)
    status_str = "============================================================\n"
    status_str += "Update: Date=%s,Filter='%s',Force=%s,Skip_History=%s\n" % (time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT),name_filter,force_all,update_skip_history)

    #get sources that have update command
    sources = SQL_EXECUTE(cur, '''SELECT * FROM orm_datasource''').fetchall()
    for source in sources:
        # Only process datasoures with update command
        if not source[ORM.DATASOURCE_UPDATE]:
            continue
        elif 'DISABLE ' in source[ORM.DATASOURCE_ATTRIBUTES]:
            # Data source disabled
            continue

        # Test filter
        if 'all' != name_filter:
            is_match = \
                (name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \
                (name_filter == source[ORM.DATASOURCE_NAME]) or \
                (name_filter == source[ORM.DATASOURCE_SOURCE]) or \
                (name_filter == source[ORM.DATASOURCE_DATA])
            if not is_match:
                status_str += "  Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION]
                continue

        # Test the update time
        if not force_all:
            # testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *,
            # testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)

            # Get the datasource values
            update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY]
            if update_frequency in (ORM.DATASOURCE_ONDEMAND,ORM.DATASOURCE_ONSTARTUP,ORM.DATASOURCE_PREINIT):
                continue
            if not source[ORM.DATASOURCE_LASTUPDATEDDATE]:
                if ORM.DATASOURCE_MINUTELY == update_frequency:
                    # Force MINUTELY to the current time)
                    last_updated_date = time_now
                else:
                    # Force update if no registed updated date for datasource (i.e. at Init phase)
                    last_updated_date = time_now - timedelta(days=365)
                sql = "UPDATE orm_datasource SET lastUpdatedDate=? WHERE id=?"
                ret = SQL_EXECUTE(cur, sql, (time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT),source[ORM.DATASOURCE_ID],) )
                SQL_COMMIT(conn)
            else:
                last_updated_date = datetime.strptime(source[ORM.DATASOURCE_LASTUPDATEDDATE], ORM.DATASOURCE_DATETIME_FORMAT)


            # Get the calculated next update datetime
            update_time = source[ORM.DATASOURCE_UPDATE_TIME]
            testdate = next_refresh_date(update_frequency,update_time,last_updated_date)

            # Not yet?
            frequency_str = ORM.get_orm_string(source[ORM.DATASOURCE_UPDATE_FREQUENCY],ORM.DATASOURCE_FREQUENCY_STR)
            if testdate > time_now:
                time_to_go = testdate - time_now
                dhms = time_delta_to_dhms(time_to_go)
                if ORM.DATASOURCE_MINUTELY == update_frequency:
                    status_str += " Pend  (next<%s in days=%2d hours=%2d mins=%02d:%02d,%7s):%s\n" % (testdate.strftime(ORM.DATASOURCE_DATETIME_FORMAT),dhms[0],dhms[1],dhms[2],dhms[3],frequency_str,source[ORM.DATASOURCE_DESCRIPTION])
                else:
                    status_str += " Pend  (next<%s in days=%2d hours=%2d minutes=%2d,%7s):%s\n" % (testdate.strftime(ORM.DATASOURCE_DATETIME_FORMAT),dhms[0],dhms[1],dhms[2],frequency_str,source[ORM.DATASOURCE_DESCRIPTION])
                continue
            else:
                status_str += " GO    (GO  >%s'    (%s) ,%7s):%s\n" % (testdate.strftime(ORM.DATASOURCE_DATETIME_FORMAT), last_updated_date , frequency_str,source[ORM.DATASOURCE_DESCRIPTION])

        # Execute the update
        if is_trial:
            print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
            status_str += "  > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
        else:
            # First update the datasource's last_updated_date to avoid dual triggers
            # (e.g. a manual test run on top of an automatic run)
            sql = "UPDATE orm_datasource SET lastUpdatedDate=? WHERE id=?"
            ret = SQL_EXECUTE(cur, sql, (time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT),source[ORM.DATASOURCE_ID],) )
            SQL_COMMIT(conn)

            print("Update required\t...\texecuting '%s' (%s)" % (source[ORM.DATASOURCE_UPDATE],time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT)))
            status_str += "  > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
            master_write("SRTOOL_UPDATE_STRT:%s:%s:%s\n" %(datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),source[ORM.DATASOURCE_DESCRIPTION],source[ORM.DATASOURCE_UPDATE]))
            update_command = source[ORM.DATASOURCE_UPDATE]
            if force_all:
                update_command += " --force"
            if update_skip_history:
                update_command += " --update-skip-history"
            if update_command.startswith('!'):
                update_command = update_command[1:]
            elif not update_command.startswith('/'):
                update_command = os.path.join(script_pathname, update_command)
            os.system("echo 'Update:%s,%s' > %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),update_command,os.path.join(script_pathname,SRT_UPDATE_TASK_FILE)))

            #
            # bin/common/srtool_job.py -c "<cmnd>" -j 1 -l update_logs/run_job.log
            os.system("bin/common/srtool_job.py --name %s --command \"%s\" --job-id 1 --log update_logs/run_job.log" % (source[ORM.DATASOURCE_NAME],update_command))
            #
            os.system("echo 'Done:%s,%s' >> %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),update_command,os.path.join(script_pathname,SRT_UPDATE_TASK_FILE)))
            master_write("SRTOOL_UPDATE_DONE:%s:%s:%s\n" %(datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),source[ORM.DATASOURCE_DESCRIPTION],source[ORM.DATASOURCE_UPDATE]))
            # Take a breath, let any commits settle
            time.sleep(10)

    SQL_CLOSE_CONN(conn)

    # Status summary
    with open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') as status_file:
        status_file.write(status_str)
    if verbose:
        print(status_str)

def fetch_updates_dhm():
    conn = SQL_CONNECT()
    cur = SQL_CURSOR(conn)
    cur_write = SQL_CURSOR(conn)
    time_now = datetime.now()  #datetime.now(pytz.utc)
    # Get sources
    sources = SQL_EXECUTE(cur, '''SELECT * FROM orm_datasource ORDER BY id ASC''').fetchall()
    for source in sources:
        update_time = source[ORM.DATASOURCE_UPDATE_TIME]
        update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY]
        frequency_str = ORM.get_orm_string(source[ORM.DATASOURCE_UPDATE_FREQUENCY],ORM.DATASOURCE_FREQUENCY_STR)
        # Non-update states
        if update_frequency in (ORM.DATASOURCE_ONDEMAND,ORM.DATASOURCE_ONSTARTUP,ORM.DATASOURCE_PREINIT):
            print("%s,(%s)" % (source[ORM.DATASOURCE_ID],frequency_str))
            continue
        elif not source[ORM.DATASOURCE_UPDATE]:
            print("%s,(NoUpdate)" % source[ORM.DATASOURCE_ID])
            continue
#        elif 'DISABLE ' in source[ORM.DATASOURCE_ATTRIBUTES]:
#            print("%s,(Disabled)" % source[ORM.DATASOURCE_ID])
#            continue
        # Get the datasource values
        if not source[ORM.DATASOURCE_LASTUPDATEDDATE]:
            last_updated_date = time_now
        else:
            last_updated_date = datetime.strptime(source[ORM.DATASOURCE_LASTUPDATEDDATE], ORM.DATASOURCE_DATETIME_FORMAT)
        # Get the calculated next update datetime
        testdate = next_refresh_date(update_frequency,update_time,last_updated_date)
        if testdate > time_now:
            time_to_go = testdate - time_now
            dhms = time_delta_to_dhms(time_to_go)
            if ORM.DATASOURCE_MINUTELY == update_frequency:
                print("%s,%02d|%02d:%02d:%02d" % (source[ORM.DATASOURCE_ID],dhms[0],dhms[1],dhms[2],dhms[3]))
            else:
                print("%s,%02d|%02d:%02d:00" % (source[ORM.DATASOURCE_ID],dhms[0],dhms[1],dhms[2]))
        else:
            print("%s,Next!" % source[ORM.DATASOURCE_ID])
    SQL_CLOSE_CONN(conn)

####################################################################
###

#time must be in '%H:%M:%S' format
def configure_ds_update(datasource_description, frequency, time):
    conn = SQL_CONNECT()
    cur = SQL_CURSOR(conn)

    sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?"
    cur.execute(sql, (frequency, time, datasource_description))

    conn.commit()
    SQL_CLOSE_CONN(conn)

#################################
# List update data sources
#

def list():
    conn = SQL_CONNECT()
    cur = SQL_CURSOR(conn)
    cur_write = SQL_CURSOR(conn)

    format_str = "%16s %9s %14s %10s %28s '%s'"

    print("SRTool Update List:")
    print(format_str % ('Data','  Source','Name','Frequency','Offset','Description'))
    print("================ ========= ============== ========== ============================ ===========================================")
    #get sources that have update command
    sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
    for source in sources:
        # Only process datasoures with update command
        if not source[ORM.DATASOURCE_UPDATE]:
            continue
        frequency_str = ORM.get_orm_string(source[ORM.DATASOURCE_UPDATE_FREQUENCY],ORM.DATASOURCE_FREQUENCY_STR)
        if 'DISABLE ' in source[ORM.DATASOURCE_ATTRIBUTES]:
            frequency_str = 'DISABLED'
        print(format_str % (source[ORM.DATASOURCE_DATA],source[ORM.DATASOURCE_SOURCE],source[ORM.DATASOURCE_NAME],frequency_str,source[ORM.DATASOURCE_UPDATE_TIME],source[ORM.DATASOURCE_DESCRIPTION]))

    if verbose:
        print('')
        run_updates(False,'all',True,True)

    SQL_CLOSE_CUR(cur)
    SQL_CLOSE_CUR(cur_write)
    SQL_CLOSE_CONN(conn)

#################################
# Start 'cron' job for updates
#

def check_updates_enabled(follow_pid_file):
    if verbose: print(f"CHECK_UPDATES_ENABLED...")
    # First check any follow PID file
    if follow_pid_file:
        if not os.path.isfile(follow_pid_file):
            if verbose: print(f"CHECK_UPDATES_ENABLED:FOLLOW_PID_FILE:NOT_EXIST:{follow_pid_file}")
            return(False)
        with open(follow_pid_file) as f:
            lines = f.readlines()
        pid = lines[0].strip()
        ret = os.system(f"ps -p {pid} > /dev/null 2>&1")
        if ret:
            if verbose: print(f"CHECK_UPDATES_ENABLED:FOLLOW_PID_FILE:NOT_RUNNING:{pid}")
            return(False)
        else:
            if verbose: print(f"CHECK_UPDATES_ENABLED:FOLLOW_PID_FILE:RUNNING:{pid}")

    # Check if master disable
    conn = SQL_CONNECT(column_names=True)
    cur = SQL_CURSOR(conn)
    sql = 'SELECT * FROM orm_srtsetting WHERE "name" = ?'
    enable_update_setting = SQL_EXECUTE(cur, sql, ('SRT_DISABLE_UPDATES',)).fetchone()
    ret = (not enable_update_setting) or ('yes' != enable_update_setting['value'])
    SQL_CLOSE_CUR(cur)
    SQL_CLOSE_CONN(conn)
    if verbose: print(f"CHECK_UPDATES_ENABLED:SRT_DISABLE_UPDATES:{'GO' if ret else 'SKIP'}:")
    return(ret)

def cron_start(follow_pid_file):
    pid = os.getpid()
    master_write("SRTOOL_UPDATE:%s:Starting -v update cron job, pid=%s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),pid))
    os.system("echo 'Start:%s,<cron_start>!' > %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),os.path.join(script_pathname,SRT_UPDATE_TASK_FILE)))

#
    print("echo 'Start:%s,<cron_start>!' > %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),os.path.join(script_pathname,SRT_UPDATE_TASK_FILE)))

    # Preserve this app's pid
    srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
    with open(srt_update_pid_file, 'w') as pidfile:
        pidfile.write("%s" % pid)

    # Start with delay to allow SRTool to bootup
    time.sleep(30)

    # Loop until app is killed
    extra_line = False
    while True:
        try:
            if check_updates_enabled(follow_pid_file):
                # Run the updates
                run_updates(False,'all',False,False)
                # Toggle an extra line in the log to make updates obvious
                if extra_line:
                    extra_line = False
                    os.system("echo '' >> %s" % os.path.join(script_pathname,UPDATE_STATUS_LOG))
                else:
                    extra_line = True
            # Default to 5 minute loop

            os.system("echo 'Sleep:%s,update in 5 minutes (%s)' > %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),check_updates_enabled(follow_pid_file),os.path.join(script_pathname,SRT_UPDATE_TASK_FILE)))

            run_updates(False,'all',True,True)

            time.sleep(5 * 60)
        except Exception as e:
            master_write("SRTOOL_UPDATE:ERROR:%s:%s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),e))

def cron_stop():
    # Fetch the stored update app's pid
    srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
    if os.path.isfile(srt_update_pid_file):
        with open(srt_update_pid_file, 'r') as pidfile:
            pid = pidfile.read()
        print("KILL UPDATE:%s" % pid)
        # Kill the update app
        os.system("kill %s" % pid)
        os.system("rm %s" % srt_update_pid_file)
        master_write("SRTOOL_UPDATE:%s:Stopping -^ update cron job, pid=%s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),pid))
        os.system("echo 'Done:%s,<cron_stop>' > %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),os.path.join(script_pathname,SRT_UPDATE_TASK_FILE)))
    else:
        print("No running update task file found")

#################################
# main loop
#

def main(argv):
    global verbose
    global master_log

    # setup
    parser = argparse.ArgumentParser(description='srtool_update.py: manage the SRTool backgtoup tasks')

    # Commands
    parser.add_argument('--cron-start', action='store_const', const='cron_start', dest='command', help='Start the SRTool background updater')
    parser.add_argument('--cron-stop', action='store_const', const='cron_stop', dest='command', help='Stop the SRTool background updater')
    parser.add_argument('--follow-pid-file', dest='follow_pid_file', help='Only update when PID in this file is running')

    # Status
    parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List data sources')
    parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='Update scheduled data sources')
    parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name')
    parser.add_argument('--status', '-s', action='store_const', const='status', dest='command', help='Current status of the run queue')
    parser.add_argument('--fetch-updates-dhm', action='store_const', const='fetch_updates_dhm', dest='command', help='Fetch next updates for all sources')
    parser.add_argument('--check-updates-enabled', action='store_const', const='check_updates_enabled', dest='command', help='Unit test the update offsets')

    # Test
    parser.add_argument('--update-unit-test', '-U', action='store_const', const='update_unit_test', dest='command', help='Unit test the update offsets')

    # Debugging support
    parser.add_argument('--force', '-f', action='store_true', dest='force', help='Flag: Force the update')
    parser.add_argument('--update-skip-history', '-H', action='store_true', dest='update_skip_history', help='Flag: Skip history updates')
    parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Flag: debug verbose output')
    parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Flag: Debugging: trial run')

    parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')

    args = parser.parse_args()

    master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a")

    verbose = args.verbose
    name_filter = 'all'
    if args.name_filter:
        name_filter = args.name_filter

    if 'list' == args.command:
        list()
    elif 'run-updates' == args.command:
        try:
            print("BEGINNING UPDATING DATASOURCES... this MAY take a long time")
            run_updates(args.force,name_filter,args.update_skip_history,args.is_trial)
            master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT)))
            print("FINISHED UPDATING ALL DATASOURCES\n")
        except Exception as e:
            print("FAILED UPDATING ALL DATASOURCES (%s)" % e)
            master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT), e))
            traceback.print_exc(file=sys.stdout)
    elif 'fetch_updates_dhm' == args.command:
        fetch_updates_dhm()
    elif 'check_updates_enabled' == args.command:
        verbose = True
        check_updates_enabled(args.follow_pid_file)
    elif args.configure_ds_update:
        try:
            print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0])
            configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2])
            master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT), args.configure_ds_update[0]))
        except Exception as e:
            print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0])
            master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT), args.configure_ds_update[0], e))
    elif 'status' == args.command:
        verbose = True
        run_updates(False,'all',True,True)

    elif 'cron_start' == args.command:
        cron_start(args.follow_pid_file)
    elif 'cron_stop' == args.command:
        cron_stop()

    elif 'update_unit_test' == args.command:
        verbose = True
        update_unit_test()

    else:
        print("srtool_update: Command not found")
    master_log.close()

if __name__ == '__main__':
    script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
    main(sys.argv[1:])