aboutsummaryrefslogtreecommitdiffstats
path: root/bin/common/srtool_update.py
blob: 64501f69b855f4b2c719de59f4d7f0cb33e74fbb (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
#!/usr/bin/env python3
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Security Response Tool Commandline Tool
#
# Copyright (C) 2018       Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import os
import sys
import argparse
import sqlite3
import json
import time

# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, dir_path)
from srt_schema import ORM

from datetime import datetime, timedelta, date
from pprint import pprint
from urllib.request import urlopen, URLError
from urllib.parse import urlparse

# setup
is_verbose = False

srtDbName = 'srt.sqlite'
UPDATE_STATUS_LOG = 'update_logs/update_status.log'
SRT_UPDATE_PID_FILE = '.srtupdate.pid'

#################################
# Common routines
#

# Safe write even when in cron backgroup mode
def master_write(msg):
    master_log.write(msg)
    master_log.flush()

# quick development/debugging support
def _log(msg):
    DBG_LVL =  os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2
    DBG_LOG =  os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log'
    if 1 == DBG_LVL:
        print(msg)
    elif 2 == DBG_LVL:
        f1=open(DBG_LOG, 'a')
        f1.write("|" + msg + "|\n" )
        f1.close()

def get_tag_key(tag,key,default=''):
    try:
        d = json.loads(tag)
        if key in d:
            return d[key]
        else:
            return default
    except Exception as e:
        print("ERROR TAG FORMAT:get_tag_key(%s,%s)=%s" % (tag,key,e))
        return default

#################################
# Update routines
#
# Example 'update_time' filters:
#    MINUTELY = 0   "{\"minutes\":\"10\"}"  # every ten minutes
#    HOURLY = 1     "{\"minute\":\"10\"}"   # at ten minutes past the hour
#    DAILY = 2      "{\"hour\":\"2\"}"   # at 2 hours after midnight
#    WEEKLY = 3     "{\"weekday\":\"6\",\"hour\":\"2\"}"  # day of week, hour
#    MONTHLY = 4    "{\"day\":\"1\"\"hour\":\"2\"}"  # day of month
#    ONDEMAND = 5   "{}"                    # only on demand
#    ONSTARTUP = 6  "{}"                    # on every SRTool start up

def run_updates(force_all,name_filter,is_trial):

    conn = sqlite3.connect(srtDbName)
    cur = conn.cursor()
    cur_write = conn.cursor()

    time_now = datetime.now()  #datetime.now(pytz.utc)
    print("SRTool Update: time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S'))
    status_str = "====================\n"
    status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all)

    #get sources that have update command
    sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
    for source in sources:
        # Only process datasoures with update command
        if not source[ORM.DATASOURCE_UPDATE]:
            continue

        # Test filter
        if 'all' != name_filter:
            is_match = \
                (name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \
                (name_filter == source[ORM.DATASOURCE_NAME]) or \
                (name_filter == source[ORM.DATASOURCE_SOURCE]) or \
                (name_filter == source[ORM.DATASOURCE_DATA])
            if not is_match:
                status_str += "  Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION]
                continue

        # Test the update time
        if not force_all:
            # testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *,
            # testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)

            #print("Update datasource:'%s'" % source[ORM.DATASOURCE_DESCRIPTION])

            # Get the datasource values
            update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY]
            if not source[ORM.DATASOURCE_LASTMODIFIEDDATE]:
                # Force update if no registed modified date for datasource (e.g. Init)
                last_modified_date = time_now-timedelta(days=365)
            else:
                last_modified_date = datetime.strptime(source[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S')
            # Get the update presets
            update_time = source[ORM.DATASOURCE_UPDATE_TIME]
            delta_minutes = get_tag_key(update_time,'minutes',None)
            delta_minute = get_tag_key(update_time,'minute',None)
            delta_hour = get_tag_key(update_time,'hour',None)
            delta_weekday = get_tag_key(update_time,'weekday',None)
            delta_day = get_tag_key(update_time,'day',None)

            # Calulate the next update datetime
            if ORM.DATASOURCE_MINUTELY == update_frequency:
                if not delta_minutes:
                    print("ERROR:Missing minutes in '%s' for '%s'" % (source[ORM.DATASOURCE_DESCRIPTION],update_time))
                    delta_minutes = 10
                testdiff = timedelta(minutes=int(delta_minutes))
            elif ORM.DATASOURCE_HOURLY == update_frequency:
                testdiff = timedelta(hours=1)
            elif ORM.DATASOURCE_DAILY == update_frequency:
                testdiff = timedelta(days=1)
            elif ORM.DATASOURCE_WEEKLY == update_frequency:
                testdiff = timedelta(weeks=1)
            elif ORM.DATASOURCE_MONTHLY == update_frequency:
                testdiff = timedelta(months=1)
            elif ORM.DATASOURCE_ONDEMAND == update_frequency:
                continue
            elif ORM.DATASOURCE_ONSTARTUP == update_frequency:
                continue
            testdate = last_modified_date + testdiff

            # Adjust for update presets
            if None != delta_minute:
                # Force to selected day of month
                testdate = datetime(testdate.year, testdate.month, testdate.day, testdate.hour, int(delta_minute), testdate.second)
            if None != delta_day:
                # Force to selected day of month
                testdate = datetime(testdate.year, testdate.month, testdate.day, int(delta_hour), testdate.minute, testdate.second)
            if None != delta_day:
                # Force to selected day of month
                testdate = datetime(testdate.year, testdate.month, int(delta_day), testdate.hour, testdate.minute, testdate.second)
            if None != delta_weekday:
                # Force to selected day of week
                testdiff = timedelta( days=(int(delta_weekday) - testdate.weekday()) )
                testdate += testdiff

            # Not yet?
            if testdate > time_now:
                status_str += "  Skip '%s': update time not reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S'))
                continue
            else:
                status_str += "  UPDATE '%s': update time reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S'))

        # Execute the update
        if is_trial:
            print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
            status_str += "  > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
        else:
            print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
            status_str += "  > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
            master_write("SRTOOL_UPDATE:%s:%s:%s\n" %(time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_DESCRIPTION],source[ORM.DATASOURCE_UPDATE]))
            update_command = source[ORM.DATASOURCE_UPDATE]
            if force_all:
                update_command += " --force"
            os.system(os.path.join(script_pathname, update_command))

            # Reset datasource's last_modified_date
            sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?"
            cur_write.execute(sql, (time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_ID],) )
            conn.commit()
    conn.close()

    # Status summary
    with open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') as status_file:
        status_file.write(status_str)
    if verbose:
        print(status_str)

#time must be in '%H:%M:%S' format
def configure_ds_update(datasource_description, frequency, time):
    conn = sqlite3.connect(srtDbName)
    cur = conn.cursor()

    sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?"
    cur.execute(sql, (frequency, time, datasource_description))

    conn.commit()
    conn.close()


#################################
# List update data sources
#

def list():
    conn = sqlite3.connect(srtDbName)
    cur = conn.cursor()
    cur_write = conn.cursor()

    format_str = "%16s %7s %14s %10s %28s %s"

    print("SRTool Update List:")
    print(format_str % ('Data','Source','Name','Frequency','Offset','Description'))
    print("================ ======= ============== ========== ============================ ===========================================")
    #get sources that have update command
    sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
    for source in sources:
        # Only process datasoures with update command
        if not source[ORM.DATASOURCE_UPDATE]:
            continue
        frequency_str = ORM.get_orm_string(source[ORM.DATASOURCE_UPDATE_FREQUENCY],ORM.DATASOURCE_FREQUENCY_STR)
        print(format_str % (source[ORM.DATASOURCE_DATA],source[ORM.DATASOURCE_SOURCE],source[ORM.DATASOURCE_NAME],frequency_str,source[ORM.DATASOURCE_UPDATE_TIME],source[ORM.DATASOURCE_DESCRIPTION]))

#################################
# Start 'cron' job for updates
#

def cron_start():
    pid = os.getpid()
    master_write("SRTOOL_UPDATE:%s:Starting -v update cron job, pid=%s\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),pid))

    # Preserve this app's pid
    srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
    with open(srt_update_pid_file, 'w') as pidfile:
        pidfile.write("%s" % pid)

    # Loop until app is killed
    extra_line = False
    while True:
        run_updates(False,'all',False)
        # Toggle an extra line in the log to make updates obvious
        if extra_line:
            extra_line = False
            os.system("echo '' >> %s" % os.path.join(script_pathname,UPDATE_STATUS_LOG))
        else:
            extra_line = True
        # Default to 5 minute loop
        time.sleep(5 * 60)

def cron_stop():
    # Fetch the stored update app's pid
    srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
    if os.path.isfile(srt_update_pid_file):
        with open(srt_update_pid_file, 'r') as pidfile:
            pid = pidfile.read()
        print("KILL UPDATE:%s" % pid)
        # Kill the update app
        os.system("kill %s" % pid)
        os.system("rm %s" % srt_update_pid_file)
        master_write("SRTOOL_UPDATE:%s:Stopping -^ update cron job, pid=%s\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),pid))
    else:
        print("No running update task file found")

#################################
# main loop
#

def main(argv):
    global verbose
    global master_log

    # setup
    parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database')

    parser.add_argument('--cron-start', action='store_const', const='cron_start', dest='command', help='Start the SRTool background updater')
    parser.add_argument('--cron-stop', action='store_const', const='cron_stop', dest='command', help='Stop the SRTool background updater')

    parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List data sources')
    parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='Update scheduled data sources')
    parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name')

    parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
    parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
    parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run')

    parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')

    args = parser.parse_args()

    master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a")

    verbose = args.verbose
    name_filter = 'all'
    if args.name_filter:
        name_filter = args.name_filter

    if 'list' == args.command:
        list()
    elif 'run-updates' == args.command:
        try:
            print("BEGINNING UPDATING DATASOURCES... this MAY take a long time")
            run_updates(args.force,name_filter,args.is_trial)
            master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today()))
            print("FINISHED UPDATING ALL DATASOURCES\n")
        except Exception as e:
            print("FAILED UPDATING ALL DATASOURCES (%s)" % e)
            master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
    elif args.configure_ds_update:
        try:
            print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0])
            configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2])
            master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0]))
        except Exception as e:
            print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0])
            master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e))
    elif 'cron_start' == args.command:
        cron_start()
    elif 'cron_stop' == args.command:
        cron_stop()
    else:
        print("Command not found")
    master_log.close()

if __name__ == '__main__':
    global script_pathname
    from os.path import abspath
    script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
    main(sys.argv[1:])