1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
|
#!/usr/bin/env python3
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Security Response Tool Commandline Tool
#
# Copyright (C) 2018 Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import argparse
import sqlite3
import json
import time
# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, dir_path)
from srt_schema import ORM
from datetime import datetime, timedelta, date
from pprint import pprint
from urllib.request import urlopen, URLError
from urllib.parse import urlparse
# setup
is_verbose = False
srtDbName = 'srt.sqlite'
UPDATE_STATUS_LOG = 'update_logs/update_status.log'
SRT_UPDATE_PID_FILE = '.srtupdate.pid'
#################################
# Common routines
#
# Safe write even when in cron backgroup mode
def master_write(msg):
master_log.write(msg)
master_log.flush()
# quick development/debugging support
def _log(msg):
DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2
DBG_LOG = os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log'
if 1 == DBG_LVL:
print(msg)
elif 2 == DBG_LVL:
f1=open(DBG_LOG, 'a')
f1.write("|" + msg + "|\n" )
f1.close()
def get_tag_key(tag,key,default=''):
try:
d = json.loads(tag)
if key in d:
return d[key]
else:
return default
except Exception as e:
print("ERROR TAG FORMAT:get_tag_key(%s,%s)=%s" % (tag,key,e))
return default
#################################
# Update routines
#
# Example 'update_time' filters:
# MINUTELY = 0 "{\"minutes\":\"10\"}" # every ten minutes
# HOURLY = 1 "{\"minute\":\"10\"}" # at ten minutes past the hour
# DAILY = 2 "{\"hour\":\"2\"}" # at 2 hours after midnight
# WEEKLY = 3 "{\"weekday\":\"6\",\"hour\":\"2\"}" # day of week, hour
# MONTHLY = 4 "{\"day\":\"1\"\"hour\":\"2\"}" # day of month
# ONDEMAND = 5 "{}" # only on demand
# ONSTARTUP = 6 "{}" # on every SRTool start up
def run_updates(force_all,name_filter,is_trial):
conn = sqlite3.connect(srtDbName)
cur = conn.cursor()
cur_write = conn.cursor()
time_now = datetime.now() #datetime.now(pytz.utc)
print("SRTool Update: time_now = %s" % time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT))
status_str = "====================\n"
status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT),name_filter,force_all)
#get sources that have update command
sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
for source in sources:
# Only process datasoures with update command
if not source[ORM.DATASOURCE_UPDATE]:
continue
# Test filter
if 'all' != name_filter:
is_match = \
(name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \
(name_filter == source[ORM.DATASOURCE_NAME]) or \
(name_filter == source[ORM.DATASOURCE_SOURCE]) or \
(name_filter == source[ORM.DATASOURCE_DATA])
if not is_match:
status_str += " Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION]
continue
# Test the update time
if not force_all:
# testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *,
# testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)
#print("Update datasource:'%s'" % source[ORM.DATASOURCE_DESCRIPTION])
# Get the datasource values
update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY]
if not source[ORM.DATASOURCE_LASTUPDATEDDATE]:
# Force update if no registed updated date for datasource (i.e. at Init phase)
last_updated_date = time_now - timedelta(days=365)
else:
last_updated_date = datetime.strptime(source[ORM.DATASOURCE_LASTUPDATEDDATE], ORM.DATASOURCE_DATETIME_FORMAT)
# Get the update presets
update_time = source[ORM.DATASOURCE_UPDATE_TIME]
delta_minutes = get_tag_key(update_time,'minutes',None)
delta_minute = get_tag_key(update_time,'minute',None)
delta_hour = get_tag_key(update_time,'hour',None)
delta_weekday = get_tag_key(update_time,'weekday',None)
delta_day = get_tag_key(update_time,'day',None)
# Calulate the next update datetime
if ORM.DATASOURCE_MINUTELY == update_frequency:
if not delta_minutes:
print("ERROR:Missing minutes in '%s' for '%s'" % (source[ORM.DATASOURCE_DESCRIPTION],update_time))
delta_minutes = 10
testdiff = timedelta(minutes=int(delta_minutes))
elif ORM.DATASOURCE_HOURLY == update_frequency:
testdiff = timedelta(hours=1)
elif ORM.DATASOURCE_DAILY == update_frequency:
testdiff = timedelta(days=1)
elif ORM.DATASOURCE_WEEKLY == update_frequency:
testdiff = timedelta(weeks=1)
elif ORM.DATASOURCE_MONTHLY == update_frequency:
testdiff = timedelta(months=1)
elif ORM.DATASOURCE_ONDEMAND == update_frequency:
continue
elif ORM.DATASOURCE_ONSTARTUP == update_frequency:
continue
testdate = last_updated_date + testdiff
# Adjust for update presets
if None != delta_minute:
# Force to selected day of month
testdate = datetime(testdate.year, testdate.month, testdate.day, testdate.hour, int(delta_minute), testdate.second)
if None != delta_day:
# Force to selected day of month
testdate = datetime(testdate.year, testdate.month, testdate.day, int(delta_hour), testdate.minute, testdate.second)
if None != delta_day:
# Force to selected day of month
testdate = datetime(testdate.year, testdate.month, int(delta_day), testdate.hour, testdate.minute, testdate.second)
if None != delta_weekday:
# Force to selected day of week
testdiff = timedelta( days=(int(delta_weekday) - testdate.weekday()) )
testdate += testdiff
# Not yet?
if testdate > time_now:
status_str += " Skip '%s': update time not reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime(ORM.DATASOURCE_DATETIME_FORMAT))
continue
else:
status_str += " UPDATE '%s': update time reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime(ORM.DATASOURCE_DATETIME_FORMAT))
# Execute the update
if is_trial:
print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
status_str += " > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
else:
print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
master_write("SRTOOL_UPDATE:%s:%s:%s\n" %(time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT),source[ORM.DATASOURCE_DESCRIPTION],source[ORM.DATASOURCE_UPDATE]))
update_command = source[ORM.DATASOURCE_UPDATE]
if force_all:
update_command += " --force"
os.system(os.path.join(script_pathname, update_command))
# Reset datasource's last_updated_date
sql = "UPDATE orm_datasource SET lastUpdatedDate=? WHERE id=?"
cur_write.execute(sql, (time_now.strftime(ORM.DATASOURCE_DATETIME_FORMAT),source[ORM.DATASOURCE_ID],) )
conn.commit()
conn.close()
# Status summary
with open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') as status_file:
status_file.write(status_str)
if verbose:
print(status_str)
#time must be in '%H:%M:%S' format
def configure_ds_update(datasource_description, frequency, time):
conn = sqlite3.connect(srtDbName)
cur = conn.cursor()
sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?"
cur.execute(sql, (frequency, time, datasource_description))
conn.commit()
conn.close()
#################################
# List update data sources
#
def list():
conn = sqlite3.connect(srtDbName)
cur = conn.cursor()
cur_write = conn.cursor()
format_str = "%16s %7s %14s %10s %28s %s"
print("SRTool Update List:")
print(format_str % ('Data','Source','Name','Frequency','Offset','Description'))
print("================ ======= ============== ========== ============================ ===========================================")
#get sources that have update command
sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
for source in sources:
# Only process datasoures with update command
if not source[ORM.DATASOURCE_UPDATE]:
continue
frequency_str = ORM.get_orm_string(source[ORM.DATASOURCE_UPDATE_FREQUENCY],ORM.DATASOURCE_FREQUENCY_STR)
print(format_str % (source[ORM.DATASOURCE_DATA],source[ORM.DATASOURCE_SOURCE],source[ORM.DATASOURCE_NAME],frequency_str,source[ORM.DATASOURCE_UPDATE_TIME],source[ORM.DATASOURCE_DESCRIPTION]))
if verbose:
print(format_str % ('',source[ORM.DATASOURCE_LASTMODIFIEDDATE],source[ORM.DATASOURCE_LASTUPDATEDDATE],'','',''))
#################################
# Start 'cron' job for updates
#
def cron_start():
pid = os.getpid()
master_write("SRTOOL_UPDATE:%s:Starting -v update cron job, pid=%s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),pid))
# Preserve this app's pid
srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
with open(srt_update_pid_file, 'w') as pidfile:
pidfile.write("%s" % pid)
# Loop until app is killed
extra_line = False
while True:
run_updates(False,'all',False)
# Toggle an extra line in the log to make updates obvious
if extra_line:
extra_line = False
os.system("echo '' >> %s" % os.path.join(script_pathname,UPDATE_STATUS_LOG))
else:
extra_line = True
# Default to 5 minute loop
time.sleep(5 * 60)
def cron_stop():
# Fetch the stored update app's pid
srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
if os.path.isfile(srt_update_pid_file):
with open(srt_update_pid_file, 'r') as pidfile:
pid = pidfile.read()
print("KILL UPDATE:%s" % pid)
# Kill the update app
os.system("kill %s" % pid)
os.system("rm %s" % srt_update_pid_file)
master_write("SRTOOL_UPDATE:%s:Stopping -^ update cron job, pid=%s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT),pid))
else:
print("No running update task file found")
#################################
# main loop
#
def main(argv):
global verbose
global master_log
# setup
parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database')
parser.add_argument('--cron-start', action='store_const', const='cron_start', dest='command', help='Start the SRTool background updater')
parser.add_argument('--cron-stop', action='store_const', const='cron_stop', dest='command', help='Stop the SRTool background updater')
parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List data sources')
parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='Update scheduled data sources')
parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name')
parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run')
parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')
args = parser.parse_args()
master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a")
verbose = args.verbose
name_filter = 'all'
if args.name_filter:
name_filter = args.name_filter
if 'list' == args.command:
list()
elif 'run-updates' == args.command:
try:
print("BEGINNING UPDATING DATASOURCES... this MAY take a long time")
run_updates(args.force,name_filter,args.is_trial)
master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today()))
print("FINISHED UPDATING ALL DATASOURCES\n")
except Exception as e:
print("FAILED UPDATING ALL DATASOURCES (%s)" % e)
master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
elif args.configure_ds_update:
try:
print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0])
configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2])
master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0]))
except Exception as e:
print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0])
master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e))
elif 'cron_start' == args.command:
cron_start()
elif 'cron_stop' == args.command:
cron_stop()
else:
print("Command not found")
master_log.close()
if __name__ == '__main__':
global script_pathname
from os.path import abspath
script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
main(sys.argv[1:])
|