1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
|
#!/usr/bin/env python3
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Security Response Tool Commandline Tool
#
# Copyright (C) 2018 Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import os
import sys
import re
import argparse
import sqlite3
import subprocess
import json
import urllib
# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, dir_path)
from srt_schema import ORM
from datetime import datetime, timedelta, date
from pprint import pprint
from urllib.request import urlopen, URLError
from urllib.parse import urlparse
# setup
is_verbose = False
srtDbName = 'srt.sqlite'
UPDATE_STATUS_LOG = 'update_status.log'
#################################
# Common routines
#
# quick development/debugging support
def _log(msg):
DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2
DBG_LOG = os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log'
if 1 == DBG_LVL:
print(msg)
elif 2 == DBG_LVL:
f1=open(DBG_LOG, 'a')
f1.write("|" + msg + "|\n" )
f1.close()
def get_tag_key(tag,key,default=''):
try:
d = json.loads(tag)
if key in d:
return d[key]
else:
return default
except:
print("ERROR TAG FORMAT:get_tag_key(%s,%s)" % (tag,key))
return default
#################################
# Update routines
#
# Example 'update_time' filters:
# MINUTELY = 0 "{\"minutes\":\"10\"}" # every ten minutes
# HOURLY = 1 "{\"minute\":\"10\"}" # at ten minutes past the hour
# DAILY = 2 "{\"hour\":\"2\"}" # at 2 hours after midnight
# WEEKLY = 3 "{\"weekday\":\"6\",\"hour\":\"2\"}" # day of week, hour
# MONTHLY = 4 "{\"day\":\"1\"\"hour\":\"2\"}" # day of month
# ONDEMAND = 5 "{}" # only on demand
def run_updates(force_all,name_filter,is_trial):
conn = sqlite3.connect(srtDbName)
cur = conn.cursor()
cur_write = conn.cursor()
time_now = datetime.now() #datetime.now(pytz.utc)
print("time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S'))
status_str = "====================\n"
status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all)
#get sources that have update command
sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
for source in sources:
# Only process datasoures with update command
if not source[ORM.DATASOURCE_UPDATE]:
continue
# Test filter
if 'all' != name_filter:
is_match = \
(name_filter == source[ORM.DATASOURCE_DESCRIPTION]) or \
(name_filter == source[ORM.DATASOURCE_NAME]) or \
(name_filter == source[ORM.DATASOURCE_SOURCE]) or \
(name_filter == source[ORM.DATASOURCE_DATA])
if not is_match:
status_str += " Skip '%s': name not a match\n" % source[ORM.DATASOURCE_DESCRIPTION]
continue
# Test the update time
if not force_all:
# testdate = datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *,
# testdiff = timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0)
#print("Update datasource:'%s'" % source[ORM.DATASOURCE_DESCRIPTION])
# Get the datasource values
update_frequency = source[ORM.DATASOURCE_UPDATE_FREQUENCY]
if not source[ORM.DATASOURCE_LASTMODIFIEDDATE]:
# Force update if no registed modified date for datasource (e.g. Init)
last_modified_date = time_now-timedelta(days=365)
else:
last_modified_date = datetime.strptime(source[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S')
# Get the update presets
update_time = source[ORM.DATASOURCE_UPDATE_TIME]
delta_minutes = get_tag_key(update_time,'minutes',None)
delta_minute = get_tag_key(update_time,'minute',None)
delta_hour = get_tag_key(update_time,'hour',None)
delta_weekday = get_tag_key(update_time,'weekday',None)
delta_day = get_tag_key(update_time,'day',None)
# Calulate the next update datetime
if ORM.DATASOURCE_MINUTELY == update_frequency:
if not delta_minutes:
print("ERROR:Missing minutes in '%s' for '%s'" % (source[ORM.DATASOURCE_DESCRIPTION],update_time))
delta_minutes = 10
testdiff = timedelta(minutes=int(delta_minutes))
elif ORM.DATASOURCE_HOURLY == update_frequency:
testdiff = timedelta(hours=1)
elif ORM.DATASOURCE_DAILY == update_frequency:
testdiff = timedelta(days=1)
elif ORM.DATASOURCE_WEEKLY == update_frequency:
testdiff = timedelta(weeks=1)
elif ORM.DATASOURCE_MONTHLY == update_frequency:
testdiff = timedelta(months=1)
elif ORM.DATASOURCE_ONDEMAND == update_frequency:
continue
testdate = last_modified_date + testdiff
# Adjust for update presets
if None != delta_minute:
# Force to selected day of month
testdate = datetime(testdate.year, testdate.month, testdate.day, testdate.hour, int(delta_minute), testdate.second)
if None != delta_day:
# Force to selected day of month
testdate = datetime(testdate.year, testdate.month, testdate.day, int(delta_hour), testdate.minute, testdate.second)
if None != delta_day:
# Force to selected day of month
testdate = datetime(testdate.year, testdate.month, int(delta_day), testdate.hour, testdate.minute, testdate.second)
if None != delta_weekday:
# Force to selected day of week
testdiff = timedelta( days=(int(delta_weekday) - testdate.weekday()) )
testdate += testdiff
# Not yet?
if testdate > time_now:
status_str += " Skip '%s': update time not reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S'))
continue
else:
status_str += " UPDATE '%s': update time reached (%s)\n" % (source[ORM.DATASOURCE_DESCRIPTION],testdate.strftime('%Y-%m-%d %H:%M:%S'))
# Execute the update
if is_trial:
print("TRIAL: Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
status_str += " > TRIAL: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
else:
print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
os.system(os.path.join(script_pathname, source[ORM.DATASOURCE_UPDATE]))
# Reset datasource's last_modified_date
sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?"
cur_write.execute(sql, (time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_ID],) )
conn.commit()
conn.close()
# Status summary
fd=open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w')
fd.write(status_str)
fd.close()
if verbose:
print(status_str)
#time must be in '%H:%M:%S' format
def configure_ds_update(datasource_description, frequency, time):
conn = sqlite3.connect(srtDbName)
cur = conn.cursor()
sql = "UPDATE orm_datasource SET update_frequency=?, update_time=? WHERE description=?"
cur.execute(sql, (frequency, time, datasource_description))
conn.commit()
conn.close()
#################################
# main loop
#
def main(argv):
global verbose
# setup
parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database')
parser.add_argument('--cron-start', action='store_const', const='cron-start', dest='command', help='Start the SRTool backgroud updater')
parser.add_argument('--cron-stop', action='store_const', const='cron-stop', dest='command', help='Stop the SRTool backgroud updater')
parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources')
parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name')
parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run')
args = parser.parse_args()
master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a")
verbose = args.verbose
name_filter = 'all'
if args.name_filter:
name_filter = args.name_filter
if 'run-updates' == args.command:
if True: #try:
print("BEGINNING UPDATING DATASOURCES... this MAY take a long time")
run_updates(args.force,name_filter,args.is_trial)
master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today()))
print("FINISHED UPDATING ALL DATASOURCES\n")
if False: #except Exception as e:
print("FAILED UPDATING ALL DATASOURCES (%s)" % e)
master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
elif args.configure_ds_update:
try:
print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0])
configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2])
master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0]))
except Exception as e:
print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0])
master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e))
else:
print("Command not found")
master_log.close()
if __name__ == '__main__':
global script_pathname
from os.path import abspath
script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
main(sys.argv[1:])
|
