aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Reyna <David.Reyna@windriver.com>2019-01-14 15:04:36 -0800
committerDavid Reyna <David.Reyna@windriver.com>2019-01-14 15:04:36 -0800
commit959cc8196aab2231b7f6f05c7e0151f8de46ea48 (patch)
tree9c3c17f03ba0e78bae986311d4f7b3368a6e34bc
parenta10fe453f2634d15cca1495e84085e43c65fbd3b (diff)
downloadsrtool-959cc8196aab2231b7f6f05c7e0151f8de46ea48.tar.gz
srtool-959cc8196aab2231b7f6f05c7e0151f8de46ea48.tar.bz2
srtool-959cc8196aab2231b7f6f05c7e0151f8de46ea48.zip
srtool: complete auto-update support
Complete the support for backgroup data source updates: * Add cron-start,cron-stop to srtool_update * Have cron update run as a user space script to avoid sudo * Hook cron-start,cron-stop into srt start,stop * Add list command to show update sources * Have force command propagate to update script calls, and add force option to all source scripts * Add 'srt manage update ...' for access to the update functions * Add flag SRT_SKIP_AUTOUPDATE and srt option noautoupdate to disable the automatic update app for development assistance Related Fixes: * Set the schema generator to always update on startup (13138) * Fix CVE 'recommend' default to the integer zero (13139) with auto-fix at startup for existing databases [YOCTO #13131] [YOCTO #13138] [YOCTO #13139] Signed-off-by: David Reyna <David.Reyna@windriver.com>
-rwxr-xr-xbin/acme/srtool_acme.py2
-rwxr-xr-xbin/common/datasource.json4
-rwxr-xr-xbin/common/srtool_common.py58
-rwxr-xr-xbin/common/srtool_update.py123
-rwxr-xr-xbin/debian/srtool_debian.py1
-rwxr-xr-xbin/dev_tools/srt_env.sh2
-rwxr-xr-xbin/mitre/srtool_mitre.py4
-rwxr-xr-xbin/nist/srtool_nist.py2
-rwxr-xr-xbin/redhat/srtool_redhat.py1
-rwxr-xr-xbin/srt19
-rwxr-xr-xbin/yp/srtool_yp.py2
-rw-r--r--lib/orm/management/commands/lsupdates.py2
-rw-r--r--lib/orm/models.py4
-rwxr-xr-xlib/srtmain/management/commands/update.py41
14 files changed, 230 insertions, 35 deletions
diff --git a/bin/acme/srtool_acme.py b/bin/acme/srtool_acme.py
index 02750ba5..c302e1cb 100755
--- a/bin/acme/srtool_acme.py
+++ b/bin/acme/srtool_acme.py
@@ -137,6 +137,8 @@ def main(argv):
parser = argparse.ArgumentParser(description='srtool_acme.py: Manage SRTool to ACME Corp')
parser.add_argument('--init-products', '-p', action='store_const', const='init_products', dest='command', help='Init and import ACME Products')
parser.add_argument('--file', dest='file', help='Source file')
+
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Verbose debugging')
args = parser.parse_args()
diff --git a/bin/common/datasource.json b/bin/common/datasource.json
index 789c67cc..4e175b40 100755
--- a/bin/common/datasource.json
+++ b/bin/common/datasource.json
@@ -22,9 +22,9 @@
"description" : "SRT database schema for command line tools",
"cve_filter" : "",
"init" : "bin/common/srtool_common.py --generate-schema-header",
- "update" : "",
+ "update" : "bin/common/srtool_common.py --generate-schema-header",
"lookup" : "",
- "update_frequency" : "5",
+ "update_frequency" : "6",
"update_time" : "{}"
},
{
diff --git a/bin/common/srtool_common.py b/bin/common/srtool_common.py
index 62cc95dc..f7e09764 100755
--- a/bin/common/srtool_common.py
+++ b/bin/common/srtool_common.py
@@ -5,7 +5,7 @@
#
# Security Response Tool Implementation
#
-# Copyright (C) 2017 Wind River Systems
+# Copyright (C) 2017-2018 Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -44,7 +44,9 @@ import pytz
# it may not exist on the first pass
try:
from srt_schema import ORM
-except ImportError:
+except:
+ # Do a pass so that '--generate-schema-header' can fix it
+ print("Warning: srt_schema not yet created or bad format")
pass
srtDbName = 'srt.sqlite'
@@ -287,7 +289,7 @@ def score_new_cves(cve_filter):
# Scan the open CVEs
if 'NEW' == cve_filter:
- sql = "SELECT * FROM orm_cve WHERE (status='%s' OR status='%s') AND recommend_list = '';" % (ORM.STATUS_NEW,ORM.STATUS_NEW_RESERVED)
+ sql = "SELECT * FROM orm_cve WHERE (status='%s' OR status='%s');" % (ORM.STATUS_NEW,ORM.STATUS_NEW_RESERVED)
cur.execute(sql)
elif cve_filter.startswith('CVE-'):
cur.execute('SELECT * FROM orm_cve WHERE name LIKE "'+cve_filter+'%"')
@@ -312,9 +314,15 @@ def score_new_cves(cve_filter):
record_count = 0
write_count = 0
ds_count = 0
+ time_now = datetime.now()
for i,cve in enumerate(cur):
cve_name = cve[ORM.CVE_NAME]
+ if cve[ORM.CVE_SCORE_DATE]:
+ #cve_score_date = datetime.strptime(source[ORM.CVE_SCORE_DATE], '%Y-%m-%d %H:%M:%S')
+ # If there is any score_date, then nothing to do here
+ continue
+
# Progress indicator support
if 0 == i % 10:
print('%04d: %20s\r' % (i,cve_name), end='')
@@ -349,9 +357,10 @@ def score_new_cves(cve_filter):
sql = ''' UPDATE orm_cve
SET recommend = ?,
recommend_list = ?,
- packages = ?
+ packages = ?,
+ score_date = ?
WHERE id = ?'''
- cur_write.execute(sql, (recommend, recommend_list, cve_packages, cve[ORM.CVE_ID]))
+ cur_write.execute(sql, (recommend, recommend_list, cve_packages, time_now.strftime('%Y-%m-%d %H:%M:%S'), cve[ORM.CVE_ID]))
write_count += 1
if verbose: print(" %d:%s:%s" % (recommend,recommend_list,cve_packages))
@@ -486,11 +495,15 @@ def gen_schema_header():
fd.write(" %s_%s = %d\n" % ('DATASOURCE','WEEKLY' ,3))
fd.write(" %s_%s = %d\n" % ('DATASOURCE','MONTHLY' ,4))
fd.write(" %s_%s = %d\n" % ('DATASOURCE','ONDEMAND' ,5))
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','ONSTARTUP' ,6))
+ fd.write(" %s_%s = '%s'\n" % ('DATASOURCE','FREQUENCY_STR', \
+ 'Minute,Hourly,Daily,Weekly,Monthly,OnDemand.OnStartup' \
+ ))
fd.write("\n")
#################################
-# fix_name_sort
+# fixups
#
# Recompute all of the CVE name_sort fields
@@ -515,6 +528,31 @@ def fix_name_sort():
cur_write.execute(sql, (name_sort, cve[ORM.CVE_ID],))
conn.commit()
+# Reset empty CVE recommend fields to the proper integer zero
+def fix_cve_recommend():
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+ cur_write = conn.cursor()
+
+ cur.execute('SELECT * FROM orm_cve WHERE recommend = ""')
+ i = 0
+ for cve in cur:
+ i += 1
+ name_sort = get_name_sort(cve[ORM.CVE_NAME])
+
+ # Progress indicator support
+ if 0 == i % 10:
+ print('%05d: %20s to %20s\r' % (i,cve[ORM.CVE_NAME],name_sort), end='')
+ if (0 == i % 200):
+ conn.commit()
+
+ sql = ''' UPDATE orm_cve
+ SET recommend = ?
+ WHERE id = ?'''
+ cur_write.execute(sql, (0, cve[ORM.CVE_ID],))
+ print("CVE RECOMMEND FIX COUNT=%d" % i)
+ conn.commit()
+
#################################
# main loop
#
@@ -529,11 +567,13 @@ def main(argv):
parser.add_argument('--init-package-keywords', '-p', action='store_const', const='init_package_keywords', dest='command', help='Initialize package keywords')
parser.add_argument('--init-notify-categories', '-n', action='store_const', const='init_notify_categories', dest='command', help='Initialize notify categories')
parser.add_argument('--score-new-cves', '-s', dest='score_new_cves', help='Score CVEs for triage [NEW|CVE-1234]')
- parser.add_argument('--generate-schema-header', action='store_const', const='gen_schema_header', dest='command', help='Generate database schema header')
+ parser.add_argument('--generate-schema-header', '-g', action='store_const', const='gen_schema_header', dest='command', help='Generate database schema header')
+ parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
parser.add_argument('--skip', dest='skip', help='Debugging: skip record count')
parser.add_argument('--count', dest='count', help='Debugging: short run record count')
parser.add_argument('--fix-name-sort', action='store_const', const='fix_name_sort', dest='command', help='Recalulate the CVE name sort values')
+ parser.add_argument('--fix-cve-recommend', action='store_const', const='fix_cve_recommend', dest='command', help='Fix the empty CVE recommend values')
args = parser.parse_args()
verbose = args.verbose
@@ -554,8 +594,12 @@ def main(argv):
score_new_cves(args.score_new_cves)
elif 'gen_schema_header' == args.command:
gen_schema_header()
+ ### TO-DO: TEMPORARY WORKAROUND
+ fix_cve_recommend()
elif 'fix_name_sort' == args.command:
fix_name_sort()
+ elif 'fix_cve_recommend' == args.command:
+ fix_cve_recommend()
else:
print("Command not found")
diff --git a/bin/common/srtool_update.py b/bin/common/srtool_update.py
index f73d6800..a093a7b6 100755
--- a/bin/common/srtool_update.py
+++ b/bin/common/srtool_update.py
@@ -22,12 +22,10 @@
import os
import sys
-import re
import argparse
import sqlite3
-import subprocess
import json
-import urllib
+import time
# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -43,12 +41,18 @@ from urllib.parse import urlparse
is_verbose = False
srtDbName = 'srt.sqlite'
-UPDATE_STATUS_LOG = 'update_status.log'
+UPDATE_STATUS_LOG = 'update_logs/update_status.log'
+SRT_UPDATE_PID_FILE = '.srtupdate.pid'
#################################
# Common routines
#
+# Safe write even when in cron backgroup mode
+def master_write(msg):
+ master_log.write(msg)
+ master_log.flush()
+
# quick development/debugging support
def _log(msg):
DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2
@@ -67,8 +71,8 @@ def get_tag_key(tag,key,default=''):
return d[key]
else:
return default
- except:
- print("ERROR TAG FORMAT:get_tag_key(%s,%s)" % (tag,key))
+ except Exception as e:
+ print("ERROR TAG FORMAT:get_tag_key(%s,%s)=%s" % (tag,key,e))
return default
#################################
@@ -88,8 +92,8 @@ def run_updates(force_all,name_filter,is_trial):
cur = conn.cursor()
cur_write = conn.cursor()
- time_now = datetime.now() #datetime.now(pytz.utc)
- print("time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S'))
+ time_now = datetime.now() #datetime.now(pytz.utc)
+ print("SRTool Update: time_now = %s" % time_now.strftime('%Y-%m-%d %H:%M:%S'))
status_str = "====================\n"
status_str += "Update: Date=%s,Filter='%s',Force=%s\n" % (time_now.strftime('%Y-%m-%d %H:%M:%S'),name_filter,force_all)
@@ -149,6 +153,8 @@ def run_updates(force_all,name_filter,is_trial):
testdiff = timedelta(months=1)
elif ORM.DATASOURCE_ONDEMAND == update_frequency:
continue
+ elif ORM.DATASOURCE_ONSTARTUP == update_frequency:
+ continue
testdate = last_modified_date + testdiff
# Adjust for update presets
@@ -180,7 +186,11 @@ def run_updates(force_all,name_filter,is_trial):
else:
print("Update required\t...\texecuting '%s'" % (source[ORM.DATASOURCE_UPDATE]))
status_str += " > EXECUTE: execute '%s'\n" % (source[ORM.DATASOURCE_UPDATE])
- os.system(os.path.join(script_pathname, source[ORM.DATASOURCE_UPDATE]))
+ master_write("SRTOOL_UPDATE:%s:%s:%s\n" %(time_now.strftime('%Y-%m-%d %H:%M:%S'),source[ORM.DATASOURCE_DESCRIPTION],source[ORM.DATASOURCE_UPDATE]))
+ update_command = source[ORM.DATASOURCE_UPDATE]
+ if force_all:
+ update_command += " --force"
+ os.system(os.path.join(script_pathname, update_command))
# Reset datasource's last_modified_date
sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE id=?"
@@ -189,9 +199,8 @@ def run_updates(force_all,name_filter,is_trial):
conn.close()
# Status summary
- fd=open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w')
- fd.write(status_str)
- fd.close()
+ with open(os.path.join(script_pathname,UPDATE_STATUS_LOG), 'w') as status_file:
+ status_file.write(status_str)
if verbose:
print(status_str)
@@ -208,25 +217,92 @@ def configure_ds_update(datasource_description, frequency, time):
#################################
+# List update data sources
+#
+
+def list():
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+ cur_write = conn.cursor()
+
+ format_str = "%14s %7s %14s %10s %28s %s"
+
+ print("SRTool Update List:")
+ status_str = "====================\n"
+ print(format_str % ('Data','Source','Name','Frequency','Offset','Description'))
+ #get sources that have update command
+ sources = cur.execute("SELECT * FROM orm_datasource").fetchall()
+ for source in sources:
+ # Only process datasoures with update command
+ if not source[ORM.DATASOURCE_UPDATE]:
+ continue
+ frequency_str = ORM.DATASOURCE_FREQUENCY_STR.split(',')[source[ORM.DATASOURCE_UPDATE_FREQUENCY]]
+ print(format_str % (source[ORM.DATASOURCE_DATA],source[ORM.DATASOURCE_SOURCE],source[ORM.DATASOURCE_NAME],frequency_str,source[ORM.DATASOURCE_UPDATE_TIME],source[ORM.DATASOURCE_DESCRIPTION]))
+
+#################################
+# Start 'cron' job for updates
+#
+
+def cron_start():
+ pid = os.getpid()
+ master_write("SRTOOL_UPDATE:%s:Starting -v update cron job, pid=%s\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),pid))
+
+ # Preserve this app's pid
+ srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
+ with open(srt_update_pid_file, 'w') as pidfile:
+ pidfile.write("%s" % pid)
+
+ # Loop until app is killed
+ extra_line = False
+ while True:
+ run_updates(False,'all',False)
+ # Toggle an extra line in the log to make updates obvious
+ if extra_line:
+ extra_line = False
+ os.system("echo '' >> %s" % os.path.join(script_pathname,UPDATE_STATUS_LOG))
+ else:
+ extra_line = True
+ # Default to 5 minute loop
+ time.sleep(5 * 60)
+
+def cron_stop():
+ # Fetch the stored update app's pid
+ srt_update_pid_file = os.path.join(script_pathname,SRT_UPDATE_PID_FILE)
+ if os.path.isfile(srt_update_pid_file):
+ with open(srt_update_pid_file, 'r') as pidfile:
+ pid = pidfile.read()
+ print("KILL UPDATE:%s" % pid)
+ # Kill the update app
+ os.system("kill %s" % pid)
+ os.system("rm %s" % srt_update_pid_file)
+ master_write("SRTOOL_UPDATE:%s:Stopping -^ update cron job, pid=%s\n" % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),pid))
+ else:
+ print("No running update task file found")
+
+#################################
# main loop
#
+
def main(argv):
global verbose
+ global master_log
# setup
parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database')
- parser.add_argument('--cron-start', action='store_const', const='cron-start', dest='command', help='Start the SRTool backgroud updater')
- parser.add_argument('--cron-stop', action='store_const', const='cron-stop', dest='command', help='Stop the SRTool backgroud updater')
+ parser.add_argument('--cron-start', action='store_const', const='cron_start', dest='command', help='Start the SRTool backgroud updater')
+ parser.add_argument('--cron-stop', action='store_const', const='cron_stop', dest='command', help='Stop the SRTool backgroud updater')
- parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources')
- parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
+ parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List data sources')
+ parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='Update scheduled data sources')
parser.add_argument('--name-filter', '-n', dest='name_filter', help='Filter for datasource name')
- parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')
+ parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run')
+ parser.add_argument('--configure_ds_update', '-T', nargs=3, help='Set update frequency and time for specified datasource. Check bin/README.txt for more info')
+
args = parser.parse_args()
master_log = open(os.path.join(script_pathname, "update_logs/master_log.txt"), "a")
@@ -236,13 +312,15 @@ def main(argv):
if args.name_filter:
name_filter = args.name_filter
- if 'run-updates' == args.command:
- if True: #try:
+ if 'list' == args.command:
+ list()
+ elif 'run-updates' == args.command:
+ try:
print("BEGINNING UPDATING DATASOURCES... this MAY take a long time")
run_updates(args.force,name_filter,args.is_trial)
master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today()))
print("FINISHED UPDATING ALL DATASOURCES\n")
- if False: #except Exception as e:
+ except Exception as e:
print("FAILED UPDATING ALL DATASOURCES (%s)" % e)
master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
elif args.configure_ds_update:
@@ -253,7 +331,10 @@ def main(argv):
except Exception as e:
print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0])
master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e))
-
+ elif 'cron_start' == args.command:
+ cron_start()
+ elif 'cron_stop' == args.command:
+ cron_stop()
else:
print("Command not found")
master_log.close()
diff --git a/bin/debian/srtool_debian.py b/bin/debian/srtool_debian.py
index 1c267f27..91092afe 100755
--- a/bin/debian/srtool_debian.py
+++ b/bin/debian/srtool_debian.py
@@ -211,6 +211,7 @@ def main(argv):
# parser.add_argument('--url-file', dest='url_file', help='CVE URL extension')
parser.add_argument('--file', dest='cve_file', help='Local CVE source file')
parser.add_argument('--cve-detail', '-d', dest='cve_detail', help='Fetch CVE detail')
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update')
parser.add_argument('--verbose', '-v', action='store_true', dest='is_verbose', help='Enable verbose debugging output')
args = parser.parse_args()
diff --git a/bin/dev_tools/srt_env.sh b/bin/dev_tools/srt_env.sh
index 65a4d886..311a2af3 100755
--- a/bin/dev_tools/srt_env.sh
+++ b/bin/dev_tools/srt_env.sh
@@ -20,6 +20,7 @@ if [ "debug" != "$mode" ] ; then
export SRTDBG_SKIP_DEFECT_IMPORT=0
export SRTDBG_SKIP_CVE_IMPORT=0
export SRTDBG_SKIP_CPE_IMPORT=0
+ export SRT_SKIP_AUTOUPDATE=0
# Email credentials
export SRT_SMTP=temp_smnp_server
# Defect (Jira) and Email credentials
@@ -37,6 +38,7 @@ else
export SRTDBG_SKIP_DEFECT_IMPORT=1
export SRTDBG_SKIP_CVE_IMPORT=0
export SRTDBG_SKIP_CPE_IMPORT=0
+ export SRT_SKIP_AUTOUPDATE=1
# Email credentials
export SRT_SMTP=temp_smnp_server
# Defect (Jira) and Email credentials
diff --git a/bin/mitre/srtool_mitre.py b/bin/mitre/srtool_mitre.py
index 021b417a..5920b4dd 100755
--- a/bin/mitre/srtool_mitre.py
+++ b/bin/mitre/srtool_mitre.py
@@ -315,7 +315,7 @@ def append_cve_database(is_init,file_xml):
sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, srt_updated, packages)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'''
- cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, status, '', '', '', '', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],'', '', '', '', '', '', datetime.now(),''))
+ cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, status, '', '', 'CVE', 'MITRE', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],0, '', '', '', '', '', datetime.now(),''))
cve_id = cur.lastrowid
print("MITRE:ADDED %20s\r" % cve_name)
@@ -407,7 +407,7 @@ def main(argv):
parser.add_argument('--url-file', dest='url_file', help='CVE URL extension')
parser.add_argument('--cve-detail', '-d', dest='cve_detail', help='Fetch CVE detail')
parser.add_argument('--file', dest='cve_file', help='Local CVE source file')
- parser.add_argument('--force-update', '-f', action='store_true', dest='force_update', help='Force update')
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update')
parser.add_argument('--verbose', '-v', action='store_true', dest='is_verbose', help='Enable verbose debugging output')
parser.add_argument('--dump', '-D', action='store_const', const='dump', dest='command', help='test dump data')
parser.add_argument('--dump2', '-2', action='store_const', const='dump2', dest='command', help='test dump data')
diff --git a/bin/nist/srtool_nist.py b/bin/nist/srtool_nist.py
index 807afbc1..8138387c 100755
--- a/bin/nist/srtool_nist.py
+++ b/bin/nist/srtool_nist.py
@@ -663,7 +663,7 @@ def main(argv):
parser.add_argument('--file', dest='cve_file', help='Local CVE source file')
parser.add_argument('--update_nist_incremental', '-i', action='store_const', const='update_nist_incremental', dest='command', help='Check nvd.nist.gov/vuln/data-feeds for updates')
parser.add_argument('--cve-detail', '-d', dest='cve_detail', help='Lookup CVE data')
- parser.add_argument('--force-update', '-f', action='store_true', dest='force_update', help='Force update')
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Force update')
args = parser.parse_args()
verbose = args.verbose
diff --git a/bin/redhat/srtool_redhat.py b/bin/redhat/srtool_redhat.py
index 9aaf0bae..27a568a7 100755
--- a/bin/redhat/srtool_redhat.py
+++ b/bin/redhat/srtool_redhat.py
@@ -186,6 +186,7 @@ def main(argv):
parser = argparse.ArgumentParser(description='srtool_redhat.py: manage Red Hat CVE data')
parser.add_argument('--cve-detail', '-d', dest='cve_detail', help='Fetch CVE detail')
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update')
parser.add_argument('--verbose', '-v', action='store_true', dest='is_verbose', help='Enable verbose debugging output')
args = parser.parse_args()
diff --git a/bin/srt b/bin/srt
index 7af2491f..41753b74 100755
--- a/bin/srt
+++ b/bin/srt
@@ -22,6 +22,7 @@ HELP="
Usage: source srt start|stop [webport=<address:port>]
Optional arguments:
[webport] Set the SRTool server port (default: localhost:8000)
+ [noautoupdate] Disable the auto update server
"
databaseCheck()
@@ -76,6 +77,11 @@ webserverKillAll()
rm ${pidfile}
fi
done
+
+ # Stop the Update app
+ if [ 0 -eq $no_auto_update ] ; then
+ ./bin/common/srtool_update.py --cron-stop
+ fi
}
webserverStartAll()
@@ -107,6 +113,11 @@ webserverStartAll()
echo "SRTool webserver started at http://$ADDR_PORT"
fi
+ # Start the Update app
+ if [ 0 -eq $no_auto_update ] ; then
+ ./bin/common/srtool_update.py --cron-start > /dev/null 2>&1 &
+ echo "SRTool update service started at PID $!"
+ fi
return $retval
}
@@ -186,6 +197,11 @@ touch $SRT_BASE_DIR/update_logs/master_log.txt
ADDR_PORT="localhost:8000"
unset CMD
manage_cmd=""
+if [ "1" = "$SRT_SKIP_AUTOUPDATE" ] ; then
+ no_auto_update=1
+else
+ no_auto_update=0
+fi
for param in $*; do
case $param in
start )
@@ -207,6 +223,9 @@ for param in $*; do
ADDR_PORT="localhost:$PORT"
fi
;;
+ noautoupdate )
+ no_auto_update=1
+ ;;
--help)
echo "$HELP"
exit 0
diff --git a/bin/yp/srtool_yp.py b/bin/yp/srtool_yp.py
index 2f9384de..552eadf1 100755
--- a/bin/yp/srtool_yp.py
+++ b/bin/yp/srtool_yp.py
@@ -122,6 +122,8 @@ def main(argv):
parser = argparse.ArgumentParser(description='srtool_wr.py: Manage SRTool to Yocto Project')
parser.add_argument('--init-products', '-p', action='store_const', const='init_products', dest='command', help='Init and import Yocto Project Products')
parser.add_argument('--file', dest='file', help='Source file')
+
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Verbose debugging')
args = parser.parse_args()
diff --git a/lib/orm/management/commands/lsupdates.py b/lib/orm/management/commands/lsupdates.py
index 7bae1720..ce085a11 100644
--- a/lib/orm/management/commands/lsupdates.py
+++ b/lib/orm/management/commands/lsupdates.py
@@ -338,7 +338,7 @@ class Command(BaseCommand):
data_sources=DataSource.objects.all().order_by('key')
for source in data_sources:
- if source.loaded:
+ if source.loaded and not (source.update_frequency == DataSource.ONSTARTUP):
logger.info("Skipping source data from %s",source.description)
print("Skipping datasource %s (already loaded)" % (source.description))
_log("Skipping datasource %s (already loaded)" % (source.description))
diff --git a/lib/orm/models.py b/lib/orm/models.py
index 99438285..8d95c319 100644
--- a/lib/orm/models.py
+++ b/lib/orm/models.py
@@ -148,6 +148,7 @@ class DataSource(models.Model):
WEEKLY = 3
MONTHLY = 4
ONDEMAND = 5
+ ONSTARTUP = 6
FREQUENCY = (
(MINUTELY, 'Minute'),
(HOURLY, 'Hourly'),
@@ -155,6 +156,7 @@ class DataSource(models.Model):
(WEEKLY, 'Weekly'),
(MONTHLY, 'Monthly'),
(ONDEMAND, 'OnDemand'),
+ (ONSTARTUP, 'OnStartup'),
)
key = models.CharField(max_length=20)
@@ -273,7 +275,7 @@ class Cve(models.Model):
packages = models.TextField(blank=True)
- score_date = models.DateField(null=True, blank=True)
+ score_date = models.DateField(default=0)
srt_updated = models.DateTimeField(auto_now=True)
@property
diff --git a/lib/srtmain/management/commands/update.py b/lib/srtmain/management/commands/update.py
new file mode 100755
index 00000000..8f746aea
--- /dev/null
+++ b/lib/srtmain/management/commands/update.py
@@ -0,0 +1,41 @@
+from django.core.management.base import BaseCommand, CommandError
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import OperationalError
+import os
+
+class Command(BaseCommand):
+ help = "Trigger a data source update"
+
+ def add_arguments(self, parser):
+ parser.add_argument('--cron-start', action='store_const', const='cron_start', dest='command', help='Start the SRTool backgroud updater')
+ parser.add_argument('--cron-stop', action='store_const', const='cron_stop', dest='command', help='Stop the SRTool backgroud updater')
+
+ parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List data sources')
+ parser.add_argument('--run-updates', '-u', action='store_const', const='run-updates', dest='command', help='update scheduled data sources')
+ parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update')
+ parser.add_argument('--name-filter', '-n', nargs='+', type=str, dest='name_filter', help='Filter for datasource name')
+
+ parser.add_argument('--verbose', action='store_true', dest='verbose', help='Debugging: verbose output')
+ parser.add_argument('--trial', '-t', action='store_true', dest='is_trial', help='Debugging: trial run')
+
+ def handle(self, *args, **options):
+ #print("UPDATE:%s|%s" % (str(args),str(options)))
+
+ command = ''
+ if 'cron_start' == options['command']: command = '--cron-start'
+ if 'cron_stop' == options['command']: command = '--cron-stop'
+ if 'list' == options['command']: command = '--list'
+ if 'run-updates' == options['command']: command = '--run-updates'
+
+ # NOTE: we have to do shenanigans with name_filter to support spaces
+ name_filter = '--name-filter "%s"' % ' '.join(options['name_filter']) if options['name_filter'] else ''
+
+ force = '--force' if options['force'] else ''
+ is_trial = '--trial' if options['is_trial'] else ''
+ verbose = '--verbose' if options['verbose'] or (options['verbosity'] > 1) else ''
+ context = '> /dev/null 2>&1 &' if 'cron_start' == options['command'] else ''
+
+ update_command = "./bin/common/srtool_update.py %s %s %s %s %s %s" % (command,name_filter,force,is_trial,verbose,context)
+ print("RUN: %s" % (update_command))
+ os.chdir(os.environ['SRT_BASE_DIR'])
+ os.system("%s" % (update_command))