diff options
| -rwxr-xr-x | bin/common/datasource.json | 20 | ||||
| -rwxr-xr-x | bin/common/srtool_common.py | 89 | ||||
| -rwxr-xr-x | bin/mitre/datasource_2015.json | 2 | ||||
| -rwxr-xr-x | bin/mitre/datasource_2016.json | 2 | ||||
| -rwxr-xr-x | bin/mitre/datasource_2017.json | 2 | ||||
| -rwxr-xr-x | bin/mitre/datasource_2018.json | 2 | ||||
| -rwxr-xr-x | bin/mitre/datasource_2019.json | 18 | ||||
| -rwxr-xr-x | bin/mitre/srtool_mitre.py | 52 | ||||
| -rwxr-xr-x | bin/nist/datasource_2015.json | 2 | ||||
| -rwxr-xr-x | bin/nist/datasource_2016.json | 2 | ||||
| -rwxr-xr-x | bin/nist/datasource_2017.json | 2 | ||||
| -rwxr-xr-x | bin/nist/datasource_2018.json | 2 | ||||
| -rwxr-xr-x | bin/nist/datasource_2019.json | 18 | ||||
| -rwxr-xr-x | bin/nist/srtool_nist.py | 96 |
14 files changed, 176 insertions, 133 deletions
diff --git a/bin/common/datasource.json b/bin/common/datasource.json index 482cc3ca..2625abca 100755 --- a/bin/common/datasource.json +++ b/bin/common/datasource.json @@ -4,6 +4,12 @@ "name" : "SRTOOL_LOGO", "helptext" : "The SRTool logo", "value" : "Yocto Project,/static/img/logo.png,/" + }, + { + "_comment_": "This is an out-of-box courtesy to provide some CVEs for triage in newly initialized systems", + "name" : "CVE_INIT_NEW_DELTA", + "helptext" : "CVEs found during the initialization phase become 'NEW' instead of 'HISTORICAL' if created within this number of days before today", + "value" : "30" } ], @@ -95,20 +101,6 @@ }, { - "key" : "0920-preset-new", - "data" : "preset_new", - "source" : "common", - "name" : "Preset New", - "description" : "Preset New CVEs", - "cve_filter" : "", - "init" : "bin/common/srtool_common.py --preset-new", - "update" : "", - "lookup" : "", - "lastModifiedDate" : "2018-03-01 01:01:01", - "update_frequency" : "0", - "update_time" : "02:00:00" - }, - { "key" : "0921-common-score", "data" : "score_cves", "source" : "common", diff --git a/bin/common/srtool_common.py b/bin/common/srtool_common.py index 6518aee7..62cc95dc 100755 --- a/bin/common/srtool_common.py +++ b/bin/common/srtool_common.py @@ -373,81 +373,6 @@ def score_new_cves(cve_filter): print("\nUpdated CVEs=%d, Added alternate sources=%d" % (write_count,ds_count)) ################################# -# preset_new -# - -def preset_new(): - global cmd_skip - - conn = sqlite3.connect(srtDbName) - cur = conn.cursor() - cur_write = conn.cursor() - cur_ds = conn.cursor() - date_delta = timedelta(days=30) - init_new_date = datetime.now(pytz.utc) - date_delta - print("\nPreset new data = %s" % init_new_date.strftime("%Y%m%d")) - init_new_date = init_new_date.strftime("%Y%m%d") - - sql = '''SELECT * FROM orm_cve;''' - cve = cur.execute(sql).fetchone() - i = 0 - while cve: - cve_name = cve[ORM.CVE_NAME] - cve_status = cve[ORM.CVE_STATUS] - publishedDate = cve[ORM.CVE_PUBLISHEDDATE].replace('-','') - lastModifiedDate = cve[ORM.CVE_LASTMODIFIEDDATE].replace('-','') - - newer = publishedDate > init_new_date - #print("%s = [%s] %s,%s" % (cve_name,cve_status,lastModifiedDate,newer)) - - if (ORM.STATUS_HISTORICAL == cve_status) and newer: - # Progress indicator support - i += 1 - if 0 == i % 10: - print('%04d: %20s %30s\r' % (i,cve_name,lastModifiedDate), end='') - if (0 == i % 200) and not cmd_skip: - conn.commit() - print('') - # Development/debug support - if cmd_skip: - if i < cmd_skip: - continue - else: - cmd_skip = 0 - if cmd_count: - if record_count < cmd_count: - record_count += 1 - else: - print("Count return: %s,%s,%s" % (i,record_count,cmd_count)) - break - - # Mark CVE as "New", or "New_Reserved" if no NIST source (e.g. only MITRE) - status = ORM.STATUS_NEW_RESERVED - sql = "SELECT * FROM orm_datasource WHERE data = ? AND source = ?" - cur_ds.execute(sql, ('cve','nist', )) - for ds in cur_ds: - if ds[ORM.DATASOURCE_CVE_FILTER] and cve[ORM.CVE_NAME].startswith(ds[ORM.DATASOURCE_CVE_FILTER]): - sql = ''' SELECT * FROM orm_cvesource WHERE cve_id = ? AND datasource_id = ?''' - if cur_write.execute(sql, (cve[ORM.CVE_ID],ds[ORM.DATASOURCE_ID],)).fetchone(): - print("STATUS_NEW: found NIST data source parent") - status = ORM.STATUS_NEW - break - else: - print("STATUS_NEW_RESERVED: no NIST data source parent") - sql = ''' UPDATE orm_cve - SET status = ? - WHERE id = ?''' - cur_write.execute(sql, (status, cve[ORM.CVE_ID])) - print("%s = HISTORICAL to NEW " % (cve_name)) - - # Next Record - cve = cur.fetchone() - conn.commit() - cur.close() - conn.close() - - -################################# # init_notify_categories # @@ -551,6 +476,17 @@ def gen_schema_header(): fd.write(" %s_%s = %d\n" % ('DEFECT','REPLACED_BY_REQUIREMENT' ,8)) fd.write(" %s_%s = %d\n" % ('DEFECT','CANNOT_REPRODUCE' ,9)) fd.write(" %s_%s = %d\n" % ('DEFECT','DONE' ,10)) + + fd.write(" %s_%s = %d\n" % ('PACKAGE','FOR' ,0)) + fd.write(" %s_%s = %d\n" % ('PACKAGE','AGAINST' ,1)) + + fd.write(" %s_%s = %d\n" % ('DATASOURCE','MINUTELY' ,0)) + fd.write(" %s_%s = %d\n" % ('DATASOURCE','HOURLY' ,1)) + fd.write(" %s_%s = %d\n" % ('DATASOURCE','DAILY' ,2)) + fd.write(" %s_%s = %d\n" % ('DATASOURCE','WEEKLY' ,3)) + fd.write(" %s_%s = %d\n" % ('DATASOURCE','MONTHLY' ,4)) + fd.write(" %s_%s = %d\n" % ('DATASOURCE','ONDEMAND' ,5)) + fd.write("\n") ################################# @@ -594,7 +530,6 @@ def main(argv): parser.add_argument('--init-notify-categories', '-n', action='store_const', const='init_notify_categories', dest='command', help='Initialize notify categories') parser.add_argument('--score-new-cves', '-s', dest='score_new_cves', help='Score CVEs for triage [NEW|CVE-1234]') parser.add_argument('--generate-schema-header', action='store_const', const='gen_schema_header', dest='command', help='Generate database schema header') - parser.add_argument('--preset-new', action='store_const', const='preset_new', dest='command', help='Preset historical to new CVEs') parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') parser.add_argument('--skip', dest='skip', help='Debugging: skip record count') parser.add_argument('--count', dest='count', help='Debugging: short run record count') @@ -619,8 +554,6 @@ def main(argv): score_new_cves(args.score_new_cves) elif 'gen_schema_header' == args.command: gen_schema_header() - elif 'preset_new' == args.command: - preset_new() elif 'fix_name_sort' == args.command: fix_name_sort() else: diff --git a/bin/mitre/datasource_2015.json b/bin/mitre/datasource_2015.json index d668438c..9d015a4d 100755 --- a/bin/mitre/datasource_2015.json +++ b/bin/mitre/datasource_2015.json @@ -7,7 +7,7 @@ "name" : "MITRE", "description" : "MITRE 2015", "cve_filter" : "CVE-2015", - "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2015' --file=data/allitems-cvrf-year-2015.xml --url-file=allitems-cvrf-year-2015.xml", + "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2015' --file=data/allitems-cvrf-year-2015.xml --url-file=allitems-cvrf-year-2015.xml", "update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2015' --file=data/allitems-cvrf-year-2015.xml --url-file=allitems-cvrf-year-2015.xml", "lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2015.xml %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/mitre/datasource_2016.json b/bin/mitre/datasource_2016.json index 212839fa..4daad16e 100755 --- a/bin/mitre/datasource_2016.json +++ b/bin/mitre/datasource_2016.json @@ -7,7 +7,7 @@ "name" : "MITRE", "description" : "MITRE 2016", "cve_filter" : "CVE-2016", - "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2016' --file=data/allitems-cvrf-year-2016.xml --url-file=allitems-cvrf-year-2016.xml", + "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2016' --file=data/allitems-cvrf-year-2016.xml --url-file=allitems-cvrf-year-2016.xml", "update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2016' --file=data/allitems-cvrf-year-2016.xml --url-file=allitems-cvrf-year-2016.xml", "lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2016.xml %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/mitre/datasource_2017.json b/bin/mitre/datasource_2017.json index 2be2f132..1b1e4b32 100755 --- a/bin/mitre/datasource_2017.json +++ b/bin/mitre/datasource_2017.json @@ -7,7 +7,7 @@ "name" : "MITRE", "description" : "MITRE 2017", "cve_filter" : "CVE-2017", - "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2017' --file=data/allitems-cvrf-year-2017.xml --url-file=allitems-cvrf-year-2017.xml", + "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2017' --file=data/allitems-cvrf-year-2017.xml --url-file=allitems-cvrf-year-2017.xml", "update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2017' --file=data/allitems-cvrf-year-2017.xml --url-file=allitems-cvrf-year-2017.xml", "lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2017.xml %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/mitre/datasource_2018.json b/bin/mitre/datasource_2018.json index fd3be9a8..cc9c560b 100755 --- a/bin/mitre/datasource_2018.json +++ b/bin/mitre/datasource_2018.json @@ -7,7 +7,7 @@ "name" : "MITRE", "description" : "MITRE 2018", "cve_filter" : "CVE-2018", - "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2018' --file=data/allitems-cvrf-year-2018.xml --url-file=allitems-cvrf-year-2018.xml", + "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2018' --file=data/allitems-cvrf-year-2018.xml --url-file=allitems-cvrf-year-2018.xml", "update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2018' --file=data/allitems-cvrf-year-2018.xml --url-file=allitems-cvrf-year-2018.xml", "lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2018.xml %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/mitre/datasource_2019.json b/bin/mitre/datasource_2019.json new file mode 100755 index 00000000..5f04ca16 --- /dev/null +++ b/bin/mitre/datasource_2019.json @@ -0,0 +1,18 @@ +{ + "datasource" : [ + { + "key" : "0020-mitre-2019", + "data" : "cve", + "source" : "mitre", + "name" : "MITRE", + "description" : "MITRE 2019", + "cve_filter" : "CVE-2019", + "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2019' --file=data/allitems-cvrf-year-2019.xml --url-file=allitems-cvrf-year-2019.xml", + "update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2019' --file=data/allitems-cvrf-year-2019.xml --url-file=allitems-cvrf-year-2019.xml", + "lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2019.xml %command%", + "lastModifiedDate" : "2018-03-01 01:01:01", + "update_frequency" : "3", + "update_time" : "02:00:00" + } + ] +} diff --git a/bin/mitre/srtool_mitre.py b/bin/mitre/srtool_mitre.py index 28bd52ae..0464156a 100755 --- a/bin/mitre/srtool_mitre.py +++ b/bin/mitre/srtool_mitre.py @@ -32,8 +32,9 @@ import re import xml.etree.ElementTree as ET import argparse import shutil -from datetime import datetime, date import sqlite3 +from datetime import datetime, date, timedelta +import pytz from urllib.request import urlopen, URLError, Request from urllib.parse import urlparse @@ -89,6 +90,39 @@ def srt_error_log(msg): f1.write("|" + msg + "|\n" ) f1.close() + +# Newly discovered or updated CVEs default to NEW for triage +# Inited CVEs default to HISTORICAL, unless they are within the courtesy CVE_INIT_NEW_DELTA +init_new_date = None +def get_cve_default_status(is_init,publishedDate): + global init_new_date + + if None == init_new_date: + # Precalculate and cache the relative 'new' date for efficiency + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + sql = '''SELECT * FROM orm_srtsetting WHERE name=?''' + CVE_INIT_NEW_DELTA = cur.execute(sql, ('CVE_INIT_NEW_DELTA',)).fetchone() + if CVE_INIT_NEW_DELTA is None: + cve_init_new_delta = 30 + else: + cve_init_new_delta = int(CVE_INIT_NEW_DELTA[ORM.SRTSETTING_VALUE]) + + date_delta = timedelta(days=cve_init_new_delta) + init_new_date = datetime.now(pytz.utc) - date_delta + #print("\nPreset new data = %s" % init_new_date.strftime("%Y%m%d")) + init_new_date = init_new_date.strftime("%Y-%m-%d") + + if is_init: + # Note: the NIST 'published date' is in the format "2017-05-11", so do a simple string compare + #print("INIT status: %s versus %s" % (init_new_date,publishedDate)) + if not publishedDate or (publishedDate > init_new_date): + return ORM.STATUS_NEW + else: + return ORM.STATUS_HISTORICAL + else: + return ORM.STATUS_NEW + ################################# # Fetch a CVRF record from Mitre # REST API, cache the results @@ -176,7 +210,6 @@ def fetch_cve(cve_name,cvrf_xml_file): with open(cache_file, 'r') as fp: for line in fp: line = line.strip() - #print("FOO:'%s'" % line) if line.startswith('Description'): results['description'] = '%s[EOL]%s' % (line.replace('Description=','').replace('\\n','').replace('\\r',' '),results['description']) elif line.startswith('Published'): results['publishedDate'] = line.replace('Published=','') elif line.startswith('Modified'): results['lastModifiedDate'] = line.replace('Modified=','') @@ -226,8 +259,10 @@ def init_mitre_file(source,url_file,datasource_file,force_update): ################################# # append_cve_database # +# Create new CVE record if not already added by NIST scan +# so that non-public CVEs can be tracked -def append_cve_database(file_xml): +def append_cve_database(is_init,file_xml): tree = ET.parse(file_xml) root = tree.getroot() @@ -275,9 +310,12 @@ def append_cve_database(file_xml): cve_id = cve[ORM.CVE_ID] print("MITRE:FOUND %20s\r" % cve_name, end='') else: + # Get the default CVE status + status = get_cve_default_status(is_init,summary['Published']) + sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, srt_updated, packages) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' - cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, ORM.STATUS_HISTORICAL, '', '', '', '', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],'', '', '', '', '', '', datetime.now(),'')) + cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, status, '', '', '', '', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],'', '', '', '', '', '', datetime.now(),'')) cve_id = cur.lastrowid print("MITRE:ADDED %20s" % cve_name) @@ -363,7 +401,7 @@ def main(argv): # setup parser = argparse.ArgumentParser(description='srtool_mitre.py: manage Mitre CVE data') - parser.add_argument('--initialize', '-i', action='store_const', const='init_mitre', dest='command', help='Download the Mitre source CVE file') + parser.add_argument('--initialize', '-I', action='store_const', const='init_mitre', dest='command', help='Download the Mitre source CVE file') parser.add_argument('--update', '-u', action='store_const', const='update_mitre', dest='command', help='Update the Mitre source CVE file') parser.add_argument('--source', dest='source', help='Local CVE source file') parser.add_argument('--url-file', dest='url_file', help='CVE URL extension') @@ -409,10 +447,10 @@ def main(argv): if 'init_mitre' == args.command: init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update) - append_cve_database(args.cve_file) + append_cve_database(True,args.cve_file) elif 'update_mitre' == args.command: - # No difference from init at this time init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update) + append_cve_database(False,args.cve_file) else: print("Command not found") diff --git a/bin/nist/datasource_2015.json b/bin/nist/datasource_2015.json index 51c21cb3..e26b0f75 100755 --- a/bin/nist/datasource_2015.json +++ b/bin/nist/datasource_2015.json @@ -7,7 +7,7 @@ "name" : "NIST", "description" : "NIST 2015", "cve_filter" : "CVE-2015", - "init" : "bin/nist/srtool_nist.py -n --source='NIST 2015' --file=data/nvdcve-1.0-2015.json --url-file=nvdcve-1.0-2015.json.gz --url-meta=nvdcve-1.0-2015.meta", + "init" : "bin/nist/srtool_nist.py -I --source='NIST 2015' --file=data/nvdcve-1.0-2015.json --url-file=nvdcve-1.0-2015.json.gz --url-meta=nvdcve-1.0-2015.meta", "update" : "bin/nist/srtool_nist.py -n --source='NIST 2015' --file=data/nvdcve-1.0-2015.json --url-file=nvdcve-1.0-2015.json.gz --url-meta=nvdcve-1.0-2015.meta", "lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2015.json %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/nist/datasource_2016.json b/bin/nist/datasource_2016.json index 82671a14..026060b8 100755 --- a/bin/nist/datasource_2016.json +++ b/bin/nist/datasource_2016.json @@ -7,7 +7,7 @@ "name" : "NIST", "description" : "NIST 2016", "cve_filter" : "CVE-2016", - "init" : "bin/nist/srtool_nist.py -n --source='NIST 2016' --file=data/nvdcve-1.0-2016.json --url-file=nvdcve-1.0-2016.json.gz --url-meta=nvdcve-1.0-2016.meta", + "init" : "bin/nist/srtool_nist.py -I --source='NIST 2016' --file=data/nvdcve-1.0-2016.json --url-file=nvdcve-1.0-2016.json.gz --url-meta=nvdcve-1.0-2016.meta", "update" : "bin/nist/srtool_nist.py -n --source='NIST 2016' --file=data/nvdcve-1.0-2016.json --url-file=nvdcve-1.0-2016.json.gz --url-meta=nvdcve-1.0-2016.meta", "lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2016.json %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/nist/datasource_2017.json b/bin/nist/datasource_2017.json index 9fec02dd..64bd0171 100755 --- a/bin/nist/datasource_2017.json +++ b/bin/nist/datasource_2017.json @@ -7,7 +7,7 @@ "name" : "NIST", "description" : "NIST 2017", "cve_filter" : "CVE-2017", - "init" : "bin/nist/srtool_nist.py -n --source='NIST 2017' --file=data/nvdcve-1.0-2017.json --url-file=nvdcve-1.0-2017.json.gz --url-meta=nvdcve-1.0-2017.meta", + "init" : "bin/nist/srtool_nist.py -I --source='NIST 2017' --file=data/nvdcve-1.0-2017.json --url-file=nvdcve-1.0-2017.json.gz --url-meta=nvdcve-1.0-2017.meta", "update" : "bin/nist/srtool_nist.py -n --source='NIST 2017' --file=data/nvdcve-1.0-2017.json --url-file=nvdcve-1.0-2017.json.gz --url-meta=nvdcve-1.0-2017.meta", "lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2017.json %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/nist/datasource_2018.json b/bin/nist/datasource_2018.json index 30860e85..216ef5a4 100755 --- a/bin/nist/datasource_2018.json +++ b/bin/nist/datasource_2018.json @@ -7,7 +7,7 @@ "name" : "NIST", "description" : "NIST 2018", "cve_filter" : "CVE-2018", - "init" : "bin/nist/srtool_nist.py -n --source='NIST 2018' --file=data/nvdcve-1.0-2018.json --url-file=nvdcve-1.0-2018.json.gz --url-meta=nvdcve-1.0-2018.meta", + "init" : "bin/nist/srtool_nist.py -I --source='NIST 2018' --file=data/nvdcve-1.0-2018.json --url-file=nvdcve-1.0-2018.json.gz --url-meta=nvdcve-1.0-2018.meta", "update" : "bin/nist/srtool_nist.py -n --source='NIST 2018' --file=data/nvdcve-1.0-2018.json --url-file=nvdcve-1.0-2018.json.gz --url-meta=nvdcve-1.0-2018.meta", "lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2018.json %command%", "lastModifiedDate" : "2018-03-01 01:01:01", diff --git a/bin/nist/datasource_2019.json b/bin/nist/datasource_2019.json new file mode 100755 index 00000000..ce9fc12d --- /dev/null +++ b/bin/nist/datasource_2019.json @@ -0,0 +1,18 @@ +{ + "datasource" : [ + { + "key" : "0010-nist-2019", + "data" : "cve", + "source" : "nist", + "name" : "NIST", + "description" : "NIST 2019", + "cve_filter" : "CVE-2019", + "init" : "bin/nist/srtool_nist.py -I --source='NIST 2019' --file=data/nvdcve-1.0-2019.json --url-file=nvdcve-1.0-2019.json.gz --url-meta=nvdcve-1.0-2019.meta", + "update" : "bin/nist/srtool_nist.py -n --source='NIST 2019' --file=data/nvdcve-1.0-2019.json --url-file=nvdcve-1.0-2019.json.gz --url-meta=nvdcve-1.0-2019.meta", + "lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2019.json %command%", + "lastModifiedDate" : "2018-03-01 01:01:01", + "update_frequency" : "3", + "update_time" : "02:00:00" + } + ] +} diff --git a/bin/nist/srtool_nist.py b/bin/nist/srtool_nist.py index 0952b0cd..c05e65d0 100755 --- a/bin/nist/srtool_nist.py +++ b/bin/nist/srtool_nist.py @@ -34,16 +34,17 @@ import sqlite3 import subprocess import json import urllib +from datetime import datetime, date, timedelta +import pytz + +from urllib.request import urlopen, URLError +from urllib.parse import urlparse # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -from datetime import datetime, date -from urllib.request import urlopen, URLError -from urllib.parse import urlparse - # setup lookupTable = [] cveIndex = {} @@ -91,6 +92,39 @@ def get_name_sort(cve_name): cve_name_sort = cve_name return cve_name_sort +# Newly discovered or updated CVEs default to NEW for triage +# Inited CVEs default to HISTORICAL, unless they are within the courtesy CVE_INIT_NEW_DELTA +init_new_date = None +def get_cve_default_status(is_init,publishedDate): + global init_new_date + + if None == init_new_date: + # Precalculate and cache the relative 'new' date for efficiency + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + sql = '''SELECT * FROM orm_srtsetting WHERE name=?''' + CVE_INIT_NEW_DELTA = cur.execute(sql, ('CVE_INIT_NEW_DELTA',)).fetchone() + if CVE_INIT_NEW_DELTA is None: + cve_init_new_delta = 30 + else: + cve_init_new_delta = int(CVE_INIT_NEW_DELTA[ORM.SRTSETTING_VALUE]) + + date_delta = timedelta(days=cve_init_new_delta) + init_new_date = datetime.now(pytz.utc) - date_delta + #print("\nPreset new data = %s" % init_new_date.strftime("%Y%m%d")) + init_new_date = init_new_date.strftime("%Y-%m-%d") + + if is_init: + # Note: the NIST 'published date' is in the format "2017-05-11", so do a simple string compare + #print("INIT status: %s > %s" % (publishedDate, init_new_date)) + if not publishedDate or (publishedDate > init_new_date): + return ORM.STATUS_NEW + else: + return ORM.STATUS_HISTORICAL + else: + return ORM.STATUS_NEW + + ################################# # check for updates and apply if any # @@ -100,7 +134,7 @@ def get_name_sort(cve_name): #gets CVE-Modified feed, determines if we are out of date, and applies updates if true #tracks history in update_log.txt #incremental argument is boolean that idicates if bulk updating or incremental updating. -def update_nist(datasource_description, url_file, url_meta, cve_file, incremental, force_update): +def update_nist(is_init,datasource_description, url_file, url_meta, cve_file, incremental, force_update): nist_cve_url = '%s/%s' % (nist_cve_url_base,url_file) nist_meta_url = '%s/%s' % (nist_meta_url_base,url_meta) @@ -138,14 +172,14 @@ def update_nist(datasource_description, url_file, url_meta, cve_file, incrementa date_new = datetime.strptime(content, 'lastModifiedDate:%Y-%m-%dT%H:%M:%S') date_past = datetime.strptime(ds[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S') - log.write("BEGINNING NIST UPDATES\n") + log.write("BEGINNING NIST %s\n" % ('INITS' if is_init else 'UPDATES')) #determine if we are out of date and apply updates if true if (date_new > date_past) or force_update: pre_update_time = datetime.now() #used for logging purposes only - nist_json(nist_cve_url, ds[ORM.DATASOURCE_ID], nist_file, log, date_new, incremental) - log.write("began updates: %s\n" % str(pre_update_time)) - log.write("finished updates: %s\n" % str(datetime.now()) ) + nist_json(is_init,nist_cve_url, ds[ORM.DATASOURCE_ID], nist_file, log, date_new, incremental) + log.write("began %s: %s\n" % ( 'init' if is_init else 'updates', str(pre_update_time) )) + log.write("finished %s: %s\n" % ( 'init' if is_init else 'updates', str(datetime.now()) )) log.write("=============================================================================\n") log.write("\n") @@ -154,12 +188,11 @@ def update_nist(datasource_description, url_file, url_meta, cve_file, incrementa c.execute(sql, (str(date_new),)) conn.commit() else: - - log.write("No update needed\n") + log.write("No %s needed\n" % ('init' if is_init else 'update')) log.write("Checked: %s\n" % datetime.now()) log.write("=============================================================================\n") log.write("\n") - print("NO UPDATE NEEDED") + print("NO %s NEEDED" % ('INIT' if is_init else 'UPDATE')) # Reset datasource's update_time as today sql = "UPDATE orm_datasource SET update_time = ? WHERE id='%s'" % ds[ORM.DATASOURCE_ID] @@ -196,7 +229,7 @@ def file_date(filename,utc=False): #parses JSON, creates CVE object, and updates database as necessary. Commits to database on success #will EITHER create new record in orm_cve if cve does not exist OR overwrite every field if existing cve out-of-date OR ignore cve #requires json to be formatted with NIST Json schema (https://csrc.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema) -def nist_json(summary_json_url, datasource_id, datasource_file, log, date_new, incremental): +def nist_json(is_init,summary_json_url, datasource_id, datasource_file, log, date_new, incremental): import traceback import gzip @@ -266,7 +299,7 @@ def nist_json(summary_json_url, datasource_id, datasource_file, log, date_new, i #check if cve object `v` need to be uploaded to database (cases: new cve, modified cve, or no changes) #if true, apply changes. Else ignore and continue - v_id, is_change = sql_cve_query(conn, v, log) + v_id, is_change = sql_cve_query(conn, v, is_init,log) #if incremental update and CVE changed, save json copy of the cve to cache @@ -366,21 +399,25 @@ class Cve(): #new CVE -> INSERT || modified CVE -> UPDATE || no change -> ignore and return #returns (CVE_ID, BOOL) tuple, True if insert or update executed ### THIS DOES NOT CALL CONNECTION.COMMIT() -def sql_cve_query(conn, cve, log): - CVE_LASTMODIFIEDDATE = 14 +def sql_cve_query(conn, cve, is_init, log): is_change = False cur = conn.cursor() sql = '''SELECT * FROM orm_cve WHERE name=?''' exists = cur.execute(sql, (cve.name,)).fetchone() cve_id = -1 if exists is None: + # Get the default CVE status + status = get_cve_default_status(is_init,cve.publishedDate) + print("BAR:%s=%s" % (cve.name,status)) + sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, srt_updated, packages) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' - cur.execute(sql, (cve.name, get_name_sort(cve.name), cve.priority, cve.status, cve.comments, cve.comments_private, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, 1, cve.publish_state, cve.publish_date, cve.description, cve.publishedDate, cve.lastModifiedDate, cve.recommend, cve.recommend_list, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, datetime.now(),'')) + cur.execute(sql, (cve.name, get_name_sort(cve.name), cve.priority, status, cve.comments, cve.comments_private, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, 1, cve.publish_state, cve.publish_date, cve.description, cve.publishedDate, cve.lastModifiedDate, cve.recommend, cve.recommend_list, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, datetime.now(),'')) is_change = True cve_id = cur.lastrowid log.write("\tINSERTED '%s'\n" % cve.name) - elif exists[CVE_LASTMODIFIEDDATE] < cve.lastModifiedDate: + + elif exists[ORM.CVE_LASTMODIFIEDDATE] < cve.lastModifiedDate: sql = ''' UPDATE orm_cve SET recommend = ?, recommend_list = ?, @@ -397,7 +434,12 @@ def sql_cve_query(conn, cve, log): cur.execute(sql, (cve.recommend, cve.recommend_list, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, cve.description, cve.lastModifiedDate, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, exists[0])) is_change = True log.write("\tUPDATED '%s'\n" % cve.name) - cve_id = exists[0] + cve_id = exists[ORM.CVE_ID] + + ### TO-DO + ### CREATE NOTIFICATION IF SCORE/SEVERITY HAS CHANGED + ### + else: is_change = False log.write("\tSKIPPED '%s'\n" % cve.name) @@ -610,6 +652,7 @@ def nist_scan_configuration_or(cpe_or_node, name, and_enum): def main(argv): global verbose parser = argparse.ArgumentParser(description='srtool_cve.py: manage the CVEs within SRTool database') + parser.add_argument('--init_nist', '-I', action='store_const', const='init_nist', dest='command', help='Initialize nvd.nist.gov/vuln/data-feeds for a specified datasource') parser.add_argument('--update_nist', '-n', action='store_const', const='update_nist', dest='command', help='Check nvd.nist.gov/vuln/data-feeds for updates on a specified datasource') parser.add_argument('--source', dest='source', help='Local CVE source file') parser.add_argument('--url-file', dest='url_file', help='CVE URL extension') @@ -653,20 +696,21 @@ def main(argv): exit(1) ret = 0 - if 'update_nist' == args.command: + if ('init_nist' == args.command) or ('update_nist' == args.command): + is_init = ('init_nist' == args.command) try: - print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time") - update_nist(args.source, args.url_file, args.url_meta, args.cve_file, False, args.force_update) - master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tUPDATED\n" % (date.today(), args.source)) - print("DATABASE UPDATE FINISHED\n") + print ("BEGINNING NIST %s PLEASE WAIT ... this can take some time" % ('INIT' if is_init else 'UPDATES')) + update_nist(is_init, args.source, args.url_file, args.url_meta, args.cve_file, False, args.force_update) + master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\t%s\n" % (date.today(), args.source, "INIT'ED" if is_init else 'UPDATED')) + print("DATABASE %s FINISHED\n" % ('INIT' if is_init else 'UPDATE')) except Exception as e: - print("DATABASE UPDATED FAILED ... %s" % e) + print("DATABASE %s FAILED ... %s" % ('INIT' if is_init else 'UPDATE',e)) master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), args.source, e)) ret = 1 elif 'update_nist_incremental' == args.command: try: print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time") - update_nist(args.source, args.url_file, args.url_meta, args.cve_file, True, args.force_update) + update_nist(False,args.source, args.url_file, args.url_meta, args.cve_file, True, args.force_update) master_log.write("SRTOOL:%s:'NIST JSON Modified Data':\t\t\t...\t\t\tUPDATED\n" % date.today()) print("DATABASE UPDATE FINISHED\n") except Exception as e: |