aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Reyna <David.Reyna@windriver.com>2019-01-12 10:48:17 -0800
committerDavid Reyna <David.Reyna@windriver.com>2019-01-12 10:48:17 -0800
commitce03c55b2425fcb0efba606c539c071e9efcc2f8 (patch)
tree317b77e3d7f7b97b27358d81413bc1f68340aad6
parent98b5bc41ed75b57de4376757a575458e72bf44c3 (diff)
downloadsrtool-ce03c55b2425fcb0efba606c539c071e9efcc2f8.zip
srtool-ce03c55b2425fcb0efba606c539c071e9efcc2f8.tar.gz
srtool-ce03c55b2425fcb0efba606c539c071e9efcc2f8.tar.bz2
srtool: improve CVE status assignments, add CVE 2019
1. Add the CVE 2019 data soures for MITRE and NIST. 2. Improve the CVE default status assignment system: * During the "Init" phase all CVEs default to HISTORICAL, unless they are within the CVE_INIT_NEW_DELTA date range. The value CVE_INIT_NEW_DELTA is defined in "bin/common/datasource.json", and is an out-of-box courtesy to provide some CVEs for triage in newly initialized systems. Changing the default value to '0' disabled this. * During the "Update" phase, CVEs default to NEW (and thus primed for triage) * Better separate the Init versus Update functions in "srtool_mitre.py" and "srtool_nist.py", and their respective datasource files. * Remove the post-process "preset_new()" in "srtool_common.py" in favor of directly computing the values in get_cve_default_status() in "srtool_mitre.py" and "srtool_nist.py", for speed and consistency. [YOCTO #13134] [YOCTO #13135] Signed-off-by: David Reyna <David.Reyna@windriver.com>
-rwxr-xr-xbin/common/datasource.json20
-rwxr-xr-xbin/common/srtool_common.py89
-rwxr-xr-xbin/mitre/datasource_2015.json2
-rwxr-xr-xbin/mitre/datasource_2016.json2
-rwxr-xr-xbin/mitre/datasource_2017.json2
-rwxr-xr-xbin/mitre/datasource_2018.json2
-rwxr-xr-xbin/mitre/datasource_2019.json18
-rwxr-xr-xbin/mitre/srtool_mitre.py52
-rwxr-xr-xbin/nist/datasource_2015.json2
-rwxr-xr-xbin/nist/datasource_2016.json2
-rwxr-xr-xbin/nist/datasource_2017.json2
-rwxr-xr-xbin/nist/datasource_2018.json2
-rwxr-xr-xbin/nist/datasource_2019.json18
-rwxr-xr-xbin/nist/srtool_nist.py96
14 files changed, 176 insertions, 133 deletions
diff --git a/bin/common/datasource.json b/bin/common/datasource.json
index 482cc3c..2625abc 100755
--- a/bin/common/datasource.json
+++ b/bin/common/datasource.json
@@ -4,6 +4,12 @@
"name" : "SRTOOL_LOGO",
"helptext" : "The SRTool logo",
"value" : "Yocto Project,/static/img/logo.png,/"
+ },
+ {
+ "_comment_": "This is an out-of-box courtesy to provide some CVEs for triage in newly initialized systems",
+ "name" : "CVE_INIT_NEW_DELTA",
+ "helptext" : "CVEs found during the initialization phase become 'NEW' instead of 'HISTORICAL' if created within this number of days before today",
+ "value" : "30"
}
],
@@ -95,20 +101,6 @@
},
{
- "key" : "0920-preset-new",
- "data" : "preset_new",
- "source" : "common",
- "name" : "Preset New",
- "description" : "Preset New CVEs",
- "cve_filter" : "",
- "init" : "bin/common/srtool_common.py --preset-new",
- "update" : "",
- "lookup" : "",
- "lastModifiedDate" : "2018-03-01 01:01:01",
- "update_frequency" : "0",
- "update_time" : "02:00:00"
- },
- {
"key" : "0921-common-score",
"data" : "score_cves",
"source" : "common",
diff --git a/bin/common/srtool_common.py b/bin/common/srtool_common.py
index 6518aee..62cc95d 100755
--- a/bin/common/srtool_common.py
+++ b/bin/common/srtool_common.py
@@ -373,81 +373,6 @@ def score_new_cves(cve_filter):
print("\nUpdated CVEs=%d, Added alternate sources=%d" % (write_count,ds_count))
#################################
-# preset_new
-#
-
-def preset_new():
- global cmd_skip
-
- conn = sqlite3.connect(srtDbName)
- cur = conn.cursor()
- cur_write = conn.cursor()
- cur_ds = conn.cursor()
- date_delta = timedelta(days=30)
- init_new_date = datetime.now(pytz.utc) - date_delta
- print("\nPreset new data = %s" % init_new_date.strftime("%Y%m%d"))
- init_new_date = init_new_date.strftime("%Y%m%d")
-
- sql = '''SELECT * FROM orm_cve;'''
- cve = cur.execute(sql).fetchone()
- i = 0
- while cve:
- cve_name = cve[ORM.CVE_NAME]
- cve_status = cve[ORM.CVE_STATUS]
- publishedDate = cve[ORM.CVE_PUBLISHEDDATE].replace('-','')
- lastModifiedDate = cve[ORM.CVE_LASTMODIFIEDDATE].replace('-','')
-
- newer = publishedDate > init_new_date
- #print("%s = [%s] %s,%s" % (cve_name,cve_status,lastModifiedDate,newer))
-
- if (ORM.STATUS_HISTORICAL == cve_status) and newer:
- # Progress indicator support
- i += 1
- if 0 == i % 10:
- print('%04d: %20s %30s\r' % (i,cve_name,lastModifiedDate), end='')
- if (0 == i % 200) and not cmd_skip:
- conn.commit()
- print('')
- # Development/debug support
- if cmd_skip:
- if i < cmd_skip:
- continue
- else:
- cmd_skip = 0
- if cmd_count:
- if record_count < cmd_count:
- record_count += 1
- else:
- print("Count return: %s,%s,%s" % (i,record_count,cmd_count))
- break
-
- # Mark CVE as "New", or "New_Reserved" if no NIST source (e.g. only MITRE)
- status = ORM.STATUS_NEW_RESERVED
- sql = "SELECT * FROM orm_datasource WHERE data = ? AND source = ?"
- cur_ds.execute(sql, ('cve','nist', ))
- for ds in cur_ds:
- if ds[ORM.DATASOURCE_CVE_FILTER] and cve[ORM.CVE_NAME].startswith(ds[ORM.DATASOURCE_CVE_FILTER]):
- sql = ''' SELECT * FROM orm_cvesource WHERE cve_id = ? AND datasource_id = ?'''
- if cur_write.execute(sql, (cve[ORM.CVE_ID],ds[ORM.DATASOURCE_ID],)).fetchone():
- print("STATUS_NEW: found NIST data source parent")
- status = ORM.STATUS_NEW
- break
- else:
- print("STATUS_NEW_RESERVED: no NIST data source parent")
- sql = ''' UPDATE orm_cve
- SET status = ?
- WHERE id = ?'''
- cur_write.execute(sql, (status, cve[ORM.CVE_ID]))
- print("%s = HISTORICAL to NEW " % (cve_name))
-
- # Next Record
- cve = cur.fetchone()
- conn.commit()
- cur.close()
- conn.close()
-
-
-#################################
# init_notify_categories
#
@@ -551,6 +476,17 @@ def gen_schema_header():
fd.write(" %s_%s = %d\n" % ('DEFECT','REPLACED_BY_REQUIREMENT' ,8))
fd.write(" %s_%s = %d\n" % ('DEFECT','CANNOT_REPRODUCE' ,9))
fd.write(" %s_%s = %d\n" % ('DEFECT','DONE' ,10))
+
+ fd.write(" %s_%s = %d\n" % ('PACKAGE','FOR' ,0))
+ fd.write(" %s_%s = %d\n" % ('PACKAGE','AGAINST' ,1))
+
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','MINUTELY' ,0))
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','HOURLY' ,1))
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','DAILY' ,2))
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','WEEKLY' ,3))
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','MONTHLY' ,4))
+ fd.write(" %s_%s = %d\n" % ('DATASOURCE','ONDEMAND' ,5))
+
fd.write("\n")
#################################
@@ -594,7 +530,6 @@ def main(argv):
parser.add_argument('--init-notify-categories', '-n', action='store_const', const='init_notify_categories', dest='command', help='Initialize notify categories')
parser.add_argument('--score-new-cves', '-s', dest='score_new_cves', help='Score CVEs for triage [NEW|CVE-1234]')
parser.add_argument('--generate-schema-header', action='store_const', const='gen_schema_header', dest='command', help='Generate database schema header')
- parser.add_argument('--preset-new', action='store_const', const='preset_new', dest='command', help='Preset historical to new CVEs')
parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
parser.add_argument('--skip', dest='skip', help='Debugging: skip record count')
parser.add_argument('--count', dest='count', help='Debugging: short run record count')
@@ -619,8 +554,6 @@ def main(argv):
score_new_cves(args.score_new_cves)
elif 'gen_schema_header' == args.command:
gen_schema_header()
- elif 'preset_new' == args.command:
- preset_new()
elif 'fix_name_sort' == args.command:
fix_name_sort()
else:
diff --git a/bin/mitre/datasource_2015.json b/bin/mitre/datasource_2015.json
index d668438..9d015a4 100755
--- a/bin/mitre/datasource_2015.json
+++ b/bin/mitre/datasource_2015.json
@@ -7,7 +7,7 @@
"name" : "MITRE",
"description" : "MITRE 2015",
"cve_filter" : "CVE-2015",
- "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2015' --file=data/allitems-cvrf-year-2015.xml --url-file=allitems-cvrf-year-2015.xml",
+ "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2015' --file=data/allitems-cvrf-year-2015.xml --url-file=allitems-cvrf-year-2015.xml",
"update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2015' --file=data/allitems-cvrf-year-2015.xml --url-file=allitems-cvrf-year-2015.xml",
"lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2015.xml %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/mitre/datasource_2016.json b/bin/mitre/datasource_2016.json
index 212839f..4daad16 100755
--- a/bin/mitre/datasource_2016.json
+++ b/bin/mitre/datasource_2016.json
@@ -7,7 +7,7 @@
"name" : "MITRE",
"description" : "MITRE 2016",
"cve_filter" : "CVE-2016",
- "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2016' --file=data/allitems-cvrf-year-2016.xml --url-file=allitems-cvrf-year-2016.xml",
+ "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2016' --file=data/allitems-cvrf-year-2016.xml --url-file=allitems-cvrf-year-2016.xml",
"update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2016' --file=data/allitems-cvrf-year-2016.xml --url-file=allitems-cvrf-year-2016.xml",
"lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2016.xml %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/mitre/datasource_2017.json b/bin/mitre/datasource_2017.json
index 2be2f13..1b1e4b3 100755
--- a/bin/mitre/datasource_2017.json
+++ b/bin/mitre/datasource_2017.json
@@ -7,7 +7,7 @@
"name" : "MITRE",
"description" : "MITRE 2017",
"cve_filter" : "CVE-2017",
- "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2017' --file=data/allitems-cvrf-year-2017.xml --url-file=allitems-cvrf-year-2017.xml",
+ "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2017' --file=data/allitems-cvrf-year-2017.xml --url-file=allitems-cvrf-year-2017.xml",
"update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2017' --file=data/allitems-cvrf-year-2017.xml --url-file=allitems-cvrf-year-2017.xml",
"lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2017.xml %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/mitre/datasource_2018.json b/bin/mitre/datasource_2018.json
index fd3be9a..cc9c560 100755
--- a/bin/mitre/datasource_2018.json
+++ b/bin/mitre/datasource_2018.json
@@ -7,7 +7,7 @@
"name" : "MITRE",
"description" : "MITRE 2018",
"cve_filter" : "CVE-2018",
- "init" : "bin/mitre/srtool_mitre.py -i --source='Mitre 2018' --file=data/allitems-cvrf-year-2018.xml --url-file=allitems-cvrf-year-2018.xml",
+ "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2018' --file=data/allitems-cvrf-year-2018.xml --url-file=allitems-cvrf-year-2018.xml",
"update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2018' --file=data/allitems-cvrf-year-2018.xml --url-file=allitems-cvrf-year-2018.xml",
"lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2018.xml %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/mitre/datasource_2019.json b/bin/mitre/datasource_2019.json
new file mode 100755
index 0000000..5f04ca1
--- /dev/null
+++ b/bin/mitre/datasource_2019.json
@@ -0,0 +1,18 @@
+{
+ "datasource" : [
+ {
+ "key" : "0020-mitre-2019",
+ "data" : "cve",
+ "source" : "mitre",
+ "name" : "MITRE",
+ "description" : "MITRE 2019",
+ "cve_filter" : "CVE-2019",
+ "init" : "bin/mitre/srtool_mitre.py -I --source='Mitre 2019' --file=data/allitems-cvrf-year-2019.xml --url-file=allitems-cvrf-year-2019.xml",
+ "update" : "bin/mitre/srtool_mitre.py -u --source='Mitre 2019' --file=data/allitems-cvrf-year-2019.xml --url-file=allitems-cvrf-year-2019.xml",
+ "lookup" : "bin/mitre/srtool_mitre.py --file=data/allitems-cvrf-year-2019.xml %command%",
+ "lastModifiedDate" : "2018-03-01 01:01:01",
+ "update_frequency" : "3",
+ "update_time" : "02:00:00"
+ }
+ ]
+}
diff --git a/bin/mitre/srtool_mitre.py b/bin/mitre/srtool_mitre.py
index 28bd52a..0464156 100755
--- a/bin/mitre/srtool_mitre.py
+++ b/bin/mitre/srtool_mitre.py
@@ -32,8 +32,9 @@ import re
import xml.etree.ElementTree as ET
import argparse
import shutil
-from datetime import datetime, date
import sqlite3
+from datetime import datetime, date, timedelta
+import pytz
from urllib.request import urlopen, URLError, Request
from urllib.parse import urlparse
@@ -89,6 +90,39 @@ def srt_error_log(msg):
f1.write("|" + msg + "|\n" )
f1.close()
+
+# Newly discovered or updated CVEs default to NEW for triage
+# Inited CVEs default to HISTORICAL, unless they are within the courtesy CVE_INIT_NEW_DELTA
+init_new_date = None
+def get_cve_default_status(is_init,publishedDate):
+ global init_new_date
+
+ if None == init_new_date:
+ # Precalculate and cache the relative 'new' date for efficiency
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+ sql = '''SELECT * FROM orm_srtsetting WHERE name=?'''
+ CVE_INIT_NEW_DELTA = cur.execute(sql, ('CVE_INIT_NEW_DELTA',)).fetchone()
+ if CVE_INIT_NEW_DELTA is None:
+ cve_init_new_delta = 30
+ else:
+ cve_init_new_delta = int(CVE_INIT_NEW_DELTA[ORM.SRTSETTING_VALUE])
+
+ date_delta = timedelta(days=cve_init_new_delta)
+ init_new_date = datetime.now(pytz.utc) - date_delta
+ #print("\nPreset new data = %s" % init_new_date.strftime("%Y%m%d"))
+ init_new_date = init_new_date.strftime("%Y-%m-%d")
+
+ if is_init:
+ # Note: the NIST 'published date' is in the format "2017-05-11", so do a simple string compare
+ #print("INIT status: %s versus %s" % (init_new_date,publishedDate))
+ if not publishedDate or (publishedDate > init_new_date):
+ return ORM.STATUS_NEW
+ else:
+ return ORM.STATUS_HISTORICAL
+ else:
+ return ORM.STATUS_NEW
+
#################################
# Fetch a CVRF record from Mitre
# REST API, cache the results
@@ -176,7 +210,6 @@ def fetch_cve(cve_name,cvrf_xml_file):
with open(cache_file, 'r') as fp:
for line in fp:
line = line.strip()
- #print("FOO:'%s'" % line)
if line.startswith('Description'): results['description'] = '%s[EOL]%s' % (line.replace('Description=','').replace('\\n','').replace('\\r',' '),results['description'])
elif line.startswith('Published'): results['publishedDate'] = line.replace('Published=','')
elif line.startswith('Modified'): results['lastModifiedDate'] = line.replace('Modified=','')
@@ -226,8 +259,10 @@ def init_mitre_file(source,url_file,datasource_file,force_update):
#################################
# append_cve_database
#
+# Create new CVE record if not already added by NIST scan
+# so that non-public CVEs can be tracked
-def append_cve_database(file_xml):
+def append_cve_database(is_init,file_xml):
tree = ET.parse(file_xml)
root = tree.getroot()
@@ -275,9 +310,12 @@ def append_cve_database(file_xml):
cve_id = cve[ORM.CVE_ID]
print("MITRE:FOUND %20s\r" % cve_name, end='')
else:
+ # Get the default CVE status
+ status = get_cve_default_status(is_init,summary['Published'])
+
sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, srt_updated, packages)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'''
- cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, ORM.STATUS_HISTORICAL, '', '', '', '', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],'', '', '', '', '', '', datetime.now(),''))
+ cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, status, '', '', '', '', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],'', '', '', '', '', '', datetime.now(),''))
cve_id = cur.lastrowid
print("MITRE:ADDED %20s" % cve_name)
@@ -363,7 +401,7 @@ def main(argv):
# setup
parser = argparse.ArgumentParser(description='srtool_mitre.py: manage Mitre CVE data')
- parser.add_argument('--initialize', '-i', action='store_const', const='init_mitre', dest='command', help='Download the Mitre source CVE file')
+ parser.add_argument('--initialize', '-I', action='store_const', const='init_mitre', dest='command', help='Download the Mitre source CVE file')
parser.add_argument('--update', '-u', action='store_const', const='update_mitre', dest='command', help='Update the Mitre source CVE file')
parser.add_argument('--source', dest='source', help='Local CVE source file')
parser.add_argument('--url-file', dest='url_file', help='CVE URL extension')
@@ -409,10 +447,10 @@ def main(argv):
if 'init_mitre' == args.command:
init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update)
- append_cve_database(args.cve_file)
+ append_cve_database(True,args.cve_file)
elif 'update_mitre' == args.command:
- # No difference from init at this time
init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update)
+ append_cve_database(False,args.cve_file)
else:
print("Command not found")
diff --git a/bin/nist/datasource_2015.json b/bin/nist/datasource_2015.json
index 51c21cb..e26b0f7 100755
--- a/bin/nist/datasource_2015.json
+++ b/bin/nist/datasource_2015.json
@@ -7,7 +7,7 @@
"name" : "NIST",
"description" : "NIST 2015",
"cve_filter" : "CVE-2015",
- "init" : "bin/nist/srtool_nist.py -n --source='NIST 2015' --file=data/nvdcve-1.0-2015.json --url-file=nvdcve-1.0-2015.json.gz --url-meta=nvdcve-1.0-2015.meta",
+ "init" : "bin/nist/srtool_nist.py -I --source='NIST 2015' --file=data/nvdcve-1.0-2015.json --url-file=nvdcve-1.0-2015.json.gz --url-meta=nvdcve-1.0-2015.meta",
"update" : "bin/nist/srtool_nist.py -n --source='NIST 2015' --file=data/nvdcve-1.0-2015.json --url-file=nvdcve-1.0-2015.json.gz --url-meta=nvdcve-1.0-2015.meta",
"lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2015.json %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/nist/datasource_2016.json b/bin/nist/datasource_2016.json
index 82671a1..026060b 100755
--- a/bin/nist/datasource_2016.json
+++ b/bin/nist/datasource_2016.json
@@ -7,7 +7,7 @@
"name" : "NIST",
"description" : "NIST 2016",
"cve_filter" : "CVE-2016",
- "init" : "bin/nist/srtool_nist.py -n --source='NIST 2016' --file=data/nvdcve-1.0-2016.json --url-file=nvdcve-1.0-2016.json.gz --url-meta=nvdcve-1.0-2016.meta",
+ "init" : "bin/nist/srtool_nist.py -I --source='NIST 2016' --file=data/nvdcve-1.0-2016.json --url-file=nvdcve-1.0-2016.json.gz --url-meta=nvdcve-1.0-2016.meta",
"update" : "bin/nist/srtool_nist.py -n --source='NIST 2016' --file=data/nvdcve-1.0-2016.json --url-file=nvdcve-1.0-2016.json.gz --url-meta=nvdcve-1.0-2016.meta",
"lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2016.json %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/nist/datasource_2017.json b/bin/nist/datasource_2017.json
index 9fec02d..64bd017 100755
--- a/bin/nist/datasource_2017.json
+++ b/bin/nist/datasource_2017.json
@@ -7,7 +7,7 @@
"name" : "NIST",
"description" : "NIST 2017",
"cve_filter" : "CVE-2017",
- "init" : "bin/nist/srtool_nist.py -n --source='NIST 2017' --file=data/nvdcve-1.0-2017.json --url-file=nvdcve-1.0-2017.json.gz --url-meta=nvdcve-1.0-2017.meta",
+ "init" : "bin/nist/srtool_nist.py -I --source='NIST 2017' --file=data/nvdcve-1.0-2017.json --url-file=nvdcve-1.0-2017.json.gz --url-meta=nvdcve-1.0-2017.meta",
"update" : "bin/nist/srtool_nist.py -n --source='NIST 2017' --file=data/nvdcve-1.0-2017.json --url-file=nvdcve-1.0-2017.json.gz --url-meta=nvdcve-1.0-2017.meta",
"lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2017.json %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/nist/datasource_2018.json b/bin/nist/datasource_2018.json
index 30860e8..216ef5a 100755
--- a/bin/nist/datasource_2018.json
+++ b/bin/nist/datasource_2018.json
@@ -7,7 +7,7 @@
"name" : "NIST",
"description" : "NIST 2018",
"cve_filter" : "CVE-2018",
- "init" : "bin/nist/srtool_nist.py -n --source='NIST 2018' --file=data/nvdcve-1.0-2018.json --url-file=nvdcve-1.0-2018.json.gz --url-meta=nvdcve-1.0-2018.meta",
+ "init" : "bin/nist/srtool_nist.py -I --source='NIST 2018' --file=data/nvdcve-1.0-2018.json --url-file=nvdcve-1.0-2018.json.gz --url-meta=nvdcve-1.0-2018.meta",
"update" : "bin/nist/srtool_nist.py -n --source='NIST 2018' --file=data/nvdcve-1.0-2018.json --url-file=nvdcve-1.0-2018.json.gz --url-meta=nvdcve-1.0-2018.meta",
"lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2018.json %command%",
"lastModifiedDate" : "2018-03-01 01:01:01",
diff --git a/bin/nist/datasource_2019.json b/bin/nist/datasource_2019.json
new file mode 100755
index 0000000..ce9fc12
--- /dev/null
+++ b/bin/nist/datasource_2019.json
@@ -0,0 +1,18 @@
+{
+ "datasource" : [
+ {
+ "key" : "0010-nist-2019",
+ "data" : "cve",
+ "source" : "nist",
+ "name" : "NIST",
+ "description" : "NIST 2019",
+ "cve_filter" : "CVE-2019",
+ "init" : "bin/nist/srtool_nist.py -I --source='NIST 2019' --file=data/nvdcve-1.0-2019.json --url-file=nvdcve-1.0-2019.json.gz --url-meta=nvdcve-1.0-2019.meta",
+ "update" : "bin/nist/srtool_nist.py -n --source='NIST 2019' --file=data/nvdcve-1.0-2019.json --url-file=nvdcve-1.0-2019.json.gz --url-meta=nvdcve-1.0-2019.meta",
+ "lookup" : "bin/nist/srtool_nist.py --file=data/nvdcve-1.0-2019.json %command%",
+ "lastModifiedDate" : "2018-03-01 01:01:01",
+ "update_frequency" : "3",
+ "update_time" : "02:00:00"
+ }
+ ]
+}
diff --git a/bin/nist/srtool_nist.py b/bin/nist/srtool_nist.py
index 0952b0c..c05e65d 100755
--- a/bin/nist/srtool_nist.py
+++ b/bin/nist/srtool_nist.py
@@ -34,16 +34,17 @@ import sqlite3
import subprocess
import json
import urllib
+from datetime import datetime, date, timedelta
+import pytz
+
+from urllib.request import urlopen, URLError
+from urllib.parse import urlparse
# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.insert(0, dir_path)
from common.srt_schema import ORM
-from datetime import datetime, date
-from urllib.request import urlopen, URLError
-from urllib.parse import urlparse
-
# setup
lookupTable = []
cveIndex = {}
@@ -91,6 +92,39 @@ def get_name_sort(cve_name):
cve_name_sort = cve_name
return cve_name_sort
+# Newly discovered or updated CVEs default to NEW for triage
+# Inited CVEs default to HISTORICAL, unless they are within the courtesy CVE_INIT_NEW_DELTA
+init_new_date = None
+def get_cve_default_status(is_init,publishedDate):
+ global init_new_date
+
+ if None == init_new_date:
+ # Precalculate and cache the relative 'new' date for efficiency
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+ sql = '''SELECT * FROM orm_srtsetting WHERE name=?'''
+ CVE_INIT_NEW_DELTA = cur.execute(sql, ('CVE_INIT_NEW_DELTA',)).fetchone()
+ if CVE_INIT_NEW_DELTA is None:
+ cve_init_new_delta = 30
+ else:
+ cve_init_new_delta = int(CVE_INIT_NEW_DELTA[ORM.SRTSETTING_VALUE])
+
+ date_delta = timedelta(days=cve_init_new_delta)
+ init_new_date = datetime.now(pytz.utc) - date_delta
+ #print("\nPreset new data = %s" % init_new_date.strftime("%Y%m%d"))
+ init_new_date = init_new_date.strftime("%Y-%m-%d")
+
+ if is_init:
+ # Note: the NIST 'published date' is in the format "2017-05-11", so do a simple string compare
+ #print("INIT status: %s > %s" % (publishedDate, init_new_date))
+ if not publishedDate or (publishedDate > init_new_date):
+ return ORM.STATUS_NEW
+ else:
+ return ORM.STATUS_HISTORICAL
+ else:
+ return ORM.STATUS_NEW
+
+
#################################
# check for updates and apply if any
#
@@ -100,7 +134,7 @@ def get_name_sort(cve_name):
#gets CVE-Modified feed, determines if we are out of date, and applies updates if true
#tracks history in update_log.txt
#incremental argument is boolean that idicates if bulk updating or incremental updating.
-def update_nist(datasource_description, url_file, url_meta, cve_file, incremental, force_update):
+def update_nist(is_init,datasource_description, url_file, url_meta, cve_file, incremental, force_update):
nist_cve_url = '%s/%s' % (nist_cve_url_base,url_file)
nist_meta_url = '%s/%s' % (nist_meta_url_base,url_meta)
@@ -138,14 +172,14 @@ def update_nist(datasource_description, url_file, url_meta, cve_file, incrementa
date_new = datetime.strptime(content, 'lastModifiedDate:%Y-%m-%dT%H:%M:%S')
date_past = datetime.strptime(ds[ORM.DATASOURCE_LASTMODIFIEDDATE], '%Y-%m-%d %H:%M:%S')
- log.write("BEGINNING NIST UPDATES\n")
+ log.write("BEGINNING NIST %s\n" % ('INITS' if is_init else 'UPDATES'))
#determine if we are out of date and apply updates if true
if (date_new > date_past) or force_update:
pre_update_time = datetime.now() #used for logging purposes only
- nist_json(nist_cve_url, ds[ORM.DATASOURCE_ID], nist_file, log, date_new, incremental)
- log.write("began updates: %s\n" % str(pre_update_time))
- log.write("finished updates: %s\n" % str(datetime.now()) )
+ nist_json(is_init,nist_cve_url, ds[ORM.DATASOURCE_ID], nist_file, log, date_new, incremental)
+ log.write("began %s: %s\n" % ( 'init' if is_init else 'updates', str(pre_update_time) ))
+ log.write("finished %s: %s\n" % ( 'init' if is_init else 'updates', str(datetime.now()) ))
log.write("=============================================================================\n")
log.write("\n")
@@ -154,12 +188,11 @@ def update_nist(datasource_description, url_file, url_meta, cve_file, incrementa
c.execute(sql, (str(date_new),))
conn.commit()
else:
-
- log.write("No update needed\n")
+ log.write("No %s needed\n" % ('init' if is_init else 'update'))
log.write("Checked: %s\n" % datetime.now())
log.write("=============================================================================\n")
log.write("\n")
- print("NO UPDATE NEEDED")
+ print("NO %s NEEDED" % ('INIT' if is_init else 'UPDATE'))
# Reset datasource's update_time as today
sql = "UPDATE orm_datasource SET update_time = ? WHERE id='%s'" % ds[ORM.DATASOURCE_ID]
@@ -196,7 +229,7 @@ def file_date(filename,utc=False):
#parses JSON, creates CVE object, and updates database as necessary. Commits to database on success
#will EITHER create new record in orm_cve if cve does not exist OR overwrite every field if existing cve out-of-date OR ignore cve
#requires json to be formatted with NIST Json schema (https://csrc.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema)
-def nist_json(summary_json_url, datasource_id, datasource_file, log, date_new, incremental):
+def nist_json(is_init,summary_json_url, datasource_id, datasource_file, log, date_new, incremental):
import traceback
import gzip
@@ -266,7 +299,7 @@ def nist_json(summary_json_url, datasource_id, datasource_file, log, date_new, i
#check if cve object `v` need to be uploaded to database (cases: new cve, modified cve, or no changes)
#if true, apply changes. Else ignore and continue
- v_id, is_change = sql_cve_query(conn, v, log)
+ v_id, is_change = sql_cve_query(conn, v, is_init,log)
#if incremental update and CVE changed, save json copy of the cve to cache
@@ -366,21 +399,25 @@ class Cve():
#new CVE -> INSERT || modified CVE -> UPDATE || no change -> ignore and return
#returns (CVE_ID, BOOL) tuple, True if insert or update executed
### THIS DOES NOT CALL CONNECTION.COMMIT()
-def sql_cve_query(conn, cve, log):
- CVE_LASTMODIFIEDDATE = 14
+def sql_cve_query(conn, cve, is_init, log):
is_change = False
cur = conn.cursor()
sql = '''SELECT * FROM orm_cve WHERE name=?'''
exists = cur.execute(sql, (cve.name,)).fetchone()
cve_id = -1
if exists is None:
+ # Get the default CVE status
+ status = get_cve_default_status(is_init,cve.publishedDate)
+ print("BAR:%s=%s" % (cve.name,status))
+
sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, srt_updated, packages)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'''
- cur.execute(sql, (cve.name, get_name_sort(cve.name), cve.priority, cve.status, cve.comments, cve.comments_private, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, 1, cve.publish_state, cve.publish_date, cve.description, cve.publishedDate, cve.lastModifiedDate, cve.recommend, cve.recommend_list, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, datetime.now(),''))
+ cur.execute(sql, (cve.name, get_name_sort(cve.name), cve.priority, status, cve.comments, cve.comments_private, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, 1, cve.publish_state, cve.publish_date, cve.description, cve.publishedDate, cve.lastModifiedDate, cve.recommend, cve.recommend_list, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, datetime.now(),''))
is_change = True
cve_id = cur.lastrowid
log.write("\tINSERTED '%s'\n" % cve.name)
- elif exists[CVE_LASTMODIFIEDDATE] < cve.lastModifiedDate:
+
+ elif exists[ORM.CVE_LASTMODIFIEDDATE] < cve.lastModifiedDate:
sql = ''' UPDATE orm_cve
SET recommend = ?,
recommend_list = ?,
@@ -397,7 +434,12 @@ def sql_cve_query(conn, cve, log):
cur.execute(sql, (cve.recommend, cve.recommend_list, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, cve.description, cve.lastModifiedDate, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, exists[0]))
is_change = True
log.write("\tUPDATED '%s'\n" % cve.name)
- cve_id = exists[0]
+ cve_id = exists[ORM.CVE_ID]
+
+ ### TO-DO
+ ### CREATE NOTIFICATION IF SCORE/SEVERITY HAS CHANGED
+ ###
+
else:
is_change = False
log.write("\tSKIPPED '%s'\n" % cve.name)
@@ -610,6 +652,7 @@ def nist_scan_configuration_or(cpe_or_node, name, and_enum):
def main(argv):
global verbose
parser = argparse.ArgumentParser(description='srtool_cve.py: manage the CVEs within SRTool database')
+ parser.add_argument('--init_nist', '-I', action='store_const', const='init_nist', dest='command', help='Initialize nvd.nist.gov/vuln/data-feeds for a specified datasource')
parser.add_argument('--update_nist', '-n', action='store_const', const='update_nist', dest='command', help='Check nvd.nist.gov/vuln/data-feeds for updates on a specified datasource')
parser.add_argument('--source', dest='source', help='Local CVE source file')
parser.add_argument('--url-file', dest='url_file', help='CVE URL extension')
@@ -653,20 +696,21 @@ def main(argv):
exit(1)
ret = 0
- if 'update_nist' == args.command:
+ if ('init_nist' == args.command) or ('update_nist' == args.command):
+ is_init = ('init_nist' == args.command)
try:
- print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time")
- update_nist(args.source, args.url_file, args.url_meta, args.cve_file, False, args.force_update)
- master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tUPDATED\n" % (date.today(), args.source))
- print("DATABASE UPDATE FINISHED\n")
+ print ("BEGINNING NIST %s PLEASE WAIT ... this can take some time" % ('INIT' if is_init else 'UPDATES'))
+ update_nist(is_init, args.source, args.url_file, args.url_meta, args.cve_file, False, args.force_update)
+ master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\t%s\n" % (date.today(), args.source, "INIT'ED" if is_init else 'UPDATED'))
+ print("DATABASE %s FINISHED\n" % ('INIT' if is_init else 'UPDATE'))
except Exception as e:
- print("DATABASE UPDATED FAILED ... %s" % e)
+ print("DATABASE %s FAILED ... %s" % ('INIT' if is_init else 'UPDATE',e))
master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), args.source, e))
ret = 1
elif 'update_nist_incremental' == args.command:
try:
print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time")
- update_nist(args.source, args.url_file, args.url_meta, args.cve_file, True, args.force_update)
+ update_nist(False,args.source, args.url_file, args.url_meta, args.cve_file, True, args.force_update)
master_log.write("SRTOOL:%s:'NIST JSON Modified Data':\t\t\t...\t\t\tUPDATED\n" % date.today())
print("DATABASE UPDATE FINISHED\n")
except Exception as e: