diff options
Diffstat (limited to 'bin/mitre/srtool_mitre.py')
| -rwxr-xr-x | bin/mitre/srtool_mitre.py | 146 |
1 files changed, 103 insertions, 43 deletions
diff --git a/bin/mitre/srtool_mitre.py b/bin/mitre/srtool_mitre.py index 3c6af89d..cdf6ff6e 100755 --- a/bin/mitre/srtool_mitre.py +++ b/bin/mitre/srtool_mitre.py @@ -31,8 +31,7 @@ import sys import xml.etree.ElementTree as ET import argparse import shutil -import sqlite3 -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date import pytz from urllib.request import urlopen @@ -41,10 +40,13 @@ from urllib.request import urlopen dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM +from common.srtool_progress import * +from common.srtool_sql import * # Setup: srtDbName = 'srt.sqlite' srtErrorLog = 'srt_errors.txt' +COMMIT_DELAY = 64 mitre_cvrf_url = 'https://cve.mitre.org/data/downloads' mitre_cvrf_xml = 'data/allitems-cvrf-year-2018.xml' @@ -56,6 +58,8 @@ mitre_cache_dir = 'data/cache/mitre' # Debugging support verbose = False +cmd_skip = 0 +cmd_count = 0 # Development support overrides = {} @@ -88,18 +92,17 @@ def srt_error_log(msg): f1.close() -# Newly discovered or updated CVEs default to NEW for triage -# Inited CVEs default to HISTORICAL, unless they are within the courtesy CVE_INIT_NEW_DELTA +# Newly discovered CVEs default to NEW_RESERVED if reserved, else NEW for triage init_new_date = None def get_cve_default_status(is_init,publishedDate,description): global init_new_date if None == init_new_date: # Precalculate and cache the relative 'new' date for efficiency - conn = sqlite3.connect(srtDbName) + conn = SQL_CONNECT() cur = conn.cursor() sql = '''SELECT * FROM orm_srtsetting WHERE name=?''' - CVE_INIT_NEW_DELTA = cur.execute(sql, ('CVE_INIT_NEW_DELTA',)).fetchone() + CVE_INIT_NEW_DELTA = SQL_EXECUTE(cur, sql, ('CVE_INIT_NEW_DELTA',)).fetchone() if CVE_INIT_NEW_DELTA is None: cve_init_new_delta = 30 else: @@ -110,18 +113,10 @@ def get_cve_default_status(is_init,publishedDate,description): #print("\nPreset new data = %s" % init_new_date.strftime("%Y-%m-%d")) init_new_date = init_new_date.strftime("%Y-%m-%d") - if is_init: - # Note: the NIST 'published date' is in the format "2017-05-11", so do a simple string compare - #print("INIT status: %s versus %s" % (init_new_date,publishedDate)) - if not publishedDate or (publishedDate > init_new_date): - # Is this reserved by Mitre? Is '** RESERVED **' within the first 20 char positions? - reserved_pos = description.find('** RESERVED **') - if (0 <= reserved_pos) and (20 > reserved_pos): - return ORM.STATUS_NEW_RESERVED - else: - return ORM.STATUS_NEW - else: - return ORM.STATUS_HISTORICAL + # Is this reserved by Mitre? Is '** RESERVED **' within the first 20 char positions? + reserved_pos = description.find('** RESERVED **') + if (0 <= reserved_pos) and (20 > reserved_pos): + return ORM.STATUS_NEW_RESERVED else: return ORM.STATUS_NEW @@ -173,6 +168,11 @@ def fetch_cve(cve_name,cvrf_xml_file): datasource_xml = os.path.join(srtool_basepath,cvrf_xml_file) cache_file = os.path.join(srtool_basepath,mitre_cache_dir,"%s.txt" % cve_name) + # Insure that the original data file exists + if not os.path.isfile(datasource_xml): + print("description=There is no loaded Mitre data.") + return + # Insure the cache dir exists cache_dir = os.path.join(srtool_basepath,mitre_cache_dir) if not os.path.isdir(cache_dir): @@ -268,36 +268,52 @@ def append_cve_database(is_init,file_xml): tree = ET.parse(file_xml) root = tree.getroot() - # Max count for development cycle - cmd_count = 20 if get_override('SRTDBG_MINIMAL_DB') else 0 - - conn = sqlite3.connect(srtDbName) + conn = SQL_CONNECT() cur = conn.cursor() cur_write = conn.cursor() cur_ds = conn.cursor() datasource_id = 0 + srtool_today = date.today() + version_date = '' + + # Progress expected max + if cmd_count: + progress_set_max(cmd_count) + else: + progress_set_max(len(root)) i = 0 for child in root: i += 1 + + # Extract document date record + # <DocumentTracking> + # <Version>2020.01.14.22</Version> + if 'DocumentTracking' in child.tag: + for child_d in child: + if 'Version' in child_d.tag: + version_date = datetime.strptime(child_d.text, '%Y.%m.%d.%H') + + # Find the Vulnerability records if not 'Vulnerability' in child.tag: continue + summary = _extract_text(child) cve_name = summary['CVE'] + progress_show(cve_name,force_newline=True) # Progress indicator support - if 0 == i % 10: - print('%04d: %20s \r' % (i,cve_name), end='') - if (0 == i % 200): - conn.commit() + if 0 == (i % COMMIT_DELAY): + SQL_COMMIT(conn) print('') + sys.stdout.flush() if cmd_count and (i > cmd_count): break # Find the datasource matching these CVE prefixes if 0 == datasource_id: sql = "SELECT * FROM orm_datasource WHERE data = ? AND source = ?" - cur_ds.execute(sql, ('cve','mitre',)) + SQL_EXECUTE(cur_ds, sql, ('cve','mitre',)) for ds in cur_ds: if ds[ORM.DATASOURCE_CVE_FILTER] and cve_name.startswith(ds[ORM.DATASOURCE_CVE_FILTER]): datasource_id = ds[ORM.DATASOURCE_ID] @@ -309,28 +325,45 @@ def append_cve_database(is_init,file_xml): # Define the CVE (if not already there - e.g. not defined by NIST) sql = ''' SELECT * FROM orm_cve WHERE name = ?''' - cve = cur_write.execute(sql, (cve_name,)).fetchone() + cve = SQL_EXECUTE(cur_write, sql, (cve_name,)).fetchone() if cve: cve_id = cve[ORM.CVE_ID] - print("MITRE:FOUND %20s\r" % cve_name, end='') + if not progress_status()[PROGRESS_STATUS_ENABLE]: + print("MITRE:FOUND %20s" % cve_name, end='\r') else: # Get the default CVE status status = get_cve_default_status(is_init,summary['Published'],summary['Description']) - sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, srt_updated, packages) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' - cur.execute(sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, status, '', '', 'CVE', 'MITRE', '', 1, ORM.PUBLISH_UNPUBLISHED, '', summary['Description'], summary['Published'], summary['Modified'],0, '', '', '', '', '', datetime.now(),'')) - cve_id = cur.lastrowid - print("MITRE:ADDED %20s\r" % cve_name) + # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 + sql = ''' INSERT INTO orm_cve (name, name_sort, priority, status, comments, comments_private, tags, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, acknowledge_date, description, "publishedDate", "lastModifiedDate", recommend, recommend_list, "cvssV3_baseScore", "cvssV3_baseSeverity", "cvssV2_baseScore", "cvssV2_severity", srt_updated, srt_created, packages) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' + SQL_EXECUTE(cur, sql, (cve_name, get_name_sort(cve_name), ORM.PRIORITY_UNDEFINED, status, '', '', '', 'CVE', 'MITRE', '', True, ORM.PUBLISH_UNPUBLISHED, '', None, summary['Description'], summary['Published'], summary['Modified'],0, '', '', '', '', '', datetime.now(), datetime.now(),'')) + # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 + cve_id = SQL_GET_LAST_ROW_INSERTED_ID(cur) + if not progress_status()[PROGRESS_STATUS_ENABLE]: + print("MITRE:ADDED %20s" % cve_name, end='\r') + + # Also create CVE history entry + update_comment = "%s {%s}" % (ORM.UPDATE_CREATE_STR % ORM.UPDATE_SOURCE_CVE,'Created from MITRE') + sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)''' + SQL_EXECUTE(cur, sql, (cve_id,update_comment,srtool_today,ORM.USER_SRTOOL_NAME,) ) # Add this data source to the CVE sql = '''SELECT * FROM orm_cvesource WHERE cve_id=? AND datasource_id=? ''' - if not cur_ds.execute(sql, (cve_id,datasource_id)).fetchone(): - sql = ''' INSERT into orm_cvesource (cve_id, datasource_id) VALUES (?, ?)''' - cur_ds.execute(sql, (cve_id,datasource_id)) + if not SQL_EXECUTE(cur_ds, sql, (cve_id,datasource_id)).fetchone(): + sql = ''' INSERT INTO orm_cvesource (cve_id, datasource_id) VALUES (?, ?)''' + SQL_EXECUTE(cur_ds, sql, (cve_id,datasource_id)) + + #update datasource's lastModifiedDate after successsfuly updating it + if datasource_id: + print("\nVersion Date=%s" % str(version_date)) + sql = """UPDATE orm_datasource SET "lastModifiedDate" = ? WHERE id='%s'""" % datasource_id + SQL_EXECUTE(cur, sql, (str(version_date),)) - conn.commit() + SQL_COMMIT(conn) print("\nTotal = %5d\n" % i) + # End progress + progress_done('Done') ################################# # test dump @@ -391,8 +424,9 @@ def dump(file_xml): print("OTHER TOP TAG=%s" % child.tag) i += 1 - if (0 == (i % 20)): - print("%5d\r" % i,end = '') + if not progress_status()[PROGRESS_STATUS_ENABLE]: + if (0 == (i % 20)): + print("%5d" % i,end = '\r') print("\nTotal = %5d\n" % i) ################################# @@ -401,25 +435,46 @@ def dump(file_xml): def main(argv): global verbose + global cmd_skip + global cmd_count # setup parser = argparse.ArgumentParser(description='srtool_mitre.py: manage Mitre CVE data') - parser.add_argument('--initialize', '-I', action='store_const', const='init_mitre', dest='command', help='Download the Mitre source CVE file') + parser.add_argument('--initialize', '-I', action='store_const', const='init_mitre', dest='command', help='Download the Mitre source CVE file, add CVEs') parser.add_argument('--update', '-u', action='store_const', const='update_mitre', dest='command', help='Update the Mitre source CVE file') parser.add_argument('--source', dest='source', help='Local CVE source file') parser.add_argument('--url-file', dest='url_file', help='CVE URL extension') + parser.add_argument('--download-only', action='store_const', const='download_mitre', dest='command', help='Download the Mitre source CVE file only') parser.add_argument('--cve-detail', '-d', dest='cve_detail', help='Fetch CVE detail') parser.add_argument('--file', dest='cve_file', help='Local CVE source file') + + parser.add_argument('--progress', '-P', action='store_true', dest='do_progress', help='Progress output') parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force update') + parser.add_argument('--update-skip-history', '-H', action='store_true', dest='update_skip_history', help='Skip history updates') parser.add_argument('--verbose', '-v', action='store_true', dest='is_verbose', help='Enable verbose debugging output') + parser.add_argument('--skip', dest='skip', help='Debugging: skip record count') + parser.add_argument('--count', dest='count', help='Debugging: short run record count') + parser.add_argument('--debug-sql', action='store_true', dest='debug_sql', help='Debug SQL writes') parser.add_argument('--dump', '-D', action='store_const', const='dump', dest='command', help='test dump data') parser.add_argument('--dump2', '-2', action='store_const', const='dump2', dest='command', help='test dump data') args = parser.parse_args() + # fetch any environment overrides + set_override('SRTDBG_MINIMAL_DB') + if args.is_verbose: verbose = True + if None != args.skip: + cmd_skip = int(args.skip) + if None != args.count: + cmd_count = int(args.count) + elif get_override('SRTDBG_MINIMAL_DB'): + cmd_count = 20 + if args.debug_sql: + SQL_DEBUG(True,'MTR') + progress_set_on(args.do_progress) if 'dump' == args.command: dump(mitre_cvrf_xml) @@ -438,8 +493,6 @@ def main(argv): fetch_cve(args.cve_detail,args.cve_file) return - # fetch any environment overrides - set_override('SRTDBG_MINIMAL_DB') # Required parameters to continue if not args.source: @@ -449,15 +502,22 @@ def main(argv): print("ERROR: missing --url_file parameter") exit(1) + # Currently no different between initialize and update actions if 'init_mitre' == args.command: init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update) append_cve_database(True,args.cve_file) elif 'update_mitre' == args.command: init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update) append_cve_database(False,args.cve_file) + elif 'download_mitre' == args.command: + init_mitre_file(args.source,args.url_file,args.cve_file,args.force_update) else: print("Command not found") + # Dump the SQL transaction data + if args.debug_sql: + SQL_DUMP() + if __name__ == '__main__': srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) |