#!/usr/bin/env python3 # # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # # Security Response Tool Commandline Tool # # Copyright (C) 2018-2019 Wind River Systems # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import sys import argparse import sqlite3 from datetime import datetime, date import time import re import subprocess import json # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM # Setup: verbose = False cmd_skip = 0 cmd_count = 0 force = False srtDbName = 'srt.sqlite' ################################# # Common routines # # quick development/debugging support def _log(msg): DBG_LVL = os.environ['SRTDBG_LVL'] if ('SRTDBG_LVL' in os.environ) else 2 DBG_LOG = os.environ['SRTDBG_LOG'] if ('SRTDBG_LOG' in os.environ) else '/tmp/srt_dbg.log' if 1 == DBG_LVL: print(msg) elif 2 == DBG_LVL: f1=open(DBG_LOG, 'a') f1.write("|" + msg + "|\n" ) f1.close() # Sub Process calls def execute_process(*args): cmd_list = [] for arg in args: if isinstance(arg, (list, tuple)): # Flatten all the way down for a in arg: cmd_list.append(a) else: cmd_list.append(arg) # Python < 3.5 compatible if sys.version_info < (3,5): process = subprocess.Popen(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: stdout, stderr = process.communicate(input) except: process.kill() process.wait() raise retcode = process.poll() return retcode, stdout, stderr else: result = subprocess.run(cmd_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return result.returncode,result.stdout,result.stderr ################################# # reset sources # # source_data = (source_id) def commit_to_source(conn, source_data): sql = ''' UPDATE orm_datasource SET loaded = ? WHERE id = ?''' cur = conn.cursor() print("UPDATE_SCORE:%s" % str(source_data)) cur.execute(sql, source_data) def sources(cmnd): conn = sqlite3.connect(srtDbName) c = conn.cursor() print('Sources(%s)' % cmnd) c.execute("SELECT * FROM orm_datasource") is_change = False for ds in c: if 'set' == cmnd: commit_to_source(conn,(True,ds[ORM.DATASOURCE_ID])) is_change = True elif 'reset' == cmnd: commit_to_source(conn,(False,ds[ORM.DATASOURCE_ID])) is_change = True elif 'reset_not_nist' == cmnd: if 'nist' != ds[ORM.DATASOURCE_SOURCE]: print("RESETTING Data source [%s] data='%s' of '%s' load state from '%s' is '%s'" % (ds[ORM.DATASOURCE_ID],ds[ORM.DATASOURCE_DATA],ds[ORM.DATASOURCE_DESCRIPTION],ds[ORM.DATASOURCE_SOURCE],ds[ORM.DATASOURCE_LOADED])) commit_to_source(conn,(False,ds[ORM.DATASOURCE_ID])) else: commit_to_source(conn,(True,ds[ORM.DATASOURCE_ID])) is_change = True elif 'triage_keywords' == cmnd: if 'triage_keywords' == ds[ORM.DATASOURCE_DATA]: print("RESETTING Data source [%s] data='%s' of '%s' load state from '%s' is '%s'" % (ds[ORM.DATASOURCE_ID],ds[ORM.DATASOURCE_DATA],ds[ORM.DATASOURCE_DESCRIPTION],ds[ORM.DATASOURCE_SOURCE],ds[ORM.DATASOURCE_LOADED])) commit_to_source(conn,(False,ds[ORM.DATASOURCE_ID])) is_change = True else: print("Data source [%s] data='%s' of '%s' load state from '%s' is '%s'" % (ds[ORM.DATASOURCE_ID],ds[ORM.DATASOURCE_DATA],ds[ORM.DATASOURCE_DESCRIPTION],ds[ORM.DATASOURCE_SOURCE],ds[ORM.DATASOURCE_LOADED])) if is_change: conn.commit() def settings(): conn = sqlite3.connect(srtDbName) c = conn.cursor() # Scan the SRTool Settings c.execute("SELECT * FROM orm_srtsetting") for setting in c: print("Setting[%s] = '%s'" % (setting[ORM.SRTSETTING_NAME], setting[ORM.SRTSETTING_VALUE][0:40])) ################################# # remove_app_sources # def remove_app_sources(master_app): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() # Scan the SRTool Settings cur.execute("SELECT * FROM orm_datasource") is_change = False for setting in cur: if master_app == setting[ORM.DATASOURCE_SOURCE]: print("Deleting [%s] = '%s','%s'" % (setting[ORM.DATASOURCE_ID], setting[ORM.DATASOURCE_SOURCE], setting[ORM.SRTSETTING_VALUE])) sql = 'DELETE FROM orm_datasource WHERE id=?' cur_write.execute(sql, (setting[ORM.DATASOURCE_ID],)) is_change = True if is_change: conn.commit() conn.close() ################################# # fix_new_reserved # # Is this reserved by Mitre? Is '** RESERVED **' within the first 20 char positions? def fix_new_reserved(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() cur.execute('SELECT * FROM orm_cve WHERE status = "%s"' % ORM.STATUS_NEW) i = 0 j = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') if (0 == i % 200): conn.commit() # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break reserved_pos = cve[ORM.CVE_DESCRIPTION].find('** RESERVED **') if (0 <= reserved_pos) and (20 > reserved_pos): print("STATUS_NEW_RESERVED:%s:%s:%s" % (cve[ORM.CVE_STATUS],cve[ORM.CVE_NAME],cve[ORM.CVE_DESCRIPTION][:40])) # NOTE: we do not touch 'cve.srt_updated' for this background change sql = ''' UPDATE orm_cve SET status = ? WHERE id = ?''' cur_write.execute(sql, (ORM.STATUS_NEW_RESERVED, cve[ORM.CVE_ID],)) j += 1 print("\nCVE COUNT=%5d,%5d" % (i,j)) conn.commit() ################################# # fix_new_tags # # Fix the None "cve.tags" fields def fix_new_tags(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() cur.execute('SELECT * FROM orm_cve') i = 0 j = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') if (0 == i % 200): conn.commit() # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break if not cve[ORM.CVE_TAGS]: # NOTE: we do not touch 'cve.srt_updated' for this background change sql = ''' UPDATE orm_cve SET tags = ? WHERE id = ?''' cur_write.execute(sql, ('', cve[ORM.CVE_ID],)) j += 1 print("\nCVE COUNT=%5d,%5d" % (i,j)) conn.commit() ################################# # fixup fix_name_sort # # Recompute all of the CVE name_sort fields def fix_name_sort(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() cur.execute('SELECT * FROM orm_cve') for i,cve in enumerate(cur): name_sort = get_name_sort(cve[ORM.CVE_NAME]) # Progress indicator support if 0 == i % 10: print('%05d: %20s to %20s\r' % (i,cve[ORM.CVE_NAME],name_sort), end='') if (0 == i % 200): conn.commit() # NOTE: we do not touch 'cve.srt_updated' for this background change sql = ''' UPDATE orm_cve SET name_sort = ? WHERE id = ?''' cur_write.execute(sql, (name_sort, cve[ORM.CVE_ID],)) conn.commit() ################################# # fixup fix_cve_recommend # # Reset empty CVE recommend fields to the proper integer zero def fix_cve_recommend(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() cur.execute('SELECT * FROM orm_cve WHERE recommend = ""') i = 0 fix_count = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') # # Fix miss-write to lastModifiedDate, missing integer for recommend # fix = False lastModifiedDate = cve[ORM.CVE_LASTMODIFIEDDATE] if '0' == lastModifiedDate: lastModifiedDate = '' fix = True recommend = cve[ORM.CVE_RECOMMEND] if not recommend: recommend = 0 fix = True # NOTE: we do not touch 'cve.srt_updated' for this background change if fix: sql = ''' UPDATE orm_cve SET recommend = ?, lastModifiedDate = ? WHERE id = ?''' cur_write.execute(sql, (recommend, lastModifiedDate, cve[ORM.CVE_ID],)) fix_count += 1 if (199 == fix_count % 200): conn.commit() print("CVE RECOMMEND FIX COUNT=%d of %d" % (fix_count,i)) if fix_count: conn.commit() conn.close() ################################# # fixup fix_srt_dates # # Reset older 'date' values as 'datetime' values def _fix_datetime(value,default): if (not value) or (not value[0].isdigit()): return(default) elif ':' in value: return(value) else: return(datetime.strptime(value, '%Y-%m-%d')) def _fix_date(value,default): if (not value) or (not value[0].isdigit()): return(False,default) elif not ':' in value: return(False,value) else: value = re.sub('\..*','',value) dt = datetime.strptime(value,ORM.DATASOURCE_DATETIME_FORMAT) return(True,dt.strftime(ORM.DATASOURCE_DATE_FORMAT)) def fix_srt_datetime(scope): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() if ('d' == scope) or ('all' == scope): cur.execute('SELECT * FROM orm_defect') i = 0 is_change_count = 0 for defect in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,defect[ORM.DEFECT_NAME]), end='') if (0 == i % 200): conn.commit() # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break defect_srt_updated = _fix_datetime(defect[ORM.DEFECT_SRT_UPDATED],defect[ORM.DEFECT_DATE_UPDATED]) if defect_srt_updated == defect[ORM.DEFECT_SRT_UPDATED]: continue sql = ''' UPDATE orm_defect SET srt_updated = ? WHERE id = ?''' cur_write.execute(sql, (defect_srt_updated, defect[ORM.DEFECT_ID],)) is_change_count += 1 print("DEFECT DATE FIX COUNT=%d/%d" % (is_change_count,i)) conn.commit() # INVESTIGATION DATE FIX COUNT=1089363, real 12m20.041s = 1472 recs/sec if ('i' == scope) or ('all' == scope): cur.execute('SELECT * FROM orm_investigation') i = 0 is_change_count = 0 for investigation in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,investigation[ORM.INVESTIGATION_NAME]), end='') if (0 == i % 200): conn.commit() time.sleep(0.1) # give time for Sqlite to sync # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break srt_updated = _fix_datetime(investigation[ORM.INVESTIGATION_SRT_UPDATED],None) srt_created = _fix_datetime(investigation[ORM.INVESTIGATION_SRT_CREATED],None) if (not srt_updated) or (not srt_created): print("ERROR[%d]: bad date field at '%s', U=%s,C=%s" % (i,investigation[ORM.INVESTIGATION_ID],investigation[ORM.INVESTIGATION_SRT_UPDATED],investigation[ORM.INVESTIGATION_SRT_CREATED])) exit(1) if (srt_updated == investigation[ORM.INVESTIGATION_SRT_UPDATED]) and (srt_created == investigation[ORM.INVESTIGATION_SRT_CREATED]): continue sql = ''' UPDATE orm_investigation SET srt_updated = ?, srt_created = ? WHERE id = ?''' cur_write.execute(sql, (srt_updated, srt_created, investigation[ORM.INVESTIGATION_ID],)) is_change_count += 1 print("INVESTIGATION DATE FIX COUNT=%d/%d" % (is_change_count,i)) conn.commit() # VULNERABILITY DATE FIX COUNT=86585, real 1m2.969s = 1374 recs/sec if ('v' == scope) or ('all' == scope): cur.execute('SELECT * FROM orm_vulnerability') i = 0 is_change_count = 0 for vulnerability in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,vulnerability[ORM.VULNERABILITY_NAME]), end='') if (0 == i % 200): conn.commit() time.sleep(0.1) # give time for Sqlite to sync # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break srt_updated = _fix_datetime(vulnerability[ORM.VULNERABILITY_SRT_UPDATED],None) srt_created = _fix_datetime(vulnerability[ORM.VULNERABILITY_SRT_CREATED],None) if (not srt_updated) or (not srt_created): print("ERROR[%d]: bad date field at '%s', U=%s,C=%s" % (i,vulnerability[ORM.VULNERABILITY_ID],vulnerability[ORM.VULNERABILITY_SRT_UPDATED],vulnerability[ORM.VULNERABILITY_SRT_CREATED])) exit(1) if (srt_updated == vulnerability[ORM.VULNERABILITY_SRT_UPDATED]) and (srt_created == vulnerability[ORM.VULNERABILITY_SRT_CREATED]): continue sql = ''' UPDATE orm_vulnerability SET srt_updated = ?, srt_created = ? WHERE id = ?''' cur_write.execute(sql, (srt_updated, srt_created, vulnerability[ORM.VULNERABILITY_ID],)) is_change_count += 1 print("VULNERABILITY DATE FIX COUNT=%d/%d" % (is_change_count,i)) conn.commit() # CVE DATE FIX COUNT=86585, real 1m2.969s = 1374 recs/sec # NOTE: only ACK dates need fixing, received bad apha content from srtool_mitre if ('c' == scope) or ('all' == scope): cur.execute('SELECT * FROM orm_cve') i = 0 # Sparse updates is_change = False is_change_count = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') if (0 == i % 200) and is_change: conn.commit() time.sleep(0.1) # give time for Sqlite to sync is_change = False # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if is_change_count > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break is_change = False if cve[ORM.CVE_ACKNOWLEDGE_DATE]: acknowledge_date = _fix_datetime(cve[ORM.CVE_ACKNOWLEDGE_DATE],'alpha') # If the default 'alpha' happens, then date had bad format and must go away if ('alpha' == acknowledge_date) or (acknowledge_date != cve[ORM.CVE_ACKNOWLEDGE_DATE]): acknowledge_date = None is_change = True srt_updated = _fix_datetime(cve[ORM.CVE_SRT_UPDATED],None) srt_created = _fix_datetime(cve[ORM.CVE_SRT_CREATED],None) if (not srt_updated) or (not srt_created): print("ERROR[%d]: bad date field at '%s', U=%s,C=%s" % (i,cve[ORM.CVE_ID],cve[ORM.CVE_SRT_UPDATED],cve[ORM.CVE_SRT_CREATED])) exit(1) if (srt_updated != cve[ORM.CVE_SRT_UPDATED]) or (srt_created != cve[ORM.CVE_SRT_CREATED]): is_change = True # Anything to do? if not is_change: continue is_change_count += 1 sql = ''' UPDATE orm_cve SET srt_updated = ?, srt_created = ?, acknowledge_date = ? WHERE id = ?''' cur_write.execute(sql, (srt_updated, srt_created, acknowledge_date, cve[ORM.CVE_ID],)) is_change_count += 1 print("CVE DATE FIX COUNT=%d/%d" % (is_change_count,i)) conn.commit() # Fix CVE History if scope in ('ch','all','history'): cur.execute('SELECT * FROM orm_cvehistory') i = 0 # Sparse updates is_change = False is_change_count = 0 for cve_history in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: \r' % (i), end='') if (0 == i % 200) and is_change: conn.commit() time.sleep(0.1) # give time for Sqlite to sync is_change = False # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if is_change_count > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break updated,history_date = _fix_date(cve_history[ORM.CVEHISTORY_DATE],'') if not updated: continue is_change = True sql = ''' UPDATE orm_cvehistory SET date = ? WHERE id = ?''' cur_write.execute(sql, (history_date, cve_history[ORM.CVEHISTORY_ID],)) is_change_count += 1 # Commit all remaining changes if is_change: conn.commit() print("CVE HISTORY DATE FIX COUNT=%d/%d" % (is_change_count,i)) # Fix Vulnerability History if scope in ('vh','all','history'): cur.execute('SELECT * FROM orm_vulnerabilityhistory') i = 0 # Sparse updates is_change = False is_change_count = 0 for vulnerabilityhistory in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: \r' % (i), end='') if (0 == i % 200) and is_change: conn.commit() time.sleep(0.1) # give time for Sqlite to sync is_change = False # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if is_change_count > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break updated,history_date = _fix_date(vulnerabilityhistory[ORM.VULNERABILITYHISTORY_DATE],'') if not updated: continue is_change = True sql = ''' UPDATE orm_vulnerabilityhistory SET date = ? WHERE id = ?''' cur_write.execute(sql, (history_date, vulnerabilityhistory[ORM.VULNERABILITYHISTORY_ID],)) is_change_count += 1 # Commit all remaining changes if is_change: conn.commit() print("VULNERABILITY HISTORY DATE FIX COUNT=%d/%d" % (is_change_count,i)) # Fix Investigation History if scope in ('ih','all','history'): cur.execute('SELECT * FROM orm_investigationhistory') i = 0 # Sparse updates is_change = False is_change_count = 0 for investigation_history in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: \r' % (i), end='') if (0 == i % 200) and is_change: conn.commit() time.sleep(0.1) # give time for Sqlite to sync is_change = False # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if is_change_count > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break updated,history_date = _fix_date(investigation_history[ORM.INVESTIGATIONHISTORY_DATE],'') if not updated: continue is_change = True sql = ''' UPDATE orm_investigationhistory SET date = ? WHERE id = ?''' cur_write.execute(sql, (history_date, investigation_history[ORM.INVESTIGATIONHISTORY_ID],)) is_change_count += 1 # Commit all remaining changes if is_change: conn.commit() print("INVESTIGATION HISTORY DATE FIX COUNT=%d/%d" % (is_change_count,i)) ################################# # fixup fix_cve_srt_create # # Reset CVE srt_create to NIST release dates def fix_reset_nist_to_create(cve_prefix): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() def date_nist2srt(nist_date,default,cve_name,i): if not nist_date or (4 > len(nist_date)): return default try: return(datetime.strptime(nist_date, '%Y-%m-%d')) except Exception as e: print("\n\ndate_nist2srt:%s,%s,%s,%s" % (cve_name,e,cve_name,i)) exit(1) return default cur.execute('SELECT * FROM orm_cve WHERE name LIKE "'+cve_prefix+'%"') i = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') if (0 == i % 200): conn.commit() print('') # Development/debug support if cmd_skip and (i < cmd_skip): continue if cmd_count and ((i - cmd_skip) > cmd_count): break nist_released = date_nist2srt(cve[ORM.CVE_PUBLISHEDDATE],cve[ORM.CVE_SRT_CREATED],cve[ORM.CVE_NAME],i) nist_modified = date_nist2srt(cve[ORM.CVE_LASTMODIFIEDDATE],cve[ORM.CVE_SRT_UPDATED],cve[ORM.CVE_NAME],i) sql = ''' UPDATE orm_cve SET srt_created = ?, srt_updated = ? WHERE id = ?''' cur_write.execute(sql, (nist_released, nist_modified, cve[ORM.CVE_ID],)) print("CVE DATE FIX COUNT=%d" % i) conn.commit() ################################# # fixup fix_missing_create_dates # # Reset CVE None creation dates to 2019-01-01, out of the way of reports def fix_missing_create_dates(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() fix_date = datetime.strptime('Jan 1 2019', '%b %d %Y') fix_count = 0 cur.execute('SELECT * FROM orm_cve') i = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') if (0 == i % 200): # conn.commit() #print('') pass # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break fix = False if not cve[ORM.CVE_SRT_CREATED] or (0 > cve[ORM.CVE_SRT_CREATED].find(':')): srt_created = fix_date fix = True else: srt_created = cve[ORM.CVE_SRT_CREATED] #srt_created = datetime.strptime(cve[ORM.CVE_SRT_CREATED],'%Y-%m-%d') if not cve[ORM.CVE_SRT_UPDATED] or (0 > cve[ORM.CVE_SRT_UPDATED].find(':')): srt_updated = fix_date fix = True else: srt_updated = cve[ORM.CVE_SRT_UPDATED] #srt_updated = datetime.strptime(cve[ORM.CVE_SRT_UPDATED],'%Y-%m-%d') if fix: sql = ''' UPDATE orm_cve SET srt_created = ?, srt_updated = ? WHERE id = ?''' cur_write.execute(sql, (srt_created, srt_updated, cve[ORM.CVE_ID],)) fix_count += 1 print("CVE DATE FIX COUNT=%d of %d" % (fix_count,i)) conn.commit() ################################# # fixup fix_public_reserved # # Reset CVE 'New-Reserved' if now public from NIST def fix_public_reserved(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_write = conn.cursor() fix_count = 0 cur.execute('SELECT * FROM orm_cve WHERE status = "%s"' % ORM.STATUS_NEW_RESERVED) i = 0 for cve in cur: i += 1 # Progress indicator support if 0 == i % 10: print('%05d: %20s %d\r' % (i,cve[ORM.CVE_NAME],cve[ORM.CVE_STATUS]), end='') if (0 == i % 200): conn.commit() #print('') pass # Development/debug support if cmd_skip: if i < cmd_skip: continue if cmd_count: if (i - cmd_skip) > cmd_count: print("Count return: %s,%s" % (i,cmd_count)) break if cve[ORM.CVE_CVSSV3_BASESCORE] or cve[ORM.CVE_CVSSV2_BASESCORE]: sql = ''' UPDATE orm_cve SET status = ? WHERE id = ?''' cur_write.execute(sql, (ORM.STATUS_NEW, cve[ORM.CVE_ID],)) fix_count += 1 print("CVE DATE FIX COUNT=%d of %d" % (fix_count,i)) conn.commit() ################################# # fix_remove_bulk_cve_history # # Remove a specific/accidental set of bulk CVE history updates intended to be background def fix_foo(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_cve = conn.cursor() cur_del = conn.cursor() fix_count = 0 print("FOO=%s\n\n" % ORM.STATUS_NEW_RESERVED) cur.execute('SELECT * FROM orm_cvehistory WHERE date LIKE "2019-03-2%"') i = 0 for cvehistory in cur: i += 1 # Progress indicator support if 9 == i % 10: # print('%05d: %20s %s \r' % (i,cvehistory[ORM.CVEHISTORY_COMMENT],cvehistory[ORM.CVEHISTORY_DATE]), end='') pass if (0 == i % 200): # conn.commit() #print('') pass # Development/debug support if cmd_skip and (i < cmd_skip): continue if cmd_count and ((i - cmd_skip) > cmd_count): break if not (cvehistory[ORM.CVEHISTORY_DATE] in ('2019-03-28','2019-03-27')): continue if not (cvehistory[ORM.CVEHISTORY_COMMENT].startswith("UPDATE(CVE):")): continue cur_cve.execute('SELECT * FROM orm_cve WHERE id = "%s"' % cvehistory[ORM.CVEHISTORY_CVE_ID]) cve = cur_cve.fetchone() if not (cve[ORM.CVE_NAME].startswith("CVE-200")): continue if 19 == fix_count % 20: print("%4d) CVE=%s,CH_Comment=%s,CH_Date=%s" % (fix_count,cve[ORM.CVE_NAME],cvehistory[ORM.CVEHISTORY_COMMENT],cvehistory[ORM.CVEHISTORY_DATE])) mydata = cur_del.execute("DELETE FROM orm_cvehistory WHERE id=?", (cvehistory[ORM.CVEHISTORY_ID],)) fix_count += 1 print("CVE DATE FIX COUNT=%d of %d" % (fix_count,i)) conn.commit() ################################# # fix_defects_to_products # # def fix_defects_to_products(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_cve = conn.cursor() cur_del = conn.cursor() fix_count = 0 # Find all products products = {} cur.execute('SELECT * FROM orm_product') for product in cur: id = product[ORM.PRODUCT_ID] name = "%s %s %s" % (product[ORM.PRODUCT_NAME],product[ORM.PRODUCT_VERSION],product[ORM.PRODUCT_PROFILE]) products[id] = name print("[%2d] %s" % (id,name)) # Test product field for all defects cur.execute('SELECT * FROM orm_defect') i = 0 for defect in cur: i += 1 # Progress indicator support if 99 == i % 100: print('%05d: %-20s\r' % (i,defect[ORM.DEFECT_NAME]), end='') pass if (0 == i % 200): # conn.commit() #print('') pass # Development/debug support if cmd_skip and (i < cmd_skip): continue if cmd_count and ((i - cmd_skip) > cmd_count): break product_id = defect[ORM.DEFECT_PRODUCT_ID] if not product_id in products: print("ERROR:[%5d] %-20s => %s" % (defect[ORM.DEFECT_ID],defect[ORM.DEFECT_NAME],product_id)) # print("CVE DATE FIX COUNT=%d of %d" % (fix_count,i)) conn.commit() ################################# # fix_bad_mitre_init # # # Fix MITRE reserved CVEs that were mistakenly set at "New" instead of # "New-Reserved" due to column ordering issue in the MITRE "Init" routine. # def fix_bad_mitre_init(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_ds = conn.cursor() cur_cve = conn.cursor() cur_del = conn.cursor() new_count = 0 mitre_count = 0 cve_name = '' nist_source_list = [] # Find NIST data sources cur.execute('SELECT * FROM orm_datasource WHERE source = "nist"') for i,ds in enumerate(cur): nist_source_list.append(ds[ORM.DATASOURCE_ID]) print('NIST DataSource List=[%s]' % nist_source_list) mitre_source_list = [] # Find MITRE data sources cur.execute('SELECT * FROM orm_datasource WHERE source = "mitre"') for i,ds in enumerate(cur): mitre_source_list.append(ds[ORM.DATASOURCE_ID]) print('MITRE DataSource List=[%s]' % mitre_source_list) # Find all bad MITRE reserved CVEs cur.execute('SELECT * FROM orm_cve WHERE description = "" AND status = %d' % ORM.STATUS_NEW) # cur.execute('SELECT * FROM orm_cve WHERE description = ""') for i,cve in enumerate(cur): new_count += 1 cur_ds.execute('SELECT * FROM orm_cvesource WHERE cve_id = %d' % cve[ORM.CVE_ID]) is_mitre = False is_nist = False for cvesource in cur_ds: if cvesource[ORM.CVESOURCE_DATASOURCE_ID] in mitre_source_list: is_mitre = True if cvesource[ORM.CVESOURCE_DATASOURCE_ID] in nist_source_list: is_nist = True if is_mitre and not is_nist: mitre_count += 1 cve_name = cve[ORM.CVE_NAME] if force: sql = ''' UPDATE orm_cve SET status = ? WHERE id = ?''' cur_cve.execute(sql, (ORM.STATUS_NEW_RESERVED,cve[ORM.CVE_ID],)) conn.commit() # Progress indicator support if 19 == i % 20: print('%05d: %-20s\r' % (i,cve[ORM.CVE_NAME]), end='') pass if (0 == i % 200): # conn.commit() print('') pass # Development/debug support if cmd_skip and (i < cmd_skip): continue if cmd_count and ((i - cmd_skip) > cmd_count): break print("\nCVE NEW_COUNT=%d, mitre=%d, name=%s, database=%s" % (new_count,mitre_count,cve_name,srtDbName)) # conn.commit() # # Fix MITRE CVEs that are missing a description in the top level CVE # records due to column ordering issue in the MITRE "Init" routine. # def foo_fix_bad_mitre_init(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_ds = conn.cursor() cur_cve = conn.cursor() cur_del = conn.cursor() fix_count = 0 reserved_count = 0 mitre_count = 0 nosource_count = 0 mitre_source_list = [] mitre_lookup = {} # Find MITRE data sources cur.execute('SELECT * FROM orm_datasource WHERE source = "mitre"') for i,ds in enumerate(cur): mitre_source_list.append(ds[ORM.DATASOURCE_ID]) mitre_lookup[ds[ORM.DATASOURCE_ID]] = ds[ORM.DATASOURCE_LOOKUP] print('MITRE DataSource List=[%s]' % mitre_source_list) # Find all bad MITRE reserved CVEs cur.execute('SELECT * FROM orm_cve WHERE description = ""') for i,cve in enumerate(cur): fix_count += 1 # reserved_pos = cve[ORM.CVE_DESCRIPTION].find('** RESERVED **') # if (0 <= reserved_pos) and (20 > reserved_pos): # reserved_count += 1 if ORM.STATUS_NEW == cve[ORM.CVE_STATUS]: reserved_count += 1 cur_ds.execute('SELECT * FROM orm_cvesource WHERE cve_id = %d' % cve[ORM.CVE_ID]) is_mitre = False mitre_ds = 0 ds_list = [] ds_count = 0 for cvesource in cur_ds: ds_count += 1 if cvesource[ORM.CVESOURCE_DATASOURCE_ID] in mitre_source_list: is_mitre = True mitre_ds = cvesource[ORM.CVESOURCE_DATASOURCE_ID] break ds_list.append(cvesource[ORM.CVESOURCE_DATASOURCE_ID]) if True: print('%05d: %-20s, SourceList=%s' % (i,cve[ORM.CVE_NAME],ds_list)) if False: if is_mitre: mitre_count += 1 lookup_command = mitre_lookup[ cvesource[ORM.CVESOURCE_DATASOURCE_ID] ].replace('%command%','--cve-detail=%s' % cve[ORM.CVE_NAME]) result_returncode,result_stdout,result_stderr = execute_process(lookup_command.split(' ')) if 0 != result_returncode: print("ERROR_LOOKUP:%s" % lookup_command) return(1) description = '' for line in result_stdout.decode("utf-8").splitlines(): try: name = line[:line.index('=')] value = line[line.index('=')+1:].replace("[EOL]","\n") except: continue if name == 'description': description = value break if description: # print("%s='%s'" % (cve[ORM.CVE_NAME],description)) sql = ''' UPDATE orm_cve SET description = ? WHERE id = ?''' cur_ds.execute(sql, (description,cve[ORM.CVE_ID],)) # conn.commit() # return(0) elif 0 == ds_count: nosource_count += 1 else: print('%05d: %-20s, SourceList=%s' % (i,cve[ORM.CVE_NAME],ds_list)) # Progress indicator support if 19 == i % 100: print('%05d: %-20s\r' % (i,cve[ORM.CVE_NAME]), end='') pass if (0 == i % 200): # conn.commit() #print('') pass # Development/debug support if cmd_skip and (i < cmd_skip): continue if cmd_count and ((i - cmd_skip) > cmd_count): break print("CVE RESERVED COUNT=%d of %d, mitre=%d, no_source=%d" % (reserved_count,fix_count,mitre_count,nosource_count)) # conn.commit() # # Fix CVE records with missing 'cvssV2_severity' values in the top-level CVE records, due to # CVE imports before a patch was sent upstream # # The NIST Modified list is processed first. If any of its CVEs are found in a regular # list, that CVE is skipped since it was preempted # def fix_severity(datasource_list): conn = sqlite3.connect(srtDbName) cur_ds = conn.cursor() cur_cs = conn.cursor() cur_cve = conn.cursor() cur_del = conn.cursor() cve_count = 0 fix_count = 0 nist_ds_list = {} modified_cve_list = [] DATA_MAP_V3_Score = 0 DATA_MAP_V3_Severity = 1 DATA_MAP_V2_Score = 2 DATA_MAP_V2_Severity = 3 # Allow "MOD" as shorthand for the modification datasource datasource_list = datasource_list.replace('MOD','NIST Modified Data') # # Gather the NIST data source list # cur_ds.execute('SELECT * FROM orm_datasource WHERE source = "nist" ORDER BY key DESC;') for i,ds in enumerate(cur_ds): nist_ds_list[ds[ORM.DATASOURCE_ID]] = ds[ORM.DATASOURCE_DESCRIPTION] # # Iterate over the NIST data sources # cur_ds.execute('SELECT * FROM orm_datasource WHERE source = "nist" ORDER BY key DESC;') for i,ds in enumerate(cur_ds): # Development/debug support if cmd_count and ((cve_count - cmd_skip) > cmd_count): break if ds[ORM.DATASOURCE_DESCRIPTION] in ['NIST Common Weakness Enumeration Data']: continue elif "ALL" == datasource_list: pass elif not ds[ORM.DATASOURCE_DESCRIPTION] in datasource_list.split(','): continue print("NIST Source:%s" % ds[ORM.DATASOURCE_DESCRIPTION]) is_modified_list = ds[ORM.DATASOURCE_DESCRIPTION] == 'NIST Modified Data' # Scan the NIST datasource file and extract required values into a map # (bin/nist/srtool_nist.py --download-only --source='NIST 2002' --file=data/nvdcve-1.1-2002.json --url-file=nvdcve-1.1-2002.json.gz --url-meta=nvdcve-1.1-2002.meta) cve_source_file = '' for param in ds[ORM.DATASOURCE_LOOKUP].split(' '): if param.startswith('--file='): cve_source_file = param.replace('--file=','') print(" File:%s" % cve_source_file) break nist_data_map = {} nist_file = os.path.join(srtool_basepath,cve_source_file) # # Gather the V3/V2 status of all the CVEs in this NIST data sources # try: if not os.path.isfile(nist_file): print("ERROR: no such file '%s'" % nist_file) exit(1) f = open(nist_file, 'r') source_dct = json.load(f) for item in source_dct["CVE_Items"]: if not 'cve' in item: continue if not 'CVE_data_meta' in item['cve']: continue if not 'ID' in item['cve']['CVE_data_meta']: continue cve_name = item['cve']['CVE_data_meta']['ID'] # Is this the NIST Modified list? if is_modified_list: # Add CVE name to Modified list modified_cve_list.append(cve_name) elif cve_name in modified_cve_list: # Skip if already process by Modified list continue # # Debugging support # if cve_name != "CVE-2016-0887": #"CVE-2020-7470","CVE-2019-15031" # continue cvssV3_baseScore = '' cvssV3_baseSeverity = '' cvssV2_baseScore = '' cvssV2_severity = '' if ('impact' in item) and ('baseMetricV3' in item['impact']): cvssV3_baseScore = "%.1f" % float(item['impact']['baseMetricV3']['cvssV3']['baseScore']) cvssV3_baseSeverity = item['impact']['baseMetricV3']['cvssV3']['baseSeverity'] if ('impact' in item) and ('baseMetricV2' in item['impact']): cvssV2_baseScore = "%.1f" % float(item['impact']['baseMetricV2']['cvssV2']['baseScore']) cvssV2_severity = item['impact']['baseMetricV2']['severity'] nist_data_map[cve_name] = [cvssV3_baseScore,cvssV3_baseSeverity,cvssV2_baseScore,cvssV2_severity] except Exception as e: print("ERROR:%s (%s)" % (e,item['impact']['baseMetricV3'])) return # # Update the V3/V2 status for all found CVE records in this datasource # for cve_name in nist_data_map: cur_cve.execute('SELECT * FROM orm_cve WHERE name = "%s"' % cve_name) cve = cur_cve.fetchone() if not cve: print("WARNING: MISSING CVE in orm_cvesource : %s" % cve_name) continue cve_name = cve[ORM.CVE_NAME] if cve_name in nist_data_map: fix_count += 1 if (nist_data_map[cve_name][DATA_MAP_V3_Score] != cve[ORM.CVE_CVSSV3_BASESCORE]) or (nist_data_map[cve_name][DATA_MAP_V3_Severity] != cve[ORM.CVE_CVSSV3_BASESEVERITY]) or \ (nist_data_map[cve_name][DATA_MAP_V2_Score] != cve[ORM.CVE_CVSSV2_BASESCORE]) or (nist_data_map[cve_name][DATA_MAP_V2_Severity] != cve[ORM.CVE_CVSSV2_SEVERITY ]): print("CHANGE: %s V3(%s to %s,%s to %s)V2(%s to %s,%s to %s) (%s,%s)" % ( cve_name, cve[ORM.CVE_CVSSV3_BASESCORE],nist_data_map[cve_name][DATA_MAP_V3_Score],cve[ORM.CVE_CVSSV3_BASESEVERITY],nist_data_map[cve_name][DATA_MAP_V3_Severity], cve[ORM.CVE_CVSSV2_BASESCORE],nist_data_map[cve_name][DATA_MAP_V2_Score],cve[ORM.CVE_CVSSV2_SEVERITY ],nist_data_map[cve_name][DATA_MAP_V2_Severity], ORM.get_orm_string(cve[ORM.CVE_STATUS],ORM.STATUS_STR),cve[ORM.CVE_COMMENTS], )) if force: sql = ''' UPDATE orm_cve SET cvssV3_baseScore = ?, cvssV3_baseSeverity = ?, cvssV2_baseScore = ?, cvssV2_severity = ? WHERE id = ?''' cur_cve.execute(sql, (nist_data_map[cve_name][DATA_MAP_V3_Score],nist_data_map[cve_name][DATA_MAP_V3_Severity],nist_data_map[cve_name][DATA_MAP_V2_Score],nist_data_map[cve_name][DATA_MAP_V2_Severity],cve[ORM.CVE_ID],)) # print('%05d: %-20s = %-20s' % (j,cve_name,nist_data_map[cve_name])) else: print("ERROR:CVE_NAME '%s' NOT MAPPED" % cve_name) # # Repair the data source mappings # * Add missing NIST links # * Replace old NIST links with found links (e.g. Modified datasource preempts regular datasources) # found_mapping = False cur_cs.execute('SELECT * FROM orm_cvesource WHERE cve_id = %d' % cve[ORM.CVE_ID]) for j,cve2ds in enumerate(cur_cs): if cve2ds[ORM.CVESOURCE_DATASOURCE_ID] in nist_ds_list: # Do we have an obsolete NIST mapping? if cve2ds[ORM.CVESOURCE_DATASOURCE_ID] != ds[ORM.DATASOURCE_ID]: # Delete old mapping print("Delete old mapping %s,%s" % (cve_name,nist_ds_list[cve2ds[ORM.CVESOURCE_DATASOURCE_ID]])) if force: sql = 'DELETE FROM orm_cvesource WHERE id=?' cur_del.execute(sql, (cve2ds[ORM.CVESOURCE_ID],)) else: # We are good to go found_mapping = True # Add if missing or deleted as obsolete if not found_mapping: print("Insert new mapping %s,%s" % (cve_name,nist_ds_list[ds[ORM.DATASOURCE_ID]])) if force: sql = ''' INSERT into orm_cvesource (cve_id, datasource_id) VALUES (?, ?)''' cur_cs.execute(sql, (cve[ORM.CVE_ID],ds[ORM.DATASOURCE_ID],)) # Development/debug support cve_count += 1 if cmd_skip and (cve_count < cmd_skip): continue if cmd_count and ((cve_count - cmd_skip) > cmd_count): break # Progress indicator support if (0 == cve_count % 1000): print('%05d: %-20s\r' % (cve_count,cve_name), end='') if force: conn.commit() print('') pass print("CVE COUNT=%d, fix_count=%d" % (cve_count,fix_count)) if force: conn.commit() # # Trim the V3/V2 scores to one decimal place, in line with NIST public pages # def fix_trim_cve_scores(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_wr = conn.cursor() def fixscore(score): if not score: return '' return '%02.1f' % float(score) cve_count = 0 fix_count = 0 cur.execute('SELECT * FROM orm_cve') for i,cve in enumerate(cur): if 0 == i % 100: print("%4d) C=%-30s\r" % (i,cve[ORM.CVE_NAME]), end='') new_v3score = fixscore(cve[ORM.CVE_CVSSV3_BASESCORE]) new_v2score = fixscore(cve[ORM.CVE_CVSSV2_BASESCORE]) if (new_v3score != cve[ORM.CVE_CVSSV3_BASESCORE]) or (new_v2score != cve[ORM.CVE_CVSSV2_BASESCORE]): fix_count += 1 if verbose: print("CHANGE:%s:%s to %s,%s to %s" % (cve[ORM.CVE_NAME],cve[ORM.CVE_CVSSV3_BASESCORE],new_v3score,cve[ORM.CVE_CVSSV2_BASESCORE],new_v2score)) if force: sql = ''' UPDATE orm_cve SET cvssV3_baseScore = ?, cvssV2_baseScore = ? WHERE id = ?''' cur_wr.execute(sql, (new_v3score,new_v2score,cve[ORM.CVE_ID],)) # Development/debug support cve_count += 1 if cmd_skip and (cve_count < cmd_skip): continue if cmd_count and ((cve_count - cmd_skip) > cmd_count): break # Progress indicator support if (0 == cve_count % 1000): print('%05d: %-20s\r' % (cve_count,cve[ORM.CVE_NAME]), end='') if force: conn.commit() print('') pass if force: conn.commit() print("CVE COUNT=%d, fix_count=%d" % (cve_count,fix_count)) # Sample code that does a CVE lookup data fetch and CVE update #def example_datasource_lookup(cve,nist_ds,cvesource,cur): # if force: # if nist_ds: # lookup_command = nist_lookup[ cvesource[ORM.CVESOURCE_DATASOURCE_ID] ].replace('%command%','--cve-detail=%s' % cve[ORM.CVE_NAME]) # result_returncode,result_stdout,result_stderr = execute_process(lookup_command.split(' ')) # if 0 != result_returncode: # print("ERROR_LOOKUP:%s" % lookup_command) # return(1) # cvssV2_severity = '' # for line in result_stdout.decode("utf-8").splitlines(): # try: # name = line[:line.index('=')] # value = line[line.index('=')+1:].replace("[EOL]","\n") # except: # continue # if name == 'cvssV2_severity': # cvssV2_severity = value # break # if cvssV2_severity: # fix_count += 1 # sql = ''' UPDATE orm_cve # SET cvssV2_severity = ? # WHERE id = ?''' # cur.execute(sql, (cvssV2_severity,cve[ORM.CVE_ID],)) # print('%05d: %-20s = %-20s' % (i,cve[ORM.CVE_NAME],cvssV2_severity)) ## return(0) ################################# # find_multiple_defects # def find_multiple_defects(): conn = sqlite3.connect(srtDbName) cur_i2d = conn.cursor() cur_inv = conn.cursor() cur_def = conn.cursor() cur_inv.execute('SELECT * FROM orm_investigation') count = 0 for i,investigation in enumerate(cur_inv): if 0 == i % 100: print("%4d) V=%-30s\r" % (i,investigation[ORM.VULNERABILITY_NAME]), end='') cur_i2d.execute('SELECT * FROM orm_investigationtodefect WHERE investigation_id = "%s"' % investigation[ORM.INVESTIGATION_ID]) i2d_list = cur_i2d.fetchall() if 1 < len(i2d_list): count += 1 for k,i2d in enumerate(i2d_list): cur_def.execute('SELECT * FROM orm_defect WHERE id = "%s"' % i2d[ORM.INVESTIGATIONTODEFECT_DEFECT_ID]) defect = cur_def.fetchone() if defect[ORM.DEFECT_NAME].startswith("LIN10"): if 0 == k: print("[%02d] Multiple defects for investigation '%s':" % (count,investigation[ORM.INVESTIGATION_NAME])) print(" [%02d] %s: %s (%s)" % (k+1,defect[ORM.DEFECT_NAME],defect[ORM.DEFECT_SUMMARY],ORM.get_orm_string(defect[ORM.DEFECT_RESOLUTION],ORM.DEFECT_RESOLUTION_STR))) conn.close() ################################# # find_duplicate_names # def find_duplicate_names(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur.execute('SELECT * FROM orm_cve') cve_dict = {} for i,cve in enumerate(cur): if 0 == i % 100: print("%4d) C=%-30s\r" % (i,cve[ORM.CVE_NAME]), end='') if not cve[ORM.CVE_NAME] in cve_dict: cve_dict[cve[ORM.CVE_NAME]] = cve[ORM.CVE_ID] else: print("\nERROR:Multiple cve names '%s'" % cve[ORM.CVE_NAME]) print(" a) id=%d" % cve_dict[cve[ORM.CVE_NAME]]) print(" b) id=%d" % cve[ORM.CVE_ID]) cve_dict = {} print('') cur.execute('SELECT * FROM orm_vulnerability') vul_dict = {} for i,vulnerability in enumerate(cur): if 0 == i % 100: print("%4d) V=%-30s\r" % (i,vulnerability[ORM.VULNERABILITY_NAME]), end='') if not vulnerability[ORM.VULNERABILITY_NAME] in vul_dict: vul_dict[vulnerability[ORM.VULNERABILITY_NAME]] = vulnerability[ORM.VULNERABILITY_ID] else: print("\nERROR:Multiple vulnerability names '%s'" % vulnerability[ORM.VULNERABILITY_NAME]) print(" a) id=%d" % vul_dict[vulnerability[ORM.VULNERABILITY_NAME]]) print(" b) id=%d" % vulnerability[ORM.VULNERABILITY_ID]) vul_dict = {} print('') cur.execute('SELECT * FROM orm_investigation') inv_dict = {} for i,investigation in enumerate(cur): if 0 == i % 100: print("%4d) I=%-30s\r" % (i,investigation[ORM.INVESTIGATION_NAME]), end='') if not investigation[ORM.INVESTIGATION_NAME] in inv_dict: inv_dict[investigation[ORM.INVESTIGATION_NAME]] = investigation[ORM.INVESTIGATION_ID] else: print("\nERROR:Multiple investigation names '%s'" % investigation[ORM.INVESTIGATION_NAME]) print(" a) id=%d" % inv_dict[investigation[ORM.INVESTIGATION_NAME]]) print(" b) id=%d" % investigation[ORM.INVESTIGATION_ID]) inv_dict = {} print('') cur.execute('SELECT * FROM orm_defect') dev_dict = {} for i,defect in enumerate(cur): if 0 == i % 100: print("%4d) D=%-30s\r" % (i,defect[ORM.DEFECT_NAME]), end='') if not defect[ORM.DEFECT_NAME] in dev_dict: dev_dict[defect[ORM.DEFECT_NAME]] = defect[ORM.DEFECT_ID] else: print("\nERROR:Multiple defect names '%s'" % defect[ORM.DEFECT_NAME]) print(" a) id=%d" % dev_dict[defect[ORM.DEFECT_NAME]]) print(" b) id=%d" % defect[ORM.DEFECT_ID]) dev_dict = {} print('') conn.close() ################################# # find_bad_links # def find_bad_links(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_del = conn.cursor() # print('\n=== CVE Source Check ===\n') # # Find the data source mapping cur.execute('SELECT * FROM orm_datasource;') datasource_map = {} for datasource in cur: # DataSource Map is [cve_file,ds_desc,ds_lastmodifieddate,ds_lastupdateddate] datasource_map[datasource[ORM.DATASOURCE_ID]] = datasource[ORM.DATASOURCE_DESCRIPTION] cur.execute('SELECT * FROM orm_cvesource') is_change = False for i,cs in enumerate(cur): cveid = cs[ORM.CVESOURCE_CVE_ID] srcid = cs[ORM.CVESOURCE_DATASOURCE_ID] if 0 == i % 100: print("%4d) CVE=%6d,SRC=%6d\r" % (cs[ORM.CVESOURCE_ID],cveid,srcid), end='') error = False if (1 > cveid): error = True if (1 > srcid): error = True if error: print("ERROR: [%4d] CVE=%6d,SRC=%6d (%s)" % (cs[ORM.CVESOURCE_ID],cveid,srcid,datasource_map[srcid])) if force: sql = 'DELETE FROM orm_cvesource WHERE id=?' cur_del.execute(sql, (cs[ORM.CVESOURCE_ID],)) is_change = True print('') if is_change: conn.commit() # print('\n=== Defect to Product Check ===\n') # # Find all products products = {} cur.execute('SELECT * FROM orm_product') for product in cur: id = product[ORM.PRODUCT_ID] name = "%s %s %s" % (product[ORM.PRODUCT_NAME],product[ORM.PRODUCT_VERSION],product[ORM.PRODUCT_PROFILE]) products[id] = name print("[%2d] %s" % (id,name)) # Test product field for all defects cur.execute('SELECT * FROM orm_defect') i = 0 for defect in cur: i += 1 # Progress indicator support if 99 == i % 100: print('%05d: %-20s\r' % (i,defect[ORM.DEFECT_NAME]), end='') pass if (0 == i % 200): # conn.commit() #print('') pass # Development/debug support if cmd_skip and (i < cmd_skip): continue if cmd_count and ((i - cmd_skip) > cmd_count): break product_id = defect[ORM.DEFECT_PRODUCT_ID] if not product_id in products: print("ERROR:[%5d] %-20s => %s" % (defect[ORM.DEFECT_ID],defect[ORM.DEFECT_NAME],product_id)) conn.close() ################################# # find_empty_status # def find_empty_status(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_del = conn.cursor() # print('\n=== CVE Empty Status Check ===\n') # cur.execute('SELECT * FROM orm_cve') empty_count = 0 date_count = 0 other_count = 0 total = 0 for i,cve in enumerate(cur): total += 1 if not cve[ORM.CVE_STATUS]: empty_count += 1 elif '-' in cve[ORM.CVE_STATUS]: date_count += 1 else: try: value = int(cve[ORM.CVE_STATUS]) except: other_count += 1 print("STATUS: Empty=%d, Date=%d, OtherBad=%d, total=%d of %d" % (empty_count,date_count,other_count,empty_count+date_count+other_count,total)) ################################# # fix_bad_new # # Remove the 'score_date' value to repair the migration to '0006_reconcile', allowing # the field for new CVEs to be regenerated. The schema for this field is 'models.DateField' # but the scoring method in # "srtool_common --score-new-cves" was setting an obsolete # date_time value. That crashes Django-2.2 (but not Django-1.11). # def fix_bad_new(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() cur_fix = conn.cursor() # print('\n=== CVE fix_bad_new Check ===\n') # cur.execute('SELECT * FROM orm_cve WHERE status = %d' % ORM.STATUS_NEW) for i,cve in enumerate(cur): for j,item in enumerate(cve): print("%s\t" % (item), end='') if force: sql = ''' UPDATE orm_cve SET score_date = ? WHERE id = ?''' cur_fix.execute(sql, (None, cve[ORM.CVE_ID],)) print("") conn.commit() ################################# # main loop # def main(argv): global verbose global cmd_skip global cmd_count global force global srtDbName # setup parser = argparse.ArgumentParser(description='srtool.py: manage the SRTool database') parser.add_argument('--sources', '-s', nargs='?', const='display', help='SRTool Sources') parser.add_argument('--reset-sources', '-r', action='store_const', const='reset_sources', dest='command', help='Reset SRTool Sources') parser.add_argument('--settings', '-S', action='store_const', const='settings', dest='command', help='Show the SRT Settings') parser.add_argument('--remove-app-sources', dest='remove_app_sources', help='Remove data sources for a previous app') parser.add_argument('--fix-name-sort', action='store_const', const='fix_name_sort', dest='command', help='Recalulate the CVE name sort values') parser.add_argument('--fix-cve-recommend', action='store_const', const='fix_cve_recommend', dest='command', help='Fix the empty CVE recommend values') parser.add_argument('--fix-new-reserved', action='store_const', const='fix_new_reserved', dest='command', help='Reset new reserved CVEs to NEW_RESERVED') parser.add_argument('--fix-new-tags', action='store_const', const='fix_new_tags', dest='command', help='Reset new cve.tags') parser.add_argument('--fix-srt-datetime', dest='fix_srt_datetime', help='Fix SRT dates to datetimes [all|c|v|i|d|history|ch|vh|ih|dh]') parser.add_argument('--fix-reset-nist-to-create', dest='fix_reset_nist_to_create', help='Bulk reset CVE [prefix*] srt_create dates to NIST release dates') parser.add_argument('--fix-missing-create-dates', action='store_const', const='fix_missing_create_dates', dest='command', help='Reset CVE srt_create dates to NIST release dates') parser.add_argument('--fix-public-reserved', action='store_const', const='fix_public_reserved', dest='command', help='Reset CVE NEW_RESERVED if now public') parser.add_argument('--fix-remove-bulk-cve-history', action='store_const', const='fix_remove_bulk_cve_history', dest='command', help='foo') parser.add_argument('--fix-bad-mitre-init', action='store_const', const='fix_bad_mitre_init', dest='command', help='foo') parser.add_argument('--fix-bad-new', action='store_const', const='fix_bad_new', dest='command', help='foo') parser.add_argument('--find-empty-status', action='store_const', const='find_empty_status', dest='command', help='foo') parser.add_argument('--fix-severity', dest='fix_severity', help='Fix bad score/severity values, broken cve source links') parser.add_argument('--fix-trim-cve-scores', action='store_const', const='fix_trim_cve_scores', dest='command', help='Trim V3/V2 scores to one decimal place standard') parser.add_argument('--find-multiple-defects', action='store_const', const='find_multiple_defects', dest='command', help='foo') parser.add_argument('--find-duplicate-names', action='store_const', const='find_duplicate_names', dest='command', help='foo') parser.add_argument('--fix-defects-to-products', action='store_const', const='fix_defects_to_products', dest='command', help='foo') parser.add_argument('--find-bad-links', action='store_const', const='find_bad_links', dest='command', help='Find bad links, e.g. "orm_cvesource" (add "-f" to fix)') parser.add_argument('--database', '-D', dest='database', help='Selected database file') parser.add_argument('--force', '-f', action='store_true', dest='force', help='Force the update') parser.add_argument('--update-skip-history', '-H', action='store_true', dest='update_skip_history', help='Skip history updates') parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') parser.add_argument('--skip', dest='skip', help='Debugging: skip record count') parser.add_argument('--count', dest='count', help='Debugging: short run record count') args = parser.parse_args() master_log = open(os.path.join(srtool_basepath, "update_logs/master_log.txt"), "a") verbose = args.verbose if None != args.skip: cmd_skip = int(args.skip) if None != args.count: cmd_count = int(args.count) force = args.force # Test for example the backup databases if args.database: srtDbName = args.database if args.sources: if args.sources.startswith('s'): sources("set") elif 0 <= args.sources.find('nist'): sources("reset_not_nist") elif args.sources.startswith('r'): sources("reset") elif args.sources.startswith('t'): sources("triage_keywords") else: sources("display") elif 'reset_sources' == args.command: sources('reset') elif 'settings' == args.command: settings() elif args.remove_app_sources: remove_app_sources(args.remove_app_sources) elif 'fix_name_sort' == args.command: fix_name_sort() elif 'fix_cve_recommend' == args.command: fix_cve_recommend() elif 'fix_new_reserved' == args.command: fix_new_reserved() elif 'fix_new_tags' == args.command: fix_new_tags() elif args.fix_srt_datetime: fix_srt_datetime(args.fix_srt_datetime) elif args.fix_reset_nist_to_create: fix_reset_nist_to_create(args.fix_reset_nist_to_create) elif 'fix_missing_create_dates' == args.command: fix_missing_create_dates() elif 'fix_public_reserved' == args.command: fix_public_reserved() elif 'fix_remove_bulk_cve_history' == args.command: fix_remove_bulk_cve_history() elif 'fix_defects_to_products' == args.command: fix_defects_to_products() elif 'fix_bad_mitre_init' == args.command: fix_bad_mitre_init() elif 'fix_bad_new' == args.command: fix_bad_new() elif args.fix_severity: fix_severity(args.fix_severity) elif 'fix_trim_cve_scores' == args.command: fix_trim_cve_scores() elif 'find_multiple_defects' == args.command: find_multiple_defects() elif 'find_duplicate_names' == args.command: find_duplicate_names() elif 'find_bad_links' == args.command: find_bad_links() elif 'find_empty_status' == args.command: find_empty_status() else: print("Command not found") master_log.close() if __name__ == '__main__': srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:])