diff options
Diffstat (limited to 'bin/common/srtool_utils.py')
| -rwxr-xr-x | bin/common/srtool_utils.py | 149 |
1 files changed, 148 insertions, 1 deletions
diff --git a/bin/common/srtool_utils.py b/bin/common/srtool_utils.py index 30ad1e9b..4d9c27cf 100755 --- a/bin/common/srtool_utils.py +++ b/bin/common/srtool_utils.py @@ -1731,6 +1731,101 @@ def fix_bad_score_date(): conn.commit() ################################# +# fix_inherit_affected_components() +# +# Inherit the "Affected Components" from CVEs +# to the new field of their children VUL/INV/DEF + +def fix_inherit_affected_components(): + + conn = sqlite3.connect(srtDbName) + cur_cve = conn.cursor() + cur_cve2vul = conn.cursor() + cur_vul = conn.cursor() + cur_vul2inv = conn.cursor() + cur_inv = conn.cursor() + cur_inv2def = conn.cursor() + cur_def = conn.cursor() + cur_write = conn.cursor() + + def merge_affected_components(alist,blist): + affected_components = '' + affected_components_list = {} + for package in alist.split(): + affected_components_list[package] = True + for package in blist.split(): + affected_components_list[package] = True + if affected_components_list: + affected_components = ' '.join(affected_components_list) + return(affected_components) + + updates = 0 + cur_cve.execute('SELECT * FROM orm_cve') + for i,cve in enumerate(cur_cve): + cve_affect_components = cve[ORM.CVE_PACKAGES] + if not cve_affect_components: + continue + print("CVE:%s, '%s'" % (cve[ORM.CVE_NAME],cve_affect_components)) + + # Find all related Vulnerabilities + cur_cve2vul.execute('SELECT * FROM orm_cvetovulnerablility WHERE cve_id = %d' % cve[ORM.CVE_ID]) + for cve2vul in cur_cve2vul: + # Update the Vulnerability status + cur_vul.execute('SELECT * FROM orm_vulnerability WHERE id = %d' % cve2vul[ORM.CVETOVULNERABLILITY_VULNERABILITY_ID]) + for vul in cur_vul: + vul_affected_components = merge_affected_components(cve_affect_components,vul[ORM.VULNERABILITY_PACKAGES]) + if vul_affected_components != vul[ORM.VULNERABILITY_PACKAGES]: + updates += 1 + if force: + sql = ''' UPDATE orm_vulnerability + SET packages = ? + WHERE id = ?''' + cur_write.execute(sql, (vul_affected_components, vul[ORM.VULNERABILITY_ID],)) + print(" Vul:%s, '%s' to '%s'" % (vul[ORM.VULNERABILITY_NAME],vul[ORM.VULNERABILITY_PACKAGES],vul_affected_components)) + + # Find all related Investigations + cur_vul2inv.execute('SELECT * FROM orm_vulnerabilitytoinvestigation WHERE vulnerability_id = %d' % vul[ORM.VULNERABILITY_ID]) + for vul2inv in cur_vul2inv: + # Update the Investigation status + cur_inv.execute('SELECT * FROM orm_investigation WHERE id = %d' % vul2inv[ORM.VULNERABILITYTOINVESTIGATION_INVESTIGATION_ID]) + for inv in cur_inv: + inv_affected_components = merge_affected_components(vul_affected_components,inv[ORM.INVESTIGATION_PACKAGES]) + if inv_affected_components != inv[ORM.INVESTIGATION_PACKAGES]: + updates += 1 + if force: + sql = ''' UPDATE orm_investigation + SET packages = ? + WHERE id = ?''' + cur_write.execute(sql, (inv_affected_components, inv[ORM.INVESTIGATION_ID],)) + print(" Inv:%s, '%s' to '%s'" % (inv[ORM.INVESTIGATION_NAME],inv[ORM.INVESTIGATION_PACKAGES],inv_affected_components)) + + # Find all related Defects + cur_inv2def.execute('SELECT * FROM orm_investigationtodefect WHERE investigation_id = %d' % inv[ORM.INVESTIGATION_ID]) + for inv2def in cur_inv2def: + # Update the Defect status + cur_def.execute('SELECT * FROM orm_defect WHERE id = %d' % inv2def[ORM.INVESTIGATIONTODEFECT_DEFECT_ID]) + for defect in cur_def: + defect_affected_components = merge_affected_components(inv_affected_components,defect[ORM.DEFECT_PACKAGES]) + if defect_affected_components != defect[ORM.DEFECT_PACKAGES]: + updates += 1 + if force: + sql = ''' UPDATE orm_defect + SET packages = ? + WHERE id = ?''' + cur_write.execute(sql, (defect_affected_components, defect[ORM.DEFECT_ID],)) + print(" Defect:%s, '%s' to '%s'" % (defect[ORM.DEFECT_NAME],defect[ORM.DEFECT_PACKAGES],defect_affected_components)) + + if 999 == (i % 1000) : + print("%7d: %-20s %6d\r" % (i+1,cve[ORM.CVE_NAME],updates),end='') + if force: conn.commit() +# if 60000 < i: +# break + + if updates and force: conn.commit() + print("Affected Component Updates = %d" % updates) + + +################################# # report_cve_status_summary() # # Report the distribution of the CVE status and V3/V2 @@ -2027,13 +2122,13 @@ def report_db_status_summary(): cur_inv = conn.cursor() cur_inv2def = conn.cursor() cur_def = conn.cursor() - cur_cve.execute('SELECT * FROM orm_cve') # # Year-specific table_status # i = 0 + cur_cve.execute('SELECT * FROM orm_cve') for count,cve in enumerate(cur_cve): year = int(cve[ORM.CVE_NAME].split('-')[1]) @@ -2367,6 +2462,52 @@ def report_unattached_records(): # extract product,cve, defect [Defect Status,Defect Resolution] # see if CVE has VUL has INV for the product +################################# +# fix_duplicate_notifications +# +# Remove older duplicate notifications +# + +def fix_duplicate_notifications(): + + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + cur_del = conn.cursor() + + notify_descriptions = {} + delete_list = [] + delete_count = 0 + + cur.execute('SELECT * FROM orm_notify ORDER BY srt_created DESC;') + for i,notify in enumerate(cur): + description = notify[ORM.NOTIFY_DESCRIPTION] + if description in notify_descriptions: + delete_count += 1 + delete_list.append(notify[ORM.NOTIFY_ID]) + else: + notify_descriptions[description] = True + + # Progress indicator support + if (0 == i % 5000): + print('%05d:%05d\r' % (i,delete_count), end='') + + print("") + if force: + print("Deleting %d..." % len(delete_list)) + for i,id in enumerate(delete_list): + sql = 'DELETE FROM orm_notify WHERE id=?' + ret = cur_del.execute(sql, (id,)) + if (0 == i % 1000): + print('%05d:\r' % (i), end='') + if (0 == i % 10000): + time.sleep(0.1) + conn.commit() + conn.commit() + + print("") + print('Delete count = %d of %d, Unique = %d' % (delete_count,i,len(notify_descriptions))) + #print(notify_descriptions) + conn.close() ################################# # main loop @@ -2401,11 +2542,13 @@ def main(argv): parser.add_argument('--fix-bad-mitre-descr', dest='fix_bad_mitre_descr', help='Fix MITRE that were created with empty descriptions') parser.add_argument('--fix-bad-score-date', action='store_const', const='fix_bad_score_date', dest='command', help='Clear score dates to fix obsolete formats') parser.add_argument('--fix-trim-cve-scores', action='store_const', const='fix_trim_cve_scores', dest='command', help='Trim V3/V2 scores to one decimal place standard') + parser.add_argument('--fix-inherit-affected-components', action='store_const', const='fix_inherit_affected_components', dest='command', help='Inherit the affected components field from CVE to its children') # Continuous maintenance validation and repair routines parser.add_argument('--fix-bad-links', action='store_const', const='fix_bad_links', dest='command', help='Find bad links, e.g. "orm_cvesource" (add "-f" to fix)') parser.add_argument('--fix-severity', dest='fix_severity', help='Find bad score/severity values, broken cve source links {ALL|"NIST 2020[,...]*"} (add "-f" to fix)') + parser.add_argument('--fix-duplicate-notifications', action='store_const', const='fix_duplicate_notifications', dest='command', help='Removed older duplicate notifications') parser.add_argument('--report-multiple-defects', action='store_const', const='report_multiple_defects', dest='command', help='Report multiple defects per investigations') parser.add_argument('--report-duplicate-names', action='store_const', const='report_duplicate_names', dest='command', help='Report duplicate names for CVE,VUL,INV,DEF') @@ -2484,11 +2627,15 @@ def main(argv): fix_bad_mitre_descr(args.fix_bad_mitre_descr) elif 'fix_bad_score_date' == args.command: fix_bad_score_date() + elif 'fix_inherit_affected_components' == args.command: + fix_inherit_affected_components() elif args.fix_severity: fix_severity(args.fix_severity) elif 'fix_trim_cve_scores' == args.command: fix_trim_cve_scores() + elif 'fix_duplicate_notifications' == args.command: + fix_duplicate_notifications() elif 'report_multiple_defects' == args.command: report_multiple_defects() |