diff options
| -rwxr-xr-x | bin/common/srtool_utils.py | 149 | ||||
| -rw-r--r-- | lib/orm/models.py | 34 | ||||
| -rw-r--r-- | lib/srtgui/api.py | 7 | ||||
| -rw-r--r-- | lib/srtgui/reports.py | 227 | ||||
| -rw-r--r-- | lib/srtgui/tables.py | 100 | ||||
| -rw-r--r-- | lib/srtgui/templates/defect.html | 3 | ||||
| -rw-r--r-- | lib/srtgui/templates/investigation.html | 5 | ||||
| -rwxr-xr-x | lib/srtgui/templates/maintenance.html | 4 | ||||
| -rw-r--r-- | lib/srtgui/templates/management.html | 6 | ||||
| -rwxr-xr-x | lib/srtgui/templates/srtool_metadata_include.html | 10 | ||||
| -rw-r--r-- | lib/srtgui/templates/vulnerability.html | 2 | ||||
| -rw-r--r-- | lib/srtgui/urls.py | 6 | ||||
| -rw-r--r-- | lib/srtgui/views.py | 59 |
13 files changed, 573 insertions, 39 deletions
diff --git a/bin/common/srtool_utils.py b/bin/common/srtool_utils.py index 30ad1e9b..4d9c27cf 100755 --- a/bin/common/srtool_utils.py +++ b/bin/common/srtool_utils.py @@ -1731,6 +1731,101 @@ def fix_bad_score_date(): conn.commit() ################################# +# fix_inherit_affected_components() +# +# Inherit the "Affected Components" from CVEs +# to the new field of their children VUL/INV/DEF + +def fix_inherit_affected_components(): + + conn = sqlite3.connect(srtDbName) + cur_cve = conn.cursor() + cur_cve2vul = conn.cursor() + cur_vul = conn.cursor() + cur_vul2inv = conn.cursor() + cur_inv = conn.cursor() + cur_inv2def = conn.cursor() + cur_def = conn.cursor() + cur_write = conn.cursor() + + def merge_affected_components(alist,blist): + affected_components = '' + affected_components_list = {} + for package in alist.split(): + affected_components_list[package] = True + for package in blist.split(): + affected_components_list[package] = True + if affected_components_list: + affected_components = ' '.join(affected_components_list) + return(affected_components) + + updates = 0 + cur_cve.execute('SELECT * FROM orm_cve') + for i,cve in enumerate(cur_cve): + cve_affect_components = cve[ORM.CVE_PACKAGES] + if not cve_affect_components: + continue + print("CVE:%s, '%s'" % (cve[ORM.CVE_NAME],cve_affect_components)) + + # Find all related Vulnerabilities + cur_cve2vul.execute('SELECT * FROM orm_cvetovulnerablility WHERE cve_id = %d' % cve[ORM.CVE_ID]) + for cve2vul in cur_cve2vul: + # Update the Vulnerability status + cur_vul.execute('SELECT * FROM orm_vulnerability WHERE id = %d' % cve2vul[ORM.CVETOVULNERABLILITY_VULNERABILITY_ID]) + for vul in cur_vul: + vul_affected_components = merge_affected_components(cve_affect_components,vul[ORM.VULNERABILITY_PACKAGES]) + if vul_affected_components != vul[ORM.VULNERABILITY_PACKAGES]: + updates += 1 + if force: + sql = ''' UPDATE orm_vulnerability + SET packages = ? + WHERE id = ?''' + cur_write.execute(sql, (vul_affected_components, vul[ORM.VULNERABILITY_ID],)) + print(" Vul:%s, '%s' to '%s'" % (vul[ORM.VULNERABILITY_NAME],vul[ORM.VULNERABILITY_PACKAGES],vul_affected_components)) + + # Find all related Investigations + cur_vul2inv.execute('SELECT * FROM orm_vulnerabilitytoinvestigation WHERE vulnerability_id = %d' % vul[ORM.VULNERABILITY_ID]) + for vul2inv in cur_vul2inv: + # Update the Investigation status + cur_inv.execute('SELECT * FROM orm_investigation WHERE id = %d' % vul2inv[ORM.VULNERABILITYTOINVESTIGATION_INVESTIGATION_ID]) + for inv in cur_inv: + inv_affected_components = merge_affected_components(vul_affected_components,inv[ORM.INVESTIGATION_PACKAGES]) + if inv_affected_components != inv[ORM.INVESTIGATION_PACKAGES]: + updates += 1 + if force: + sql = ''' UPDATE orm_investigation + SET packages = ? + WHERE id = ?''' + cur_write.execute(sql, (inv_affected_components, inv[ORM.INVESTIGATION_ID],)) + print(" Inv:%s, '%s' to '%s'" % (inv[ORM.INVESTIGATION_NAME],inv[ORM.INVESTIGATION_PACKAGES],inv_affected_components)) + + # Find all related Defects + cur_inv2def.execute('SELECT * FROM orm_investigationtodefect WHERE investigation_id = %d' % inv[ORM.INVESTIGATION_ID]) + for inv2def in cur_inv2def: + # Update the Defect status + cur_def.execute('SELECT * FROM orm_defect WHERE id = %d' % inv2def[ORM.INVESTIGATIONTODEFECT_DEFECT_ID]) + for defect in cur_def: + defect_affected_components = merge_affected_components(inv_affected_components,defect[ORM.DEFECT_PACKAGES]) + if defect_affected_components != defect[ORM.DEFECT_PACKAGES]: + updates += 1 + if force: + sql = ''' UPDATE orm_defect + SET packages = ? + WHERE id = ?''' + cur_write.execute(sql, (defect_affected_components, defect[ORM.DEFECT_ID],)) + print(" Defect:%s, '%s' to '%s'" % (defect[ORM.DEFECT_NAME],defect[ORM.DEFECT_PACKAGES],defect_affected_components)) + + if 999 == (i % 1000) : + print("%7d: %-20s %6d\r" % (i+1,cve[ORM.CVE_NAME],updates),end='') + if force: conn.commit() +# if 60000 < i: +# break + + if updates and force: conn.commit() + print("Affected Component Updates = %d" % updates) + + +################################# # report_cve_status_summary() # # Report the distribution of the CVE status and V3/V2 @@ -2027,13 +2122,13 @@ def report_db_status_summary(): cur_inv = conn.cursor() cur_inv2def = conn.cursor() cur_def = conn.cursor() - cur_cve.execute('SELECT * FROM orm_cve') # # Year-specific table_status # i = 0 + cur_cve.execute('SELECT * FROM orm_cve') for count,cve in enumerate(cur_cve): year = int(cve[ORM.CVE_NAME].split('-')[1]) @@ -2367,6 +2462,52 @@ def report_unattached_records(): # extract product,cve, defect [Defect Status,Defect Resolution] # see if CVE has VUL has INV for the product +################################# +# fix_duplicate_notifications +# +# Remove older duplicate notifications +# + +def fix_duplicate_notifications(): + + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + cur_del = conn.cursor() + + notify_descriptions = {} + delete_list = [] + delete_count = 0 + + cur.execute('SELECT * FROM orm_notify ORDER BY srt_created DESC;') + for i,notify in enumerate(cur): + description = notify[ORM.NOTIFY_DESCRIPTION] + if description in notify_descriptions: + delete_count += 1 + delete_list.append(notify[ORM.NOTIFY_ID]) + else: + notify_descriptions[description] = True + + # Progress indicator support + if (0 == i % 5000): + print('%05d:%05d\r' % (i,delete_count), end='') + + print("") + if force: + print("Deleting %d..." % len(delete_list)) + for i,id in enumerate(delete_list): + sql = 'DELETE FROM orm_notify WHERE id=?' + ret = cur_del.execute(sql, (id,)) + if (0 == i % 1000): + print('%05d:\r' % (i), end='') + if (0 == i % 10000): + time.sleep(0.1) + conn.commit() + conn.commit() + + print("") + print('Delete count = %d of %d, Unique = %d' % (delete_count,i,len(notify_descriptions))) + #print(notify_descriptions) + conn.close() ################################# # main loop @@ -2401,11 +2542,13 @@ def main(argv): parser.add_argument('--fix-bad-mitre-descr', dest='fix_bad_mitre_descr', help='Fix MITRE that were created with empty descriptions') parser.add_argument('--fix-bad-score-date', action='store_const', const='fix_bad_score_date', dest='command', help='Clear score dates to fix obsolete formats') parser.add_argument('--fix-trim-cve-scores', action='store_const', const='fix_trim_cve_scores', dest='command', help='Trim V3/V2 scores to one decimal place standard') + parser.add_argument('--fix-inherit-affected-components', action='store_const', const='fix_inherit_affected_components', dest='command', help='Inherit the affected components field from CVE to its children') # Continuous maintenance validation and repair routines parser.add_argument('--fix-bad-links', action='store_const', const='fix_bad_links', dest='command', help='Find bad links, e.g. "orm_cvesource" (add "-f" to fix)') parser.add_argument('--fix-severity', dest='fix_severity', help='Find bad score/severity values, broken cve source links {ALL|"NIST 2020[,...]*"} (add "-f" to fix)') + parser.add_argument('--fix-duplicate-notifications', action='store_const', const='fix_duplicate_notifications', dest='command', help='Removed older duplicate notifications') parser.add_argument('--report-multiple-defects', action='store_const', const='report_multiple_defects', dest='command', help='Report multiple defects per investigations') parser.add_argument('--report-duplicate-names', action='store_const', const='report_duplicate_names', dest='command', help='Report duplicate names for CVE,VUL,INV,DEF') @@ -2484,11 +2627,15 @@ def main(argv): fix_bad_mitre_descr(args.fix_bad_mitre_descr) elif 'fix_bad_score_date' == args.command: fix_bad_score_date() + elif 'fix_inherit_affected_components' == args.command: + fix_inherit_affected_components() elif args.fix_severity: fix_severity(args.fix_severity) elif 'fix_trim_cve_scores' == args.command: fix_trim_cve_scores() + elif 'fix_duplicate_notifications' == args.command: + fix_duplicate_notifications() elif 'report_multiple_defects' == args.command: report_multiple_defects() diff --git a/lib/orm/models.py b/lib/orm/models.py index 47d59415..9b4f99ce 100644 --- a/lib/orm/models.py +++ b/lib/orm/models.py @@ -553,6 +553,7 @@ class Cve(models.Model): cvssV2_baseScore = models.CharField(max_length=50, blank=True) cvssV2_severity = models.CharField(max_length=50, blank=True) + # AKA Affected Components packages = models.TextField(blank=True) score_date = models.DateField(null=True, blank=True) @@ -949,6 +950,9 @@ class Vulnerability(models.Model): outcome = models.IntegerField(choices=OUTCOME, default=OPEN) priority = models.IntegerField(choices=PRIORITY, default=LOW) + # AKA Affected Components + packages = models.TextField(blank=True) + srt_updated = models.DateTimeField(auto_now=True, null=True) srt_created = models.DateTimeField(auto_now_add=True, null=True) @@ -1161,6 +1165,9 @@ class Defect(models.Model): date_created = models.CharField(max_length=50) date_updated = models.CharField(max_length=50) + # AKA Affected Components + packages = models.TextField(blank=True) + srt_updated = models.DateTimeField(auto_now=True) # Methods @@ -1292,6 +1299,9 @@ class Investigation(models.Model): outcome = models.IntegerField(choices=OUTCOME, default=INVESTIGATE) priority = models.IntegerField(choices=PRIORITY, default=LOW) + # AKA Affected Components + packages = models.TextField(blank=True) + srt_updated = models.DateTimeField(auto_now=True, null=True) srt_created = models.DateTimeField(auto_now_add=True, null=True) @@ -1415,8 +1425,6 @@ class Notify(models.Model): priority = models.IntegerField(default=0) url = models.TextField(blank=True) author = models.TextField(blank=True) -## srt_updated = models.DateTimeField(auto_now_add=True) -## srt_created = models.DateTimeField(auto_now=True) srt_updated = models.DateTimeField(auto_now=True, null=True) srt_created = models.DateTimeField(auto_now_add=True, null=True) @@ -1462,6 +1470,28 @@ class PublishSet(models.Model): return self.PUBLISH_SET_STATE[self.PUBLISH_SET_ERROR][1] return self.PUBLISH_SET_STATE[self.state][1] +# Error Log +class ErrorLog(models.Model): + search_allowed_fields = ['description'] + + # Severity + INFO = 0 + WARNING = 1 + ERROR = 2 + SEVERITY = ( + (INFO, 'Info'), + (WARNING, 'Warning'), + (ERROR, 'Error'), + ) + + severity = models.IntegerField(default=0) + description = models.TextField(blank=True) + srt_created = models.DateTimeField(auto_now_add=True, null=True) + + @property + def get_severity_text(self): + return ErrorLog.SEVERITY[int(self.severity)][1] + # # Database Cache Support # diff --git a/lib/srtgui/api.py b/lib/srtgui/api.py index e84113a0..761839a8 100644 --- a/lib/srtgui/api.py +++ b/lib/srtgui/api.py @@ -47,6 +47,13 @@ def _log(msg): f1.write("|" + msg + "|\n" ) f1.close() +def error_log(severity,description): + from orm.models import ErrorLog + if (severity < ErrorLog.INFO) or (severity > ErrorLog.ERROR): + severity = ErrorLog.ERROR + error = ErrorLog.objects.create(severity=severity,description=description,) + error.save() + # Sub Process calls def execute_process(*args): cmd_list = [] diff --git a/lib/srtgui/reports.py b/lib/srtgui/reports.py index a1f0de00..715c5606 100644 --- a/lib/srtgui/reports.py +++ b/lib/srtgui/reports.py @@ -20,13 +20,14 @@ import os import logging -from datetime import datetime +from datetime import datetime, timedelta import csv from orm.models import Cve, CveSource, Vulnerability, Investigation, Defect, Product from orm.models import Package from orm.models import SRTool, SrtSetting from orm.models import PublishSet, DefectHistory +from orm.models import Notify, ErrorLog from srtgui.api import readCveDetails, summaryCveDetails from django.db.models import Q @@ -2273,7 +2274,7 @@ class HistoryDefectReport(Report): separator = ";" if csv_separator == 'comma': separator = "," if csv_separator == 'tab': separator = "\t" - writer = csv.writer(csvfile, delimiter=separator, + writer = csv.writer(report_name, delimiter=separator, quotechar='"', quoting=csv.QUOTE_MINIMAL) else: separator = "," @@ -2297,6 +2298,223 @@ class HistoryDefectReport(Report): ############################################################################### # +# Notifications reports +# + +class NotificationsReport(Report): + """Report for the Notifications Page""" + + def __init__(self, parent_page, *args, **kwargs): + _log_args("WR_NOTIFICATION_INIT(%s)" % parent_page, *args, **kwargs) + super(NotificationsReport, self).__init__(parent_page, *args, **kwargs) + + def get_context_data(self, *args, **kwargs): + _log_args("WR_NOTIFICATION_CONTEXT", *args, **kwargs) + context = super(NotificationsReport, self).get_context_data(*args, **kwargs) + + context['report_type_list'] = '\ + <option value="summary">Notification List</option> \ + ' + + context['report_columnrange_list'] = '' + context['report_format_list'] = '\ + <input type="radio" name="format" value="txt" checked> Text<br> \ + <input type="radio" name="format" value="csv"> CSV \ + (Separator: \ + <select name="csv_separator"> \ + <option value="semi">Semi-colon</option> \ + <option value="comma">Comma</option> \ + <option value="tab">Tab</option> \ + </select>) \ + <br> \ + ' + + context['report_recordrange_list'] = '\ + <input type="radio" name="records" value="all"> All<br> \ + ' + + # Add a date range + date_start = datetime.today() - timedelta(days=30) + date_stop = datetime.today() + context['report_date_list'] = '\ + Start: <input type="text" name="date_start" value="%s"><br> \ + Stop: <input type="text" name="date_stop" value="%s"> \ + ' % (date_start.strftime('%m/%d/%Y'),date_stop.strftime('%m/%d/%Y')) + + # Done! + return context + + def exec_report(self, *args, **kwargs): + _log_args("WR_NOTIFICATION_EXEC", *args, **kwargs) + + request_POST = self.request.POST + + records = request_POST.get('records', '') + format = request_POST.get('format', '') +# title = request_POST.get('title', '') + report_type = request_POST.get('report_type', '') + record_list = request_POST.get('record_list', '') + csv_separator = request_POST.get('csv_separator', 'semi') + + # Dates (make as no timezone) + msg = '' + try: + msg = 'Start:%s' % request_POST.get('date_start', '') + date_start = datetime.strptime(request_POST.get('date_start', ''), '%m/%d/%Y') + msg = 'Stop:%s' % request_POST.get('date_stop', '') + date_stop = datetime.strptime(request_POST.get('date_stop', ''), '%m/%d/%Y') + if date_stop < date_start: + return 'Error:stop date is before start date','' + except Exception as e: + return 'Error:bad format for dates (must be mm/dd/yyyy) (%s)(%s)' % (msg,e),'' + + date_start = date_start.strftime('%Y-%m-%d') + date_stop = date_stop.strftime('%Y-%m-%d') + + report_name = '%s/notifications_%s_%s.%s' % (SRT_REPORT_DIR,report_type,datetime.today().strftime('%Y%m%d_%H%M'),format) + with open(report_name, 'w') as file: + + if 'csv' == format: + separator = ";" + if csv_separator == 'comma': separator = "," + if csv_separator == 'tab': separator = "\t" + writer = csv.writer(file, delimiter=separator, + quotechar='"', quoting=csv.QUOTE_MINIMAL) + else: + separator = "," + + if ('summary' == report_type): + if 'csv' == format: + writer.writerow(['Date','Category','Priority','Decription','URL','Author']) + if 'txt' == format: + file.write("Report : Notifications\n") + file.write("\n") + text_format='%02d) %-10s %-25s %-10s "%s",%s,%s\n' + file.write(text_format % (0,'Date','Category','Priority','Decription','URL','Author')) + +# for i,notify in enumerate(Notify.objects.filter(srt_updated__gte=date_start,srt_updated__lte=date_stop).order_by('-srt_updated')): + for i,notify in enumerate(Notify.objects.all().order_by('-srt_updated')): + srt_updated = notify.srt_updated.strftime('%Y-%m-%d') + if (date_start > srt_updated) or (date_stop < srt_updated): + continue + + if 'csv' == format: + writer.writerow([i+1,srt_updated,notify.category,notify.get_priority_text,notify.description,notify.url,notify.author]) + if 'txt' == format: + file.write(text_format % (i+1,srt_updated,notify.category,notify.get_priority_text,notify.description,notify.url,notify.author)) + + return report_name,os.path.basename(report_name) + +############################################################################### +# +# ErrorLogs reports +# + +class ErrorLogsReport(Report): + """Report for the Error Logs Page""" + + def __init__(self, parent_page, *args, **kwargs): + _log_args("WR_ERRORLOGS_INIT(%s)" % parent_page, *args, **kwargs) + super(ErrorLogsReport, self).__init__(parent_page, *args, **kwargs) + + def get_context_data(self, *args, **kwargs): + _log_args("WR_ERRORLOGS_CONTEXT", *args, **kwargs) + context = super(ErrorLogsReport, self).get_context_data(*args, **kwargs) + + context['report_type_list'] = '\ + <option value="summary">Error Log List</option> \ + ' + + context['report_columnrange_list'] = '' + context['report_format_list'] = '\ + <input type="radio" name="format" value="txt" checked> Text<br> \ + <input type="radio" name="format" value="csv"> CSV \ + (Separator: \ + <select name="csv_separator"> \ + <option value="semi">Semi-colon</option> \ + <option value="comma">Comma</option> \ + <option value="tab">Tab</option> \ + </select>) \ + <br> \ + ' + + context['report_recordrange_list'] = '\ + <input type="radio" name="records" value="all"> All<br> \ + ' + + # Add a date range + date_start = datetime.today() - timedelta(days=30) + date_stop = datetime.today() + context['report_date_list'] = '\ + Start: <input type="text" name="date_start" value="%s"><br> \ + Stop: <input type="text" name="date_stop" value="%s"> \ + ' % (date_start.strftime('%m/%d/%Y'),date_stop.strftime('%m/%d/%Y')) + + # Done! + return context + + def exec_report(self, *args, **kwargs): + _log_args("WR_ERRORLOGS_EXEC", *args, **kwargs) + + request_POST = self.request.POST + + records = request_POST.get('records', '') + format = request_POST.get('format', '') +# title = request_POST.get('title', '') + report_type = request_POST.get('report_type', '') + record_list = request_POST.get('record_list', '') + csv_separator = request_POST.get('csv_separator', 'semi') + + # Dates (make as no timezone) + msg = '' + try: + msg = 'Start:%s' % request_POST.get('date_start', '') + date_start = datetime.strptime(request_POST.get('date_start', ''), '%m/%d/%Y') + msg = 'Stop:%s' % request_POST.get('date_stop', '') + date_stop = datetime.strptime(request_POST.get('date_stop', ''), '%m/%d/%Y') + if date_stop < date_start: + return 'Error:stop date is before start date','' + except Exception as e: + return 'Error:bad format for dates (must be mm/dd/yyyy) (%s)(%s)' % (msg,e),'' + + date_start = date_start.strftime('%Y-%m-%d') + date_stop = date_stop.strftime('%Y-%m-%d') + + report_name = '%s/errorlogs_%s_%s.%s' % (SRT_REPORT_DIR,report_type,datetime.today().strftime('%Y%m%d_%H%M'),format) + with open(report_name, 'w') as file: + + if 'csv' == format: + separator = ";" + if csv_separator == 'comma': separator = "," + if csv_separator == 'tab': separator = "\t" + writer = csv.writer(file, delimiter=separator, + quotechar='"', quoting=csv.QUOTE_MINIMAL) + else: + separator = "," + + if ('summary' == report_type): + if 'csv' == format: + writer.writerow(['Date','Severity','Decription']) + if 'txt' == format: + file.write("Report : Error Logs\n") + file.write("Start=%s,Stop=%s\n" % (date_start,date_stop)) + text_format='%02d) %-10s %-10s "%s"\n' + file.write(text_format % (0,'Date','Severity','Decription')) + +# for i,notify in enumerate(ErrorLog.objects.filter(srt_created__gte=date_start,srt_created__lte=date_stop).order_by('-srt_created')): + for i,notify in enumerate(ErrorLog.objects.all().order_by('-srt_created')): + srt_created = notify.srt_created.strftime('%Y-%m-%d') + if (date_start > srt_created) or (date_stop < srt_created): + continue + if 'csv' == format: + writer.writerow([i+1,srt_created,notify.severity,notify.description]) + if 'txt' == format: + file.write(text_format % (i+1,srt_created,notify.get_severity_text,notify.description)) + + return report_name,os.path.basename(report_name) + +############################################################################### +# class DefaultReport(Report): """Report for the Default Page""" @@ -2377,6 +2595,11 @@ class ReportManager(): elif 'cpes_srtool' == parent_page: return CpesSrtoolReport(parent_page, *args, **kwargs) + elif 'manage_notifications' == parent_page: + return NotificationsReport(parent_page, *args, **kwargs) + elif 'error_logs' == parent_page: + return ErrorLogsReport(parent_page, *args, **kwargs) + elif 'history_defect' == parent_page: return HistoryDefectReport(parent_page, *args, **kwargs) diff --git a/lib/srtgui/tables.py b/lib/srtgui/tables.py index e0a6fb42..b8ff6f67 100644 --- a/lib/srtgui/tables.py +++ b/lib/srtgui/tables.py @@ -23,6 +23,7 @@ import re import json from srtgui.widgets import ToasterTable +from orm.models import SRTool from orm.models import Cve, Vulnerability, Investigation, CweTable, Product from orm.models import Package from orm.models import CpeTable, CpeFilter, Defect, DataSource, SrtSetting @@ -30,6 +31,7 @@ from orm.models import PublishPending from orm.models import Notify, NotifyCategories from orm.models import CveHistory, VulnerabilityHistory, InvestigationHistory, DefectHistory from orm.models import PublishSet +from orm.models import ErrorLog from users.models import UserSafe from django.db.models import Q @@ -593,31 +595,31 @@ class DefectsTable(ToasterTable): # SRT Priority filter is_srt_priority = TableFilter(name="is_srt_priority", - title="Filter defects by 'Priority'") - for priority in range(len(Defect.SRT_PRIORITY)): - if Defect.PRIORITY_ERROR == Defect.SRT_PRIORITY[priority][0]: + title="Filter defects by 'SRT Priority'") + for priority in range(len(SRTool.SRT_PRIORITY)): + if SRTool.PRIORITY_ERROR == SRTool.SRT_PRIORITY[priority][0]: continue is_srt_priority.add_action(TableFilterActionToggle( - Defect.SRT_PRIORITY[priority][1].lower().replace(' ','_'), - Defect.SRT_PRIORITY[priority][1], - Q(priority=Defect.SRT_PRIORITY[priority][0])) + SRTool.SRT_PRIORITY[priority][1].lower().replace(' ','_'), + SRTool.SRT_PRIORITY[priority][1], + Q(priority=SRTool.SRT_PRIORITY[priority][0])) ) self.add_filter(is_srt_priority) # SRTool Status filter is_srt_status = TableFilter(name="is_srt_status", - title="Filter defects by 'Status'") - for status in range(len(Defect.SRT_STATUS)): + title="Filter defects by 'SRT Status'") + for status in range(len(SRTool.SRT_STATUS)): is_srt_status.add_action(TableFilterActionToggle( - Defect.SRT_STATUS[status][1].lower().replace(' ','_'), - Defect.SRT_STATUS[status][1], - Q(status=Defect.SRT_STATUS[status][0])) + SRTool.SRT_STATUS[status][1].lower().replace(' ','_'), + SRTool.SRT_STATUS[status][1], + Q(status=SRTool.SRT_STATUS[status][0])) ) self.add_filter(is_srt_status) # SRTool Outcome filter is_srt_outcome = TableFilter(name="is_srt_outcome", - title="Filter defects by 'Outcome'") + title="Filter defects by 'SRT Outcome'") for status in range(len(Defect.SRT_OUTCOME)): is_srt_outcome.add_action(TableFilterActionToggle( Defect.SRT_OUTCOME[status][1].lower().replace(' ','_'), @@ -2057,7 +2059,7 @@ class NotificationsTable(ToasterTable): orderable=True, field_name="srt_created", static_data_name="srt_created", - static_data_template='{{data.srt_updated | date:"m/d/y H:i"}}' + static_data_template='{{data.srt_created | date:"m/d/y H:i"}}' ) self.add_column(title="Category", @@ -2120,14 +2122,70 @@ class NotificationsTable(ToasterTable): # static_data_template='''{{data.author.name}}''', # ) - manage_link_template = ''' - <span class="glyphicon glyphicon-edit edit-notify" id="notify_edit_'+{{data.id}}+'" x-data="{{data.id}}"></span> - ''' -# <span class="glyphicon glyphicon-trash trash-notify" id="notify_trash_'+{{data.id}}+'" x-data="{{data.id}}"></span> - self.add_column(title="Manage", - static_data_name="manage", - static_data_template=manage_link_template, - ) + if False: + manage_link_template = ''' + <span class="glyphicon glyphicon-edit edit-notify" id="notify_edit_'+{{data.id}}+'" x-data="{{data.id}}"></span> + ''' +# <span class="glyphicon glyphicon-trash trash-notify" id="notify_trash_'+{{data.id}}+'" x-data="{{data.id}}"></span> + self.add_column(title="Manage", + static_data_name="manage", + static_data_template=manage_link_template, + ) + + +class ErrorLogsTable(ToasterTable): + """Table of ErrorLogs in SRTool""" + + def __init__(self, *args, **kwargs): + super(ErrorLogsTable, self).__init__(*args, **kwargs) + self.default_orderby = "-srt_created" + + def get_context_data(self,**kwargs): + context = super(ErrorLogsTable, self).get_context_data(**kwargs) + return context + + def setup_queryset(self, *args, **kwargs): + self.queryset = ErrorLog.objects.all() + self.queryset = self.queryset.order_by(self.default_orderby) + + def setup_columns(self, *args, **kwargs): + + self.add_column(title="Select", + field_name="Select", + hideable=False, + static_data_name="select", + static_data_template='<input type="checkbox" value="{{data.pk}}" name="select-notify" />', + ) + + self.add_column(title="SRT Created", + hideable=False, + orderable=True, + field_name="srt_created", + static_data_name="srt_created", + static_data_template='{{data.srt_created | date:"m/d/y H:i"}}' + ) + + self.add_column(title="Severity", + field_name="severity", + orderable=True, + static_data_name="severity", + static_data_template='''{{ data.get_severity_text }}''', + ) + + self.add_column(title="Description", + field_name="description", + hideable=False, + orderable=True, + ) + + if False: + manage_link_template = ''' + <span class="glyphicon glyphicon-trash trash-errorlog" id="errorlog_trash_'+{{data.id}}+'" x-data="{{data.id}}"></span> + ''' + self.add_column(title="Manage", + static_data_name="manage", + static_data_template=manage_link_template, + ) class PackageFilterTable(ToasterTable): diff --git a/lib/srtgui/templates/defect.html b/lib/srtgui/templates/defect.html index ed00fd90..2cae9514 100644 --- a/lib/srtgui/templates/defect.html +++ b/lib/srtgui/templates/defect.html @@ -63,6 +63,9 @@ <dt>SRTool Outcome:</dt> <dd>{{object.get_outcome_text}}</dd> + <dt>Affected Components:</dt> + <dd>{{object.packages}}</dd> + <dt>Publish:</dt> <dd>{{object.publish}}</dd> diff --git a/lib/srtgui/templates/investigation.html b/lib/srtgui/templates/investigation.html index f934d052..c2bf92d7 100644 --- a/lib/srtgui/templates/investigation.html +++ b/lib/srtgui/templates/investigation.html @@ -477,6 +477,9 @@ Created={{object.srt_created}} Updated={{object.srt_updated}} alert("error on request:\n" + data.error); return; } + if (data.note.startsWith("DEFECT-")) { + alert("ERROR:Defect creation failed, temporary defect '" + data.note + "' created. See Error Log."); + } // reload the page with the updated tables location.reload(true); } @@ -745,6 +748,7 @@ Created={{object.srt_created}} Updated={{object.srt_updated}} var priority=$('#select-priority-state').val(); var status=$('#select-status-state').val(); var outcome=$('#select-outcome-state').val(); + var affected_components=$('#text-affected-components').val(); postCommitAjaxRequest({ "action" : 'submit-quickedit', "priority" : priority, @@ -753,6 +757,7 @@ Created={{object.srt_created}} Updated={{object.srt_updated}} "private_note" : private_note, "tags" : tags, "outcome" : outcome, + "affected_components" : affected_components, }); }); diff --git a/lib/srtgui/templates/maintenance.html b/lib/srtgui/templates/maintenance.html index 63c60f33..a0bb1845 100755 --- a/lib/srtgui/templates/maintenance.html +++ b/lib/srtgui/templates/maintenance.html @@ -25,6 +25,10 @@ </thead> <tr> + <td><a class="btn btn-info btn-lg" href="{% url 'error_logs' %}">Error Logs</a></td> + <td>Examine Error Logs ({{errorlog_total}})</td> + </tr> + <tr> <td><a class="btn btn-info btn-lg" href="{% url 'history_cve' %}">History CVE</a></td> <td>Examine History for CVEs</td> </tr> diff --git a/lib/srtgui/templates/management.html b/lib/srtgui/templates/management.html index 9b1e6456..b99f4613 100644 --- a/lib/srtgui/templates/management.html +++ b/lib/srtgui/templates/management.html @@ -26,12 +26,12 @@ <tr> <td><a class="btn btn-info btn-lg" href="{% url 'triage_cves' %}">Triage CVE's</a></td> - <td>Triage the CVE's</td> + <td>Triage the CVE's ({{cve_new}})</td> </tr> <tr> <td><a class="btn btn-info btn-lg" href="{% url 'manage_notifications' %}">Pending notifications</a></td> - <td>Triage the pending notifications</td> + <td>Triage the pending notifications ({{notification_total}})</td> </tr> <tr> @@ -57,7 +57,7 @@ <tr> <td><a class="btn btn-info btn-lg" href="{% url 'maintenance' %}?nocache=1">Maintenance</a></td> - <td>Maintenance utilities</td> + <td>Maintenance utilities ({{errorlog_total}})</td> </tr> {% endif %} diff --git a/lib/srtgui/templates/srtool_metadata_include.html b/lib/srtgui/templates/srtool_metadata_include.html index eb83c05f..05c62d3d 100755 --- a/lib/srtgui/templates/srtool_metadata_include.html +++ b/lib/srtgui/templates/srtool_metadata_include.html @@ -30,11 +30,9 @@ <LI> <i>Tags:</i> {{object.tags}} </LI> - {% if default_category == "CVE" %} - <LI> - <i>Affected Components:</i> {{object.packages}} - </LI> - {% endif %} + <LI> + <i>Affected Components:</i> {{object.packages}} + </LI> </UL> </fieldset> @@ -92,8 +90,8 @@ <p>Private Comments: <input type="text" placeholder="Edit private comments" id="text-private-note" size="80" value="{{object.comments_private}}"></p> {% endif %} <p>Tags: <input type="text" placeholder="Edit tags" id="text-tags" size="80" value="{{object.tags}}"></p> + <p>Affected Components: <input type="text" placeholder="Edit affected components" id="text-affected-components" size="80" value="{{object.packages}}"></p> {% if default_category == "CVE" %} - <p>Affected Components: <input type="text" placeholder="Edit affected components" id="text-affected-components" size="80" value="{{object.packages}}"></p> <i>Acknowledge Date</i> = <input type="text" placeholder="Acknowledge Date" id="text-acknowledge-date" size="40" value="{{object.acknowledge_date|date:'Y-m-d'}}"> (YYYY-MM-DD, or empty string for None)<p> {% endif %} <p><p> diff --git a/lib/srtgui/templates/vulnerability.html b/lib/srtgui/templates/vulnerability.html index 9290a1ef..cd174737 100644 --- a/lib/srtgui/templates/vulnerability.html +++ b/lib/srtgui/templates/vulnerability.html @@ -716,6 +716,7 @@ Created={{object.srt_created}} Updated={{object.srt_updated}} var priority=$('#select-priority-state').val(); var status=$('#select-status-state').val(); var outcome=$('#select-outcome-state').val(); + var affected_components=$('#text-affected-components').val(); postCommitAjaxRequest({ "action" : 'submit-quickedit', "note" : note, @@ -724,6 +725,7 @@ Created={{object.srt_created}} Updated={{object.srt_updated}} "status" : status, "outcome" : outcome, "priority" : priority, + "affected_components" : affected_components, }); }); diff --git a/lib/srtgui/urls.py b/lib/srtgui/urls.py index c0df1c89..ef91f16b 100644 --- a/lib/srtgui/urls.py +++ b/lib/srtgui/urls.py @@ -126,6 +126,9 @@ urlpatterns = [ url(r'^xhr_notifications/$', views.xhr_notifications, name='xhr_notifications'), + url(r'^xhr_errorlogs/$', views.xhr_errorlogs, + name='xhr_errorlogs'), + url(r'^xhr_packages/$', views.xhr_packages, name='xhr_packages'), @@ -153,6 +156,9 @@ urlpatterns = [ url(r'^publish_diff_history/$', views.publish_diff_history, name='publish_diff_history'), url(r'^maintenance/$', views.maintenance, name='maintenance'), + url(r'^error_logs/$', + tables.ErrorLogsTable.as_view(template_name="errorlog-toastertable.html"), + name='error_logs'), url(r'^history_cve/$', tables.HistoryCveTable.as_view(template_name="history-cve-toastertable.html"), name='history_cve'), diff --git a/lib/srtgui/views.py b/lib/srtgui/views.py index 7fdfad98..d3601181 100644 --- a/lib/srtgui/views.py +++ b/lib/srtgui/views.py @@ -40,6 +40,7 @@ from orm.models import DataSource from orm.models import Defect, DefectHistory, PublishPending, PublishSet from orm.models import Notify, NotifyAccess, NotifyCategories from orm.models import SRTool, Update +from orm.models import ErrorLog from users.models import SrtUser, UserSafe @@ -59,7 +60,7 @@ SRT_BASE_DIR = os.environ['SRT_BASE_DIR'] logger = logging.getLogger("srt") # quick development/debugging support -from srtgui.api import _log +from srtgui.api import error_log, _log # # ================= Helper Routines ============================================ @@ -494,6 +495,10 @@ def management(request): 'defect_p2' : defect_p2, 'package_total' : Package.objects.all().count(), + + 'notification_total' : Notify.objects.all().count(), + 'errorlog_total' : ErrorLog.objects.all().count(), + } return render(request, 'management.html', context) @@ -503,6 +508,7 @@ def maintenance(request): return redirect(landing) context = { + 'errorlog_total' : ErrorLog.objects.all().count(), 'history_cve_total' : CveHistory.objects.all().count(), 'history_vulnerability_total' : VulnerabilityHistory.objects.all().count(), 'history_investigation_total' : InvestigationHistory.objects.all().count(), @@ -668,7 +674,7 @@ def vulnerability(request, vulnerability_pk): except: return redirect(landing) - products = Product.objects.all() + products = Product.objects.all().order_by('order') # does this user have permission to see this record? if (not vulnerability_object.public) and (not UserSafe.is_admin(request.user)): @@ -756,12 +762,15 @@ def investigation(request, investigation_pk): # Calculate the default 'affected_components' list, if any affected_components = '' - affected_components_list = [] + affected_components_list = {} + for package in investigation_object.packages.split(): + affected_components_list[package] = True vulnerability = investigation_object.vulnerability vc_list = vulnerability.vulnerability_to_cve.all() for vc in vc_list: if vc.cve.packages: - affected_components_list.append(vc.cve.packages) + for package in vc.cve.packages.split(): + affected_components_list[package] = True if affected_components_list: affected_components = ' '.join(affected_components_list) @@ -1382,6 +1391,8 @@ def _create_defect(investigation,reason,defect_reason,domain_components,affected d_name = params[0] d_url = params[1] _log("SRT_DEFECT3c|%s|%s|" % (d_name,d_url)) + else: + error_log(ErrorLog.ERROR,"DEFECT_CREATION_FAIL(%d)'%s':'%s'" % (result_returncode,result_stdout,result_stderr)) ### TO-DO: Trigger dialog in a production system if not defect created at this point ### For now provide a defect number simulation if not d_name: @@ -1407,6 +1418,7 @@ def _create_defect(investigation,reason,defect_reason,domain_components,affected d.srt_status = Defect.VULNERABLE d.srt_outcome = Defect.OPEN d.url = d_url + d.packages = investigation.packages d.save() _log("NEW_DEFECT:%s|%s|%s|%s" % (d.name,summary,components,priority)) # Create Investigation to Defect @@ -1606,6 +1618,7 @@ def xhr_triage_commit(request): vulnerability.status = new_status vulnerability.outcome = Vulnerability.OPEN vulnerability.comments = reason + vulnerability.packages = cve.packages vulnerability.save() notify_message += " %s" % v_name created_list += ' %s' % vulnerability.name @@ -1645,6 +1658,7 @@ def xhr_triage_commit(request): investigation = Investigation.objects.create(name=i_name,product=product,vulnerability = vulnerability) investigation.priority = cve_priority investigation.outcome = Investigation.OPEN + investigation.packages = cve.packages # Check to see if product is active _log("BOO1:") if 'no' == product.get_product_tag('active','yes'): @@ -1834,6 +1848,7 @@ def xhr_cve_commit(request): status = cve.status, priority = cve.priority, comments = cve.comments, + packages = cve.packages, ) vulnerability.save() history_update.append(Update.ATTACH_INV % (vname)) @@ -1931,6 +1946,7 @@ def xhr_vulnerability_commit(request): priority = int(request.POST['priority']) status = int(request.POST['status']) outcome = int(request.POST['outcome']) + affected_components = request.POST['affected_components'].strip() v = Vulnerability.objects.get(id=v_id) if (v.priority != priority): history_update.append(Update.PRIORITY % (SRTool.priority_text(v.priority),SRTool.priority_text(priority))) @@ -1950,6 +1966,9 @@ def xhr_vulnerability_commit(request): if (tags != v.tags): history_update.append(Update.TAG) v.tags = tags + if (affected_components != v.packages): + history_update.append(Update.AFFECTED_COMPONENT % (v.packages,affected_components)) + v.packages = affected_components v.save() if 'submit-addproduct' == action: products = request.POST['products'] @@ -1968,6 +1987,7 @@ def xhr_vulnerability_commit(request): priority = vulnerability_obj.priority, product = product_obj, comments = vulnerability_obj.comments, + packages = vulnerability_obj.packages, ) vul2inv = VulnerabilityToInvestigation.objects.create(vulnerability=vulnerability_obj,investigation=investigation_obj) vul2inv.save() @@ -2118,6 +2138,30 @@ def xhr_notifications(request): _log("xhr_notifications_commit:no(%s)" % e) return HttpResponse(json.dumps({"error":str(e) + "\n"}), content_type = "application/json") +def xhr_errorlogs(request): + _log("xhr_errorlogs(%s)" % request.POST) + if not 'action' in request.POST: + return HttpResponse(json.dumps({"error":"missing action\n"}), content_type = "application/json") + + action = request.POST['action'] + + _log("xhr_errorlogs1") + + try: + results_msg = '' + if 'delete-errorlogs' == action: + log_list = request.POST['log_list'] + for log_id in log_list.split(','): + ErrorLog.objects.get(pk=log_id).delete() + return_data = { + "error": "ok", + "results_msg": results_msg, + } + return HttpResponse(json.dumps( return_data ), content_type = "application/json") + except Exception as e: + _log("xhr_errorlogs_commit:ERROR(%s)" % e) + return HttpResponse(json.dumps({"error":str(e) + "\n"}), content_type = "application/json") + def xhr_packages(request): _log("xhr_packages(%s)" % request.POST) if not 'action' in request.POST: @@ -2155,6 +2199,7 @@ def xhr_investigation_commit(request): username = UserSafe.user_name(request.user) try: history_update = [] + xhr_note = '' if 'submit-quickedit' == action: priority = int(request.POST['priority']) status = int(request.POST['status']) @@ -2162,6 +2207,7 @@ def xhr_investigation_commit(request): note = request.POST['note'].strip() private_note = request.POST['private_note'].strip() tags = request.POST['tags'].strip() + affected_components = request.POST['affected_components'].strip() invst = Investigation.objects.get(id=invst_id) if (invst.priority != priority): history_update.append(Update.PRIORITY % (SRTool.priority_text(invst.priority),SRTool.priority_text(priority))) @@ -2181,6 +2227,9 @@ def xhr_investigation_commit(request): if (invst.tags != tags): invst.tags = tags history_update.append(Update.TAG) + if (invst.packages != affected_components): + history_update.append(Update.AFFECTED_COMPONENT % (invst.packages,affected_components)) + invst.packages = affected_components invst.save() if 'submit-attachdefectlist' == action: defects = request.POST['defects'] @@ -2230,6 +2279,7 @@ def xhr_investigation_commit(request): affected_components = request.POST['affected_components'].strip() defect_name,created = _create_defect(investigation,'',defect_reason,components,affected_components,username) history_update.append(Update.ATTACH_DEV % defect_name) + xhr_note = defect_name if 'submit-detachdefect' == action: defect_name = request.POST['defect'] product_id = Investigation.objects.get(id=invst_id).product_id @@ -2293,6 +2343,7 @@ def xhr_investigation_commit(request): InvestigationHistory.objects.create(investigation_id=invst_id, comment=update_comment, date=datetime.now().strftime('%Y-%m-%d'), author=username) return_data = { "error": "ok", + "note": xhr_note, } return HttpResponse(json.dumps( return_data ), content_type = "application/json") |