#!/usr/bin/env python3 # # ex:ts=4:sw=4:sts=4:et # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # # Security Response Tool Commandline Tool # # Copyright (C) 2018 Wind River Systems # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as # published by the Free Software Foundation. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. ### Usage Examples (run from top level directory) # Updating defect Issues: ./bin/srtool_defect.py --update import os import sys import re import csv import xml.etree.ElementTree as ET import argparse import sqlite3 import subprocess import json import urllib from time import sleep try: from datetime import datetime, date from urllib.request import urlopen, URLError from urllib.parse import urlparse except ImportError: from urllib2 import urlopen, URLError from urlparse import urlparse srtDbName = 'srt.sqlite' ################################# # Helper methods # overrides = {} def set_override(key,value=None): if not value is None: overrides[key] = value elif key in os.environ.keys(): overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no' else: overrides[key] = '' print("OVERRIDE: %s = %s" % (key,overrides[key])) def get_override(key): if key in overrides.keys(): return overrides[key] return '' ################################# # Access Rights # srt_user = os.environ.get('SRT_USER') srt_passwd = os.environ.get('SRT_PASSWD') if not srt_user: srt_user = 'user' if not srt_passwd: srt_user = 'passwd' ################################# # class to hold fields of a defect # class Defect: id = -1 name = '' summary = '' url = '' priority = -1 status = 0 resolution = 0 publish = 'Unpublished' # Fixed release release_version = '' product_id = 1 date_created = '' date_updated = '' project = '' # extra fields cve_status = 0 vi_status = 0 vi_outcome = 0 ################################# # import defect states # #if too slow, change to check update times and ignore those that need nothing (should do anyway to be honest...) #can also move parsing JSON so that it doesnt happen if record is up to date def update_defects(): try: ### TODO: open connection to defect database # defect_db = ... pass except Exception as e: print("CONNECTION TO DEFECT DATABASE FAILED") return conn = sqlite3.connect(srtDbName) c = conn.cursor() today = datetime.today() weeknum = today.strftime("%W") weekday = today.isoweekday() log = open("./update_logs/update_defects_log_%s_%s.txt" % (weeknum, weekday), "a") PRODUCT_ID = 0 PRODUCT_NAME = 1 PRODUCT_VERSION = 2 PRODUCT_PROFILE = 3 PRODUCT_PREFIX = 6 DEFECT_ID = 0 DEFECT_DATE_UPDATED = 10 PUBLISHED_FIELD = 'customfield_10010' FIX_VERSION_FIELD = 'customfield_11002' log.write("BEGINNING DEFECT UPDATES\n") pre_update_time = datetime.now() products = c.execute('''SELECT * FROM orm_product''').fetchall() for i,product in enumerate(products): if get_override('SRTDBG_MINIMAL_DB') and (i > 1): break #specify which fields to get in order to speed up request! #print("\tupdating ... " + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE] + "\tloading " + spinner[block_num % 3], end='\r', flush=True) log.write("\tUPDATING ... " + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE] + "\n") block_size = 500 block_num = 0 spinner = [' ', '. ', '.. ', '...'] while True: print("\tloading" + spinner[block_num % 4] + "\t" + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE], flush=True) start_idx = block_num*block_size #searches current project's bug issues that contain "cve" in their text ### TODO: FETCH DEFECT RECORDS FOR THIS PRODUCT FROM DEFECT DATABASE if len(issues) == 0: # Retrieve issues until there are no more to come break # Development support block_num += 1 update_project_issues(product, issues, conn, log) conn.commit() #commit to db after each block sleep(1.0) # give time for Sqlite to sync print("\tfinished \t" + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE], flush=True) conn.commit() log.write("began updates: %s\n" % str(pre_update_time)) log.write("finished updates: %s\n" % str(datetime.now())) log.write("=============================================================================\n") log.write("\n") # Reset datasource's update_time as today sql = "UPDATE orm_datasource SET update_time=?,lastModifiedDate=? WHERE data='defect_status'" date_string = datetime.today().strftime('%Y-%m-%d %H:%M:%S') c.execute(sql, (date_string,date_string,) ) conn.commit() c.close() conn.close() #############################################################################3 ### def new_vulnerability_name(c): CVI_ID = 0 CVI_VALUE = 3 sql = "SELECT * FROM orm_srtsetting WHERE name='current_vulnerability_index'" cvi = c.execute(sql).fetchone() if not cvi: index = 100 sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' c.execute(sql, ('current_vulnerability_index', '', index)) else: index = int(cvi[CVI_VALUE]) + 1 sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' c.execute(sql, (index, cvi[CVI_ID])) return "V%05d" % index def new_investigation_name(c): CVI_ID = 0 CVI_VALUE = 3 sql = "SELECT * FROM orm_srtsetting WHERE name='current_investigation_index'" cvi = c.execute(sql).fetchone() if not cvi: index = 100 sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' c.execute(sql, ('current_investigation_index', '', index)) else: index = int(cvi[CVI_VALUE]) + 1 sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' c.execute(sql, (index, cvi[CVI_ID])) return "I%05d" % index def translate_priority(j,p): MINOR = 0 LOW = 1 MEDIUM = 2 HIGH = 3 Priority = ( (MINOR, 'P4'), (LOW, 'P3'), (MEDIUM, 'P2'), (HIGH, 'P1'), ) for i in range(len(Priority)): if p == Priority[i][1]: return str(Priority[i][0]) print("ERROR: unknown priority string '%s=%s'" % (j,p)) log.write("ERROR: unknown priority string '%s=%s'" % (j,p)) return '0' def translate_status(j,s): OPEN = 0 IN_PROGRESS = 1 ON_HOLD = 2 CHECKED_IN = 3 RESOLVED = 4 CLOSED = 5 Status = ( (OPEN, 'Open'), (IN_PROGRESS, 'In progress'), (ON_HOLD, 'On Hold'), (CHECKED_IN, 'Checked In'), (RESOLVED, 'Resolved'), (CLOSED, 'Closed'), ) for i in range(len(Status)): if s == Status[i][1]: return str(Status[i][0]) print("ERROR: unknown status string '%s=%s'" % (j,s)) log.write("ERROR: unknown status string '%s=%s'" % (j,s)) return '0' def translate_resolution(j,r): UNRESOLVED = 0 RESOLVED = 1 FIXED = 2 WILL_NOT_FIX = 3 WITHDRAWN = 4 REJECTED = 5 DUPLICATE = 6 NOT_APPLICABLE = 7 REPLACED_BY_REQUIREMENT = 8 CANNOT_REPRODUCE = 9 DONE = 10 CVE_NEW = 0 CVE_INVESTIGATE = 1 CVE_VULNERABLE = 2 CVE_NOT_VULNERABLE = 3 VI_INVESTIGATE = 0 VI_NOT_VULNERABLE = 1 VI_VULNERABLE = 2 VI_OPEN = 0 VI_CLOSED = 1 VI_FIXED = 2 VI_NOT_FIX = 3 Resolution = ( (UNRESOLVED, 'Unresolved', CVE_VULNERABLE,VI_VULNERABLE,VI_OPEN), (RESOLVED, 'Resolved', CVE_VULNERABLE,VI_VULNERABLE,VI_FIXED), (FIXED, 'Fixed', CVE_VULNERABLE,VI_VULNERABLE,VI_FIXED), (WILL_NOT_FIX, 'Won\'t Fix', CVE_VULNERABLE,VI_VULNERABLE,VI_NOT_FIX), (WITHDRAWN, 'Withdrawn', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), (REJECTED, 'Rejected', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), (DUPLICATE, 'Duplicate', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), (NOT_APPLICABLE, 'Not Applicable', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), (REPLACED_BY_REQUIREMENT, 'Replaced By Requirement',CVE_VULNERABLE,VI_VULNERABLE,VI_CLOSED), (CANNOT_REPRODUCE, 'Cannot Reproduce', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), (DONE, 'Done', CVE_VULNERABLE,VI_VULNERABLE,VI_CLOSED), ) for i in range(len(Resolution)): if r == Resolution[i][1]: return Resolution[i][0],Resolution[i][2],Resolution[i][3],Resolution[i][4] print("ERROR: unknown resolution string '%s=%s'" % (j,r)) log.write("ERROR: unknown resolution string '%s=%s'" % (j,r)) return 0,0,0,0 #handles updating a list of issues for a single product/project #DOES NOT CALL COMMIT (should change this?) def update_project_issues(project, issues, conn, log): global force_update PRODUCT_ID = 0 PRODUCT_NAME = 1 PRODUCT_VERSION = 2 PRODUCT_PROFILE = 3 PRODUCT_PREFIX = 6 #CREATE TABLE "orm_defect" ( #0 "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT #1 "name" varchar(50) NOT NULL #2 "summary" text NOT NULL #3 "url" text NOT NULL #4 "priority" integer NOT NULL #5 "status" integer NOT NULL #6 "resolution" integer NOT NULL #7 "publish" text NOT NULL #8 "release_version" varchar(50) NOT NULL #9 "product_id" integer NULL REFERENCES "orm_product" ("id") #10 "date_created" varchar(50) NOT NULL #11 "date_updated" varchar(50) NOT NULL); # ORM record indexes DEFECT_ID = 0 DEFECT_DATE_UPDATED = 10 ID = 0 NAME = 1 CVE_DESCRIPTION = 13 CV_V_ID = 1 CV_C_ID = 2 I_VULNERABILITY_ID = 2 I_PRODUCT_ID = 3 USER_SRTOOL_ID = "SRTool" PUBLISHED_FIELD = 'customfield_10010' FIX_VERSION_FIELD = 'customfield_11002' d = Defect() d.project = project[PRODUCT_NAME] + " " + project[PRODUCT_VERSION] + " " + project[PRODUCT_PROFILE] d.product_id = project[PRODUCT_ID] cve_regex = re.compile("CVE-\d+-\d+") c = conn.cursor() d_cursor = conn.cursor() for i,issue in enumerate(issues): if get_override('SRTDBG_MINIMAL_DB') and (i > 10): break ### TODO: fill in values from defect database record #d.name = ... #d.date_updated = ... #d.date_created = ... #d.summary = ... #d.url = ... #d.priority = ... #d.status = ... #d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name, ...) #d.publish = ... #d.release_version = ... sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name defect = c.execute(sql).fetchone() #if defect does not exists then create it, if defect is out of date then update the database record, else ignore if defect is None: log.write("\tINSERTING %s\n" % d.name) sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)''' c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated)) # Get the new id sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name defect = c.execute(sql).fetchone() defect_id = defect[DEFECT_ID] elif force_update or (d.date_updated > defect[DEFECT_DATE_UPDATED]): log.write("\tUPDATING %s\n" % d.name) sql = '''UPDATE orm_defect SET summary=?, priority=?, status=?, resolution=?, publish=?, release_version=?, date_updated=? WHERE id = ?''' c.execute(sql, (d.summary, d.priority, d.status, d.resolution, d.publish, d.release_version, d.date_updated, defect[DEFECT_ID])) defect_id = defect[DEFECT_ID] else: log.write("\tSKIPPING %s\n" % d.name) continue # # Update CVE -> Vulnerability -> Investigation -> this Defect chain # # V/I severity and status minimum from defect # Add audit lines # Find parent CVE m = cve_regex.search(d.summary) if m: cve_name = m.group(0) else: print("WARNING: Missing CVE in defect name '%s'" % (d.summary)) continue sql = "SELECT * FROM orm_cve WHERE name='%s'" % cve_name cve = c.execute(sql).fetchone() if not cve: # create the placeholder CVE log.write("\tINSERTING CVE for %s\n" % cve_name) print("INSERTING CVE for %s,%s" % (cve_name,d.name)) sql = ''' INSERT into orm_cve (name, source, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cpe_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV3_vectorString, cvssV3_exploitabilityScore, cvssV3_impactScore, cvssV3_attackVector, cvssV3_attackComplexity, cvssV3_privilegesRequired, cvssV3_userInteraction, cvssV3_scope, cvssV3_confidentialityImpact, cvssV3_integrityImpact, cvssV3_availabilityImpact, cvssV2_baseScore, cvssV2_severity, cvssV2_vectorString, cvssV2_exploitabilityScore, cvssV2_impactScore, cvssV2_accessVector, cvssV2_accessComplexity, cvssV2_authentication, cvssV2_confidentialityImpact, cvssV2_integrityImpact) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' c.execute(sql, (cve_name, '', d.priority, cve_status, '', '', '', '', '', 1, 0, '', 'Created from defect %s' % d.name, '', '', 0, '', '', '','','','','','','','','','','','','','','','','','','','','','','')) # Find the new id print("FINDING CVE ID for %s\n" % cve_name) c.execute("SELECT * FROM orm_cve where name = '%s'" % cve_name) cve = c.fetchone() c_id = cve[ID] # Also create CVE history entry sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)''' c.execute(sql, (c_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) # sleep(0.1) else: c_id = cve[ID] # Find CVE's vulnerability, else create d_cursor.execute("SELECT * FROM orm_cvetovulnerablility where cve_id = '%s'" % c_id) c2v = d_cursor.fetchone() if not c2v: # Create Vulnerability v_name = new_vulnerability_name(d_cursor) log.write("\tINSERTING VULNERABILITY %s for %s\n" % (v_name,cve_name)) print("INSERTING VULNERABILITY for (%s,%s)" % (cve_name,v_name)) sql = '''INSERT INTO orm_vulnerability (name,description,cve_primary_name,public,comments,comments_private,status,outcome,severity) VALUES (?,?,?,?,?,?,?,?,?)''' c.execute(sql, (v_name, cve[CVE_DESCRIPTION], cve_name, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome ,d.priority)) # Find the new id d_cursor.execute("SELECT * FROM orm_vulnerability where name = '%s'" % v_name) v = d_cursor.fetchone() v_id = v[ID] # Also create CVE to Vulnerability sql = '''INSERT INTO orm_cvetovulnerablility (vulnerability_id, cve_id) VALUES (?,?)''' c.execute(sql, (v_id,c_id)) # Also create Vulnerability history entry sql = '''INSERT INTO orm_vulnerabilityhistory (vulnerability_id, comment, date, author) VALUES (?,?,?,?)''' c.execute(sql, (v_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) # sleep(0.1) else: print("FOUND VULNERABILITY ID for %s" % (cve_name)) v_id = c2v[CV_V_ID] # Find CVE's investigation, else create sql = "SELECT * FROM orm_investigation where vulnerability_id = '%s' AND product_id = '%s';" % (v_id,d.product_id) print("I_TEST:%s" % sql) d_cursor.execute(sql) investigation = d_cursor.fetchone() if not investigation: # Create Investigation i_name = new_investigation_name(d_cursor) log.write("\tINSERTING INVESTIGATION for %s\n" % cve_name) print("INSERTING INVESTIGATION for %s,%s" % (i_name,d.name)) sql = '''INSERT INTO orm_investigation (name,vulnerability_id,product_id,public,comments,comments_private,status,outcome,severity) VALUES (?,?,?,?,?,?,?,?,?)''' c.execute(sql, (i_name, v_id, d.product_id, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome, d.priority)) # Find the new id d_cursor.execute("SELECT * FROM orm_investigation where name = '%s'" % i_name) investigation = d_cursor.fetchone() i_id = investigation[ID] # Also create Investigation to Vulnerability AFFECTED = 0 sql = '''INSERT INTO orm_vulnerabilityproduct (vulnerability_id, investigation_id, product_id, relation) VALUES (?,?,?,?)''' c.execute(sql, (v_id,i_id,d.product_id,AFFECTED)) # Also create Investigation history entry sql = '''INSERT INTO orm_investigationhistory (investigation_id, comment, date, author) VALUES (?,?,?,?)''' c.execute(sql, (i_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) # sleep(0.1) else: print("FOUND INVESTIGATION ID for %s" % (cve_name)) i_id = investigation[ID] # Add this defect to the investigation d_cursor.execute("SELECT * FROM orm_investigationtodefect where investigation_id = '%s' and product_id = '%s' and defect_id = '%s'" % (i_id,d.product_id,defect_id)) i2d = d_cursor.fetchone() if not i2d: # Create Investigation i_name = new_investigation_name(d_cursor) log.write("\tINSERTING INVESTIGATION to DEFECT for %s\n" % i_name) sql = '''INSERT INTO orm_investigationtodefect (investigation_id, product_id, defect_id) VALUES (?,?,?)''' c.execute(sql, (i_id,d.product_id,defect_id)) # sleep(0.1) #print("=========================================================================================\n") #print("\n") c.close() ################################# # Add defect record summary to SRTool defect table # def add_to_defect_db(defect_name): defect_name = defect_name.strip().upper() #try connecting to defect database try: ### TODO: open connection to defect database # defect_db = ... conn = sqlite3.connect(srtDbName) c = conn.cursor() except Exception as e: print("xhr_investigation_commit:CONNECTION TO DATABASE FAILED:(%s)\n" % e, file=sys.stderr) return 1 srtool_today = datetime.today().strftime('%Y-%m-%d') #Import the issue into the SRTool try: ### TODO: read defect record from defect database # issue = ... pass except Exception as e: print("ERROR:key '%s' does not exist(%s)" % (defect_name,e), file=sys.stderr) return 1 d = Defect() try: ### TODO: fill in values from defect database record d.name = defect_name d.date_updated = srtool_today # ... d.date_created = srtool_today # ... d.summary = 'temp defect record' # ... #d.url = ... d.priority = translate_priority(d.name,'P3') # ... d.status = translate_status(d.name,'Open') # ... d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved') # ... #d.publish = ... #d.release_version = ... # Get the product ID PRODUCT_ID = 0 PRODUCT_NAME = 1 PRODUCT_VERSION = 2 PRODUCT_PROFILE = 3 PRODUCT_PREFIX = 6 products = c.execute('''SELECT * FROM orm_product''').fetchall() d.product_id = 1 for product in products: if d.name.startswith(product[PRODUCT_PREFIX]): d.product_id = product[PRODUCT_ID] break #log.write("\tINSERTING %s\n" % d.name) sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)''' c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated)) conn.commit() c.close() conn.close() except Exception as e: print("ERROR:could not find/import defect(%s)" % e, file=sys.stderr) return 1 ################################# # main loop # def main(argv): global force_update parser = argparse.ArgumentParser(description='srtool_defect.py: manage the SRTool defect database') parser.add_argument('--update', '-U', action='store_const', const='update', dest='command', help='Import defect states and update SRTool defect table') parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force updates') parser.add_argument('--add', nargs=1, help='Add an existing defect to SRTool defect database') args = parser.parse_args() master_log = open("./update_logs/master_log.txt", "a") force_update = False if None != args.force_update: force_update = args.force_update if args.add: add_to_defect_db(args.add[0]) elif 'update' == args.command: try: print("BEGINNING DEFECT UPDATES PLEASE WAIT ... this can take some time") update() master_log.write("SRTOOL:%s:DEFECT TABLE & DEFECT ISSUES:\t\t\t...\t\t\tUPDATED\n" % (date.today())) print("DATABASE UPDATE FINISHED\n") except Exception as e: master_log.write("SRTOOL:%s:DEFECT TABLE & DEFECT ISSUES:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) print("DATABASE UPDATES FAILED ... %s" % e) else: print("Command not found") if __name__ == '__main__': global script_pathname # fetch any environment overrides set_override('SRTDBG_MINIMAL_DB') set_override('SRTDBG_SKIP_DEFECT_IMPORT') if get_override('SRTDBG_SKIP_DEFECT_IMPORT'): exit(0) script_pathname=os.path.dirname(sys.argv[0]) main(sys.argv[1:])