diff options
-rwxr-xr-x | bin/acme/patcher.json | 20 | ||||
-rwxr-xr-x | bin/acme/srtool_jira_acme.py | 944 | ||||
-rw-r--r--[-rwxr-xr-x] | bin/common/srtool_jira_template.py (renamed from bin/acme/srtool_jira.py) | 346 | ||||
-rwxr-xr-x | bin/common/srtool_patcher.py | 252 |
4 files changed, 1391 insertions, 171 deletions
diff --git a/bin/acme/patcher.json b/bin/acme/patcher.json new file mode 100755 index 00000000..365990fa --- /dev/null +++ b/bin/acme/patcher.json @@ -0,0 +1,20 @@ +{ + "_comments_" : "Blank values indicate defaults", + "label" : "ACME", + "patcher_dir" : "bin/acme/patcher", + "patch_set" : [ + { + "_comments_" : "The ACME custom version of the Jira integration script", + "original" : "bin/common/srtool_jira_template.py", + "custom" : "bin/acme/srtool_jira_acme.py", + "patch" : "", + "options" : "" + }, + { + "original" : "bin/srt", + "custom" : "bin/srt", + "patch" : "", + "options" : "DISABLE" + } + ] +} diff --git a/bin/acme/srtool_jira_acme.py b/bin/acme/srtool_jira_acme.py new file mode 100755 index 00000000..d7ef0507 --- /dev/null +++ b/bin/acme/srtool_jira_acme.py @@ -0,0 +1,944 @@ +#!/usr/bin/env python3 +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Security Response Tool Commandline Tool +# +# Copyright (C) 2018-2019 Wind River Systems +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +### Usage Examples (run from top level directory) +# Updating Jira Issues: ./bin/<app>/srtool_jira<_app>.py -u + +## +## THIS IS A JIRA INTEGRATION SCRIPT FOR +## MANAGING AN ORGANIZATION'S SRTOOL INTEGRATION +## +## You can use the "bin/common/srtool_patcher.py" to extend this template +## with your custom changes yet keep in sync with the shared upstream file. +## +## See the example: "bin/acme/srtool_jira_acme.py" +## +## INSTALLATION INSTRUCTIONS AND DOCUMENATION OF THE JIRA PYTHON +## LIBRARIES CAN BE FOUND HERE: +## https://jira.readthedocs.io +## + +### ACME_EXTENSION_BEGIN ### +# +# This is the ACME 'patcher' extension of: +# 'bin/common/srtool_jira_template.py' +# +# To merge edits in common areas to upstream, run: +# $ ./bin/common/srtool_patcher.py -j bin/acme/patcher.json --merge-custom +# To merge upstream into this custom extension, run: +# $ ./bin/common/srtool_patcher.py -j bin/acme/patcher.json --merge-original +# +### ACME_EXTENSION_END ### + +import os +import sys +import re +import argparse +import sqlite3 +import json +from datetime import datetime, date + +# load the srt.sqlite schema indexes +dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +sys.path.insert(0, dir_path) +from common.srt_schema import ORM + +# Setup: +master_log = '' +srt_user = '' +srt_passwd = '' +force_update = False + +srtDbName = 'srt.sqlite' +srtErrorLog = 'srt_errors.txt' + +# +# Load Jira test +# + +try: + from jira import JIRA +except: + # could be SRTDBG_SKIP_DEFECT_IMPORT + pass + +## +## THESE ARE JIRA INTEGRATION LINKS: +## The 'production' link is to the main Jira server +## The 'test' link is to a test Jira server, for example to pre-test defect creation +## The 'browse' link is how to format the Jira defect lookup link that is passed to the SRTool users +## + +# Jira constants: examples +JIRA_PRODUCTION_LINK = 'https://jira.sample.com' +JIRA_BROWSE_LINK_FORMAT = 'http://jira.sample.com/browse/%s' +JIRA_TESTSERVER_LINK = 'https://jira.sampletest.com' +JIRA_NEW_ASSIGNEE = 'kilroy' + +### ACME_EXTENSION_BEGIN ### +JIRA_PRODUCTION_LINK = 'https://jira.wrs.com' +JIRA_BROWSE_LINK_FORMAT = 'http://jira.wrs.com/browse/%s' +JIRA_TESTSERVER_LINK = 'http://jiratest.wrs.com' +JIRA_NEW_ASSIGNEE = 'dreyna' +### ACME_EXTENSION_END ### + +## +## THESE ARE SAMPLE JIRA CUSTOM FIELD LINKS: +## These are example links that you can use to define and populate the respective fields for the SRTool +## These are not defined in the standard Jira schema, and would presumably be custom added +## You can extend and cusomize the Jira integration to SRTool by added additional custom fields using this model +## The two indicated fields below are part of the SRTool design. They can be present as empty strings if they +## do not exist in your schema: +## The 'PUBLISHED' field is used to indicate if and when a Jira defect has been published to the customers +## The 'FIX_VERSION' field is used to indicate which version/release/update/service pack of the product has the fix +## + +# Custom Jira fields +# Custom Jira fields +JIRA_PUBLISHED_FIELD = 'customfield_10001' ### REPLACE WITH YOUR CUSTOM FIELD VALUE +JIRA_FIX_VERSION_FIELD = 'customfield_10002' ### REPLACE WITH YOUR CUSTOM FIELD VALUE +# ... + +### ACME_EXTENSION_BEGIN ### +JIRA_PUBLISHED_FIELD = 'customfield_10010' +JIRA_FIX_VERSION_FIELD = 'customfield_11002' +### ACME_EXTENSION_END ### + +################################# +# Helper methods +# + +verbose = False + +def debugMsg(msg): + if verbose: + print(msg) + +overrides = {} + +def set_override(key,value=None,quite=False): + if not value is None: + overrides[key] = value + elif key in os.environ.keys(): + overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no' + else: + overrides[key] = 'no' + if 'yes' == overrides[key] and not quite: + print("OVERRIDE: %s = %s" % (key,overrides[key])) + +def get_override(key): + if key in overrides.keys(): + return 'yes' == overrides[key] + return False + +def srt_error_log(msg): + f1=open(srtErrorLog, 'a') + f1.write("|" + msg + "|\n" ) + f1.close() + +def get_tag_key(tag,key): + d = json.loads(tag) + return d[key] + +################################# +# class to hold fields of a Jira issues +# + +class Defect(): + project = '' + product_id = '' + id = -1 + name = '' + summary = '' + url = '' + priority = -1 + status = '' + resolution = 'Unresolved' + publish = '' + # RCPL + release_version = '' + date_created = '' + date_updated = '' + # extra fields + cve_status = '' + vi_status = '' + vi_outcome = '' + +################################# +# import Jira states +# +#if too slow, change to check update times and ignore those that need nothing (should do anyway to be honest...) +#can also move parsing JSON so that it doesnt happen if record is up to date + +def do_update_jira(): + try: + jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) + except Exception as e: + print("CONNECTION TO JIRA FAILED") + return 1 + + conn = sqlite3.connect(srtDbName) + c = conn.cursor() + + today = datetime.today() + weeknum = today.strftime("%W") + weekday = today.isoweekday() + + update_log = open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a") + + update_log.write("BEGINNING JIRA UPDATES\n") + pre_update_time = datetime.now() + + products = c.execute('''SELECT * FROM orm_product''').fetchall() + for i,product in enumerate(products): +# print("JIRA_PRODUCT_SCAN:%s,%s" % (product[ORM.PRODUCT_DEFECT_TAGS],get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key'))) + product_defect_prefix = get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key') + + if get_override('SRTDBG_MINIMAL_DB') and (i > 1): + break + + #specify which fields to get in order to speed up request! + #print("\tupdating ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\tloading " + spinner[block_num % 3], end='\r', flush=True) + update_log.write("\tUPDATING ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\n") + + block_size = 500 + block_num = 0 + spinner = [' ', '. ', '.. ', '...'] + while True: + print("\tloading" + spinner[block_num % 4] + "\t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], end='\r', flush=True) + start_idx = block_num*block_size + #searches current project's bug issues that contain "cve" in their text + issues = jira.search_issues('project=%s AND text ~ "cve*" AND type = Bug' % product_defect_prefix, start_idx, block_size, False, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD)) + if len(issues) == 0: + # Retrieve issues until there are no more to come + break + # Development support + block_num += 1 + update_project_issues(product, issues, conn, update_log) +# sleep(1.0) # give time for Sqlite to sync + conn.commit() #commit to db after each block +# sleep(1.0) # give time for Sqlite to sync +# conn.commit() #commit to db after each product + print("\tfinished \t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], flush=True) + conn.commit() + update_log.write("began updates: %s\n" % str(pre_update_time)) + update_log.write("finished updates: %s\n" % str(datetime.now())) + update_log.write("=============================================================================\n") + update_log.write("\n") + + # Reset datasource's lastModifiedDate as today + sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE name='Jira'" + date_string = datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT) + ret = c.execute(sql, (date_string,) ) + conn.commit() + + c.close() + conn.close() + +#############################################################################3 +### + +def new_vulnerability_name(cur): + sql = "SELECT * FROM orm_srtsetting WHERE name='current_vulnerability_index'" + cvi = cur.execute(sql).fetchone() + if not cvi: + index = 100 + sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' + cur.execute(sql, ('current_vulnerability_index', '', index)) + else: + index = int(cvi[ORM.SRTSETTING_VALUE]) + 1 + sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' + cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID])) + return "VUL-%05d" % index + +def new_investigation_name(cur): + sql = "SELECT * FROM orm_srtsetting WHERE name='current_investigation_index'" + cvi = cur.execute(sql).fetchone() + if not cvi: + index = 100 + sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' + cur.execute(sql, ('current_investigation_index', '', index)) + else: + index = int(cvi[ORM.SRTSETTING_VALUE]) + 1 + sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' + cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID])) + return "INV-%05d" % index + +def translate_priority(j,p): + NONE = 0 + MINOR = 1 + LOW = 2 + MEDIUM = 3 + HIGH = 4 + Priority = ( + (NONE, 'None'), + (MINOR, 'P4'), + (LOW, 'P3'), + (MEDIUM, 'P2'), + (HIGH, 'P1'), + ) + + for i in range(len(Priority)): + if p == Priority[i][1]: + return str(Priority[i][0]) + print("ERROR: unknown priority string '%s=%s'" % (j,p)) + srt_error_log("ERROR: unknown priority string '%s=%s'" % (j,p)) + return '0' + +def translate_priority_srt_to_jira(s): + Priority = ( + ('Undefined', 'None'), + ('Minor', 'P4'), + ('Low', 'P3'), + ('Medium', 'P2'), + ('High', 'P1'), + ) + for i in range(len(Priority)): + if s == Priority[i][0]: + return str(Priority[i][1]) + print("ERROR: unknown priority string '%s'" % (s)) + srt_error_log("ERROR: unknown priority string '%s'" % (s)) + return 'None' + +def translate_status(j,s): + OPEN = 0 + IN_PROGRESS = 1 + ON_HOLD = 2 + CHECKED_IN = 3 + RESOLVED = 4 + CLOSED = 5 + Status = ( + (OPEN, 'Open'), + (IN_PROGRESS, 'In progress'), + (ON_HOLD, 'On Hold'), + (CHECKED_IN, 'Checked In'), + (RESOLVED, 'Resolved'), + (CLOSED, 'Closed'), + ) + for i in range(len(Status)): + if s == Status[i][1]: + return str(Status[i][0]) + print("ERROR: unknown status string '%s=%s'" % (j,s)) + srt_error_log("ERROR: unknown status string '%s=%s'" % (j,s)) + return '0' + +def translate_resolution(j,r,log): + Resolution = ( + (ORM.DEFECT_UNRESOLVED, 'Unresolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_OPEN), + (ORM.DEFECT_RESOLVED, 'Resolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_FIXED), + (ORM.DEFECT_FIXED, 'Fixed', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_FIXED), + (ORM.DEFECT_WILL_NOT_FIX, 'Won\'t Fix', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_NOT_FIX), + (ORM.DEFECT_WITHDRAWN, 'Withdrawn', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED), + (ORM.DEFECT_REJECTED, 'Rejected', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED), + (ORM.DEFECT_DUPLICATE, 'Duplicate', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED), + (ORM.DEFECT_NOT_APPLICABLE, 'Not Applicable', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED), + (ORM.DEFECT_REPLACED_BY_REQUIREMENT, 'Replaced By Requirement',ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_CLOSED), + (ORM.DEFECT_CANNOT_REPRODUCE, 'Cannot Reproduce', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED), + (ORM.DEFECT_DONE, 'Done', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_CLOSED), + ) + for i in range(len(Resolution)): + if r == Resolution[i][1]: + return Resolution[i][0],Resolution[i][2],Resolution[i][3],Resolution[i][4], + print("ERROR: unknown resolution string '%s=%s'" % (j,r)) + log.write("ERROR: unknown resolution string '%s=%s'" % (j,r)) + return 0,0,0,0 + +#handles updating a list of issues for a single product/project +#DOES NOT CALL COMMIT (should change this?) +def update_project_issues(project, issues, conn, log): + global force_update + + #CREATE TABLE "orm_defect" ( + #0 "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT + #1 "name" varchar(50) NOT NULL + #2 "summary" text NOT NULL + #3 "url" text NOT NULL + #4 "priority" integer NOT NULL + #5 "status" integer NOT NULL + #6 "resolution" integer NOT NULL + #7 "publish" text NOT NULL + #8 "release_version" varchar(50) NOT NULL + #9 "product_id" integer NULL REFERENCES "orm_product" ("id") + #10 "date_created" varchar(50) NOT NULL + #11 "date_updated" varchar(50) NOT NULL); + + USER_SRTOOL_ID = "SRTool" + + project_str = project[ORM.PRODUCT_NAME] + " " + project[ORM.PRODUCT_VERSION] + " " + project[ORM.PRODUCT_PROFILE] + product_id = project[ORM.PRODUCT_ID] + + cve_regex = re.compile(r"CVE-\d+-\d+") + srtool_today = datetime.today().strftime('%Y-%m-%d') + + c = conn.cursor() + d_cursor = conn.cursor() + for i,issue in enumerate(issues): + + if get_override('SRTDBG_MINIMAL_DB') and (i > 10): + break + + issue_json = issue.raw + + d = Defect() + d.project = project_str + d.product_id = product_id + + d.name = issue_json['key'] + d.date_updated = issue_json['fields']['updated'] + d.date_created = issue_json['fields']['created'] + d.summary = issue_json['fields']['summary'] + d.url = JIRA_BROWSE_LINK_FORMAT % d.name + d.priority = translate_priority(d.name,issue_json['fields']['priority']['name']) + d.status = translate_status(d.name,issue_json['fields']['status']['name']) + + if issue_json['fields']['resolution'] is not None: + d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log) + else: + d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,'Unresolved',log) + if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None: + d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value'] + if JIRA_FIX_VERSION_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_FIX_VERSION_FIELD] is not None: + d.release_version = issue_json['fields'][JIRA_FIX_VERSION_FIELD]['name'] + + sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name + defect = c.execute(sql).fetchone() + + #if defect does not exists then create it, if defect is out of date then update the database record, else ignore + if defect is None: + log.write("\tINSERTING %s\n" % d.name) + sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated, srt_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)''' + c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated, datetime.now())) + # Get the new id + sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name + defect = c.execute(sql).fetchone() + defect_id = defect[ORM.DEFECT_ID] + elif force_update or (d.date_updated > defect[ORM.DEFECT_DATE_UPDATED]): + log.write("\tUPDATING %s\n" % d.name) + sql = '''UPDATE orm_defect SET summary=?, priority=?, status=?, resolution=?, publish=?, release_version=?, date_updated=? WHERE id = ?''' + c.execute(sql, (d.summary, d.priority, d.status, d.resolution, d.publish, d.release_version, d.date_updated, defect[ORM.DEFECT_ID])) + defect_id = defect[ORM.DEFECT_ID] + else: + log.write("\tSKIPPING %s\n" % d.name) + continue + + # + # Update CVE -> Vulnerability -> Investigation -> this Defect chain + # + + # V/I priority and status minimum from defect + # Add audit lines + + # Find parent CVE + m = cve_regex.search(d.summary) + if m: + cve_name = m.group(0) + else: + print("WARNING: Missing CVE in defect name '%s'" % (d.summary)) + continue + sql = "SELECT * FROM orm_cve WHERE name='%s'" % cve_name + cve = c.execute(sql).fetchone() + if not cve: + # create the placeholder CVE + log.write("\tINSERTING CVE for %s\n" % cve_name) + print("INSERTING CVE for %s,%s" % (cve_name,d.name)) + + try: + a = cve_name.split('-') + cve_name_sort = '%s-%s-%07d' % (a[0],a[1],int(a[2])) + except: + cve_name_sort = cve.name + + sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, packages, srt_updated) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' + c.execute(sql, (cve_name, cve_name_sort, d.priority, cve_status, '', '', '', '', '', 1, 0, '', 'Created from defect %s' % d.name, '', '', 0, '', '', '', '', '', '', datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT))) + + # Find the new id + print("FINDING CVE ID for %s\n" % cve_name) + c.execute("SELECT * FROM orm_cve where name = '%s'" % cve_name) + cve = c.fetchone() + c_id = cve[ORM.CVE_ID] + # Also create CVE history entry + sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)''' + c.execute(sql, (c_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) +# sleep(0.1) + else: + c_id = cve[ORM.CVE_ID] + + # Find CVE's vulnerability, else create + d_cursor.execute("SELECT * FROM orm_cvetovulnerablility where cve_id = '%s'" % c_id) + c2v = d_cursor.fetchone() + if not c2v: + # Create Vulnerability + v_name = new_vulnerability_name(d_cursor) + log.write("\tINSERTING VULNERABILITY %s for %s\n" % (v_name,cve_name)) + print("INSERTING VULNERABILITY for (%s,%s)" % (cve_name,v_name)) + sql = '''INSERT INTO orm_vulnerability (name,description,cve_primary_name,public,comments,comments_private,status,outcome,priority) VALUES (?,?,?,?,?,?,?,?,?)''' + c.execute(sql, (v_name, cve[ORM.CVE_DESCRIPTION], cve_name, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome ,d.priority)) + # Find the new id + d_cursor.execute("SELECT * FROM orm_vulnerability where name = '%s'" % v_name) + v = d_cursor.fetchone() + v_id = v[ORM.VULNERABILITY_ID] + # Also create CVE to Vulnerability + sql = '''INSERT INTO orm_cvetovulnerablility (vulnerability_id, cve_id) VALUES (?,?)''' + c.execute(sql, (v_id,c_id)) + # Also create Vulnerability history entry + sql = '''INSERT INTO orm_vulnerabilityhistory (vulnerability_id, comment, date, author) VALUES (?,?,?,?)''' + c.execute(sql, (v_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) +# sleep(0.1) + else: + print("FOUND VULNERABILITY ID for %s" % (cve_name)) + v_id = c2v[ORM.CVETOVULNERABLILITY_VULNERABILITY_ID] + + # Find CVE's investigation, else create + sql = "SELECT * FROM orm_investigation where vulnerability_id = '%s' AND product_id = '%s';" % (v_id,d.product_id) +# print("I_TEST:%s" % sql) + d_cursor.execute(sql) + investigation = d_cursor.fetchone() + if not investigation: + # Create Investigation + i_name = new_investigation_name(d_cursor) + log.write("\tINSERTING INVESTIGATION for %s\n" % cve_name) + print("INSERTING INVESTIGATION for %s,%s" % (i_name,d.name)) + sql = '''INSERT INTO orm_investigation (name,vulnerability_id,product_id,public,comments,comments_private,status,outcome,priority) VALUES (?,?,?,?,?,?,?,?,?)''' + c.execute(sql, (i_name, v_id, d.product_id, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome, d.priority)) + # Find the new id + d_cursor.execute("SELECT * FROM orm_investigation where name = '%s'" % i_name) + investigation = d_cursor.fetchone() + i_id = investigation[ORM.INVESTIGATION_ID] + # Also create Vulnerability to Investigation + sql = '''INSERT INTO orm_vulnerabilitytoinvestigation (vulnerability_id, investigation_id) VALUES (?,?)''' + c.execute(sql, (v_id,i_id)) + # Also create Investigation history entry + sql = '''INSERT INTO orm_investigationhistory (investigation_id, comment, date, author) VALUES (?,?,?,?)''' + c.execute(sql, (i_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) +# sleep(0.1) + else: + print("FOUND INVESTIGATION ID for %s" % (cve_name)) + i_id = investigation[ORM.INVESTIGATION_ID] + + # Add this defect to the investigation + d_cursor.execute("SELECT * FROM orm_investigationtodefect where investigation_id = '%s' and product_id = '%s' and defect_id = '%s'" % (i_id,d.product_id,defect_id)) + i2d = d_cursor.fetchone() + if not i2d: + # Create Investigation + i_name = new_investigation_name(d_cursor) + log.write("\tINSERTING INVESTIGATION to DEFECT for %s\n" % i_name) + sql = '''INSERT INTO orm_investigationtodefect (investigation_id, product_id, defect_id) VALUES (?,?,?)''' + c.execute(sql, (i_id,d.product_id,defect_id)) +# sleep(0.1) + + #print("=========================================================================================\n") + #print("\n") + c.close() + +################################# +# Jira list update +# + +def jira_update_list(jira_list): + try: + jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) + except Exception as e: + print("CONNECTION TO JIRA FAILED") + return 1 + + conn = sqlite3.connect(srtDbName) + c = conn.cursor() + + products = c.execute('''SELECT * FROM orm_product''').fetchall() + log = sys.stdout + + for jira_name in jira_list.split(','): + print("Updating:\t" + jira_name, flush=True) + # find the matching parent project + for i,product in enumerate(products): + product_defect_prefix = get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key') + print("\tfrom:\t%s %s %s %s" %(product[ORM.PRODUCT_NAME],product[ORM.PRODUCT_VERSION],product[ORM.PRODUCT_PROFILE],product_defect_prefix), flush=True) + block_size = 500 + block_num = 0 + while True: + start_idx = block_num*block_size + #searches current project's bug issues that contain "cve" in their text + issues = jira.search_issues('project=%s AND text ~ "%s" AND type = Bug' % (product_defect_prefix,jira_name), start_idx, block_size, False, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD)) #project argument could be better written I guess :) + if len(issues) == 0: + # Retrieve issues until there are no more to come + break + # Development support + block_num += 1 + update_project_issues(product, issues, conn, log) +# sleep(1.0) # give time for Sqlite to sync + conn.commit() #commit to db after each block + +################################# +# Jira Enhancement Requests +# + +#Gets all JIRA Enhancement Requests for the specified project +def get_jira_er(project_prefix): + print ("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix) + try: + jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) + except Exception as e: + print("CONNECTION TO JIRA FAILED") + return 1 + + block_size = 100 + block_num = 0 + while True: + start_idx = block_num*block_size + #searches current project's bug issues that contain "cve" in their text + issues = jira.search_issues('project=%s AND type = "Enhancement Request"' % project_prefix, start_idx, block_size, False) + if len(issues) == 0: + # Retrieve issues until there are no more to come + break + block_num += 1 + for issue in issues: + print (issue.raw['key']) + +################################# +# Jira add to investigation +# + +def jira_add_to_defect_db(jira_name): + print("Jira_name=%s" % (jira_name)) + jira_name = jira_name.strip().upper() + + # Just error errors to terminal + log = sys.stdout + + #try connecting to jira + try: + jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) + conn = sqlite3.connect(srtDbName) + c = conn.cursor() + except Exception as e: + print("xhr_investigation_commit:CONNECTION TO JIRA FAILED:(%s)\n" % e, file=sys.stderr) + return 1 + + #Import the issue into the SRTool + try: + issue = jira.issue(jira_name, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD)) + except Exception as e: + print("ERROR:key '%s' does not exist(%s)" % (jira_name,e), file=sys.stderr) + return 1 + + d = Defect() + try: + issue_json = issue.raw + d.name = issue_json['key'] + d.date_updated = issue_json['fields']['updated'] + d.date_created = issue_json['fields']['created'] + d.summary = issue_json['fields']['summary'] + d.url = JIRA_BROWSE_LINK_FORMAT % d.name + d.priority = translate_priority(d.name,issue_json['fields']['priority']['name']) + d.status = translate_status(d.name,issue_json['fields']['status']['name']) + + if issue_json['fields']['resolution'] is not None: + d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log) + else: + d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved',log) + if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None: + d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value'] + else: + d.publish='' + if JIRA_FIX_VERSION_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_FIX_VERSION_FIELD] is not None: + d.release_version = issue_json['fields'][JIRA_FIX_VERSION_FIELD]['name'] + else: + d.release_version = '' + + # Get the product ID + products = c.execute('''SELECT * FROM orm_product''').fetchall() + d.product_id = 1 + for product in products: + if d.name.startswith(get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')): + d.product_id = product[ORM.PRODUCT_ID] + break + + #_log("\tINSERTING %s\n" % d.name) + sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)''' + c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated)) + conn.commit() + c.close() + conn.close() + except Exception as e: + print("ERROR:could not find/import defect(%s)" % e, file=sys.stderr) + return 1 + +################################# +# jira_del_from_defect_db +# + +# Not yet implemented +def jira_del_from_defect_db(jira_name): + return 1 + +################################# +# New defect +# + +# Use "jiratest" for development testing +JIRA_IS_TEST = True +JIRA_IS_SIMULATE = True + +def simulate_new_defect_name(product_prefix): + conn = sqlite3.connect(srtDbName) + cur = conn.cursor() + + sql = "SELECT * FROM orm_srtsetting WHERE name='current_defect_simulation_index'" + cvi = cur.execute(sql).fetchone() + if not cvi: + index = 100 + sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' + cur.execute(sql, ('current_defect_simulation_index', '', index)) + else: + index = int(cvi[ORM.SRTSETTING_VALUE]) + 1 + sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' + cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID])) + conn.commit() #commit to db + conn.close() + + defect_name = "%s-%05d" % (product_prefix,index) + return defect_name + +def simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link,jira_url): + product_defect_prefix = get_tag_key(product_defect_tags,'key') + defect_name = simulate_new_defect_name(product_defect_prefix) + print("CREATED:%s,%s/browse/%s" % (defect_name,jira_url,defect_name)) + +def jira_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link): + srt_error_log("NEW DEFECT:%s|%s|%s|%s|%s|%s|%s|%s" % (product_defect_tags,summary,cve_list,description,reason,priority,components,link)) + + if JIRA_IS_TEST: + # Test URL + jira_url = JIRA_TESTSERVER_LINK + else: + # Production URL + jira_url = JIRA_PRODUCTION_LINK + + if JIRA_IS_SIMULATE: + return simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link,jira_url) + + # Connect to Jira server + try: + jira = JIRA(jira_url, auth=(srt_user, srt_passwd)) + except Exception as e: + print("CONNECTION TO JIRA FAILED") + return 1 + #srt_error_log("Jira connection made") + + conn = sqlite3.connect(srtDbName) + c = conn.cursor() + + # append the jira link to description + description += "\n\n\n%s" % link + + product_defect_prefix = get_tag_key(product_defect_tags,'key') + + # Translate the SRTool priority to Jira priority + priority = translate_priority_srt_to_jira(priority) + + #print("FOO1:%s,%s,%s" % (product_defect_prefix,summary,description)) + + jira_components=list() + for component in components.split(' '): + jira_components.append({'name' : component}) + + issue_dict = { + 'project': {'key': product_defect_prefix}, + 'summary': summary, + 'description': description, + 'issuetype': {'name': 'Bug'}, + 'priority': {'name': priority}, + 'components': jira_components, # Components by name + 'assignee': {"name":JIRA_NEW_ASSIGNEE}, # assign to the SRTool developer for now + } + +## +## Add custom fields here, using the format... +## issue_dict['custom_field_10001'] = value +## + + ### ACME_EXTENSION_BEGIN ### + ## HERE IS AN EXAMPLE OF ADJUSTMENTS TO THE JIRA QUERY + issue_dict['customfield_13304'] = {"value":"Review"} # "Where Found" = "Review" + ### ACME_EXTENSION_END ### + + new_issue = jira.create_issue(fields=issue_dict) + #print("CREATE:%s" % issue_dict) + #new_issue = '%s-%s' % (product_defect_prefix,datetime.now().strftime('%H%M%S')) + + new_issue = str(new_issue.key) + if not new_issue.startswith(product_defect_prefix): + print("ERROR:%s" % new_issue) + else: + print("CREATED:%s,%s/browse/%s" % (new_issue,jira_url,new_issue)) + + +def get_projects(): + try: + jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) + except Exception as e: + print("CONNECTION TO JIRA FAILED") + return 1 + + # Get all projects viewable by anonymous users. + projects = jira.projects() + print("Projects:%s" % projects) + + +################################# +# Init/Update from Jira status +# + +def update_jira(is_init): + try: + print("BEGINNING JIRA UPDATES PLEASE WAIT ... this can take some time") + do_update_jira() + master_log.write("SRTOOL:%s:DEFECT TABLE & JIRA ISSUES:\t\t\t...\t\t\tUPDATED\n" % (date.today())) + print("DATABASE UPDATE FINISHED\n") + except Exception as e: + master_log.write("SRTOOL:%s:DEFECT TABLE & JIRA ISSUES:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) + print("DATABASE UPDATES FAILED ... %s" % e) + return(1) + return(0) + +################################# +# main loop +# + +def main(argv): + global force_update + global verbose + global master_log + global srt_user + global srt_passwd + + parser = argparse.ArgumentParser(description='srtool_jira.py: Manage the SRTool to Jira connection') + parser.add_argument('--init-jira', '-i', action='store_const', const='init_jira', dest='command', help='Init and import Jira states and update defects') + parser.add_argument('--update-jira', '-u', action='store_const', const='update_jira', dest='command', help='Import Jira states and update defects') + parser.add_argument('--update-jira-list', '-l', dest='jira_update_list', help='List of Jira defects to update in SRTool database') + parser.add_argument('--jira_er', '-e', nargs=1, help='Query list of pending ERs under a project, review them, and assign to Rally themes') + parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force updates') + parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Verbose debugging') + parser.add_argument('--user', dest='user', help='User name for Jira access') + parser.add_argument('--passwd', dest='passwd', help='User password for Jira access') + + parser.add_argument('--add', nargs=1, help='Add an existing defect to SRTool defect database') + parser.add_argument('--delete', nargs=1, help='Delete an existing defect from SRTool defect database') + parser.add_argument('--new', action='store_const', const='new', dest='command', help='Create a new defect "--new --product-tags --summary --cve --description --reason --priority --components --link"') + parser.add_argument('--jira-projects', '-j', action='store_const', const='jira-projects', dest='command', help='Jira projects') + + # Be flexible with arguments to support sub-parse trees + args, argv = parser.parse_known_args() + + master_log = open("./update_logs/master_log.txt", "a") + + if get_override('SRTDBG_SKIP_DEFECT_IMPORT'): + print("...Skipping Defect import") + exit(0) + + # Authorization + if args.user: + srt_user = args.user + else: + srt_user = os.environ.get('SRT_USER') + if args.passwd: + srt_passwd = args.passwd + else: + srt_passwd = os.environ.get('SRT_PASSWD') + if not srt_user or not srt_passwd: + msg = "FATAL ERROR: Missing user/password for Jira access" + print(msg) + srt_error_log(msg) + return 1 + + force_update = False + if None != args.force_update: + force_update = args.force_update + if None != args.verbose: + verbose = True + if verbose: + srt_error_log("srtool_jira: NOTE unprocessed arguments: %s" % argv) + + jira_list = '' + if None != args.jira_update_list: + jira_list = args.jira_update_list + + ret = 0 + if args.jira_er: + ret = get_jira_er(args.jira_er[0]) + elif args.add: + ret = jira_add_to_defect_db(args.add[0]) + elif args.delete: + ret = jira_del_from_defect_db(args.add[0]) + elif 'new' == args.command: + # Instantiate a sub-parse tree for "new" specific arguments + # Example: + # $ ./bin/<app>/srtool_jira.py --new --product-tags '{"key":"MERRIE"}' --summary 'User error' \ + # --cve 'CVE-2019-0000' --description 'Incorrect equipement use' --reason 'beep' \ + # --priority High --components "acme" --link 'www.acme.com/cve/CVE-2019-0000' + new_parser = argparse.ArgumentParser(parents=[parser],add_help=False) + new_parser.add_argument('--product-tags','-T', dest='tags', help='Product tags for defect tool integration') + new_parser.add_argument('--summary','-S', help='Defect summary') + new_parser.add_argument('--cve','-C', help='CVE list') + new_parser.add_argument('--description','-D', help='Defect summary') + new_parser.add_argument('--reason','-R', help='Defect reason hint') + new_parser.add_argument('--priority','-P', help='Defect priority') + new_parser.add_argument('--components','-O', help='Affected components') + new_parser.add_argument('--link','-L', help='Link to upstream CVE') + args, argv = new_parser.parse_known_args() + if verbose: + srt_error_log("SRTool_Jira:NEW: NOTE unprocessed arguments: %s" % argv) + jira_new_defect( + args.tags if args.tags else '{}', + args.summary if args.summary else '', + args.cve if args.cve else '', + args.description if args.description else '', + args.reason if args.reason else '', + args.priority if args.priority else '', + args.components if args.components else '', + args.link if args.link else '', + ) + elif 'init_jira' == args.command: + ret = update_jira(True) + elif 'update_jira' == args.command: + ret = update_jira(False) + elif jira_list: + ret = jira_update_list(jira_list) + elif 'jira-projects' == args.command: + ret = get_projects() + else: + print("Command not found") + + if 0 != ret: + exit(ret) + + +if __name__ == '__main__': + if verbose: print("srtool_jira(%s)" % sys.argv[1:]) + + # fetch any environment overrides + set_override('SRTDBG_SKIP_DEFECT_IMPORT') + set_override('SRTDBG_MINIMAL_DB') + + srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) + main(sys.argv[1:]) diff --git a/bin/acme/srtool_jira.py b/bin/common/srtool_jira_template.py index f9b8103b..5f29355d 100755..100644 --- a/bin/acme/srtool_jira.py +++ b/bin/common/srtool_jira_template.py @@ -5,7 +5,7 @@ # # Security Response Tool Commandline Tool # -# Copyright (C) 2018 Wind River Systems +# Copyright (C) 2018-2019 Wind River Systems # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -21,33 +21,29 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. ### Usage Examples (run from top level directory) -# Updating Jira Issues: ./bin/srtool_jira.py -U - - -### -### THIS IS A SAMPLE JIRA INTEGRATION SCRIPT FOR -### MANAGING AN ORGANIZATION'S SRTOOL INTEGRATION -### -### INSTALLATION INSTRUCTIONS AND DOCUMENATION OF THE JIRA PYTHON -### LIBRARIES CAN BE FOUND HERE: -### https://jira.readthedocs.io -### - +# Updating Jira Issues: ./bin/<app>/srtool_jira<_app>.py -u + +## +## THIS IS A JIRA INTEGRATION SCRIPT FOR +## MANAGING AN ORGANIZATION'S SRTOOL INTEGRATION +## +## You can use the "bin/common/srtool_patcher.py" to extend this template +## with your custom changes yet keep in sync with the shared upstream file. +## +## See the example: "bin/acme/srtool_jira_acme.py" +## +## INSTALLATION INSTRUCTIONS AND DOCUMENATION OF THE JIRA PYTHON +## LIBRARIES CAN BE FOUND HERE: +## https://jira.readthedocs.io +## import os import sys import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -from jira import JIRA -from time import sleep from datetime import datetime, date -from urllib.request import urlopen, URLError -from urllib.parse import urlparse # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) @@ -55,35 +51,52 @@ sys.path.insert(0, dir_path) from common.srt_schema import ORM # Setup: +master_log = '' +srt_user = '' +srt_passwd = '' +force_update = False + srtDbName = 'srt.sqlite' srtErrorLog = 'srt_errors.txt' -### -### THESE ARE SAMPLE JIRA INTEGRATION LINKS: -### The 'production' link is to the main Jira server -### The 'test' link is to a test Jira server, for example to pre-test defect creation -### The 'browse' link is how to format the Jira defect lookup link that is passed to the SRTool users -### +# +# Load Jira test +# -# Jira constants -JIRA_PRODUCTION_LINK = 'https://jira.acme.com' -JIRA_BROWSE_LINK_FORMAT = 'http://jira.acme.com/browse/%s' -JIRA_TESTSERVER_LINK = 'https://jira.acmetest.com' +try: + from jira import JIRA +except: + # could be SRTDBG_SKIP_DEFECT_IMPORT + pass -### -### THESE ARE SAMPLE JIRA CUSTOM FIELD LINKS: -### These are example links that you can use to define and populate the respective fields for the SRTool -### These are not defined in the standard Jira schema, and would presumably be custom added -### You can extend and cusomize the Jira integration to SRTool by added additional custom fields using this model -### The two indicated fields below are part of the SRTool design. They can be present as empty strings if they -### do not exist in your schema: -### The 'PUBLISHED' field is used to indicate if and when a Jira defect has been published to the customers -### The 'FIX_VERSION' field is used to indicate which version/release/update/service pack of the product has the fix -### +## +## THESE ARE JIRA INTEGRATION LINKS: +## The 'production' link is to the main Jira server +## The 'test' link is to a test Jira server, for example to pre-test defect creation +## The 'browse' link is how to format the Jira defect lookup link that is passed to the SRTool users +## + +# Jira constants: examples +JIRA_PRODUCTION_LINK = 'https://jira.sample.com' +JIRA_BROWSE_LINK_FORMAT = 'http://jira.sample.com/browse/%s' +JIRA_TESTSERVER_LINK = 'https://jira.sampletest.com' +JIRA_NEW_ASSIGNEE = 'kilroy' + +## +## THESE ARE SAMPLE JIRA CUSTOM FIELD LINKS: +## These are example links that you can use to define and populate the respective fields for the SRTool +## These are not defined in the standard Jira schema, and would presumably be custom added +## You can extend and cusomize the Jira integration to SRTool by added additional custom fields using this model +## The two indicated fields below are part of the SRTool design. They can be present as empty strings if they +## do not exist in your schema: +## The 'PUBLISHED' field is used to indicate if and when a Jira defect has been published to the customers +## The 'FIX_VERSION' field is used to indicate which version/release/update/service pack of the product has the fix +## # Custom Jira fields -JIRA_PUBLISHED_FIELD = 'customfield_10010' ### REPLACE WITH YOUR CUSTOM FIELD VALUE -JIRA_FIX_VERSION_FIELD = 'customfield_10011' ### REPLACE WITH YOUR CUSTOM FIELD VALUE +# Custom Jira fields +JIRA_PUBLISHED_FIELD = 'customfield_10001' ### REPLACE WITH YOUR CUSTOM FIELD VALUE +JIRA_FIX_VERSION_FIELD = 'customfield_10002' ### REPLACE WITH YOUR CUSTOM FIELD VALUE # ... ################################# @@ -98,14 +111,14 @@ def debugMsg(msg): overrides = {} -def set_override(key,value=None): +def set_override(key,value=None,quite=False): if not value is None: overrides[key] = value elif key in os.environ.keys(): overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no' else: overrides[key] = 'no' - if 'yes' == overrides[key]: + if 'yes' == overrides[key] and not quite: print("OVERRIDE: %s = %s" % (key,overrides[key])) def get_override(key): @@ -126,47 +139,38 @@ def get_tag_key(tag,key): # class to hold fields of a Jira issues # -class Defect: +class Defect(): project = '' product_id = '' - - def __init__(self,*args, **kwargs): - self.reset() - - # reset all but persistent project info - def reset(self): - self.id = -1 - self.name = '' - self.summary = '' - self.url = '' - self.priority = -1 - self.status = '' - self.resolution = 'Unresolved' - self.publish = '' - - # RCPL - self.release_version = '' - - self.date_created = '' - self.date_updated = '' - - # extra fields - self.cve_status = '' - self.vi_status = '' - self.vi_outcome = '' + id = -1 + name = '' + summary = '' + url = '' + priority = -1 + status = '' + resolution = 'Unresolved' + publish = '' + # RCPL + release_version = '' + date_created = '' + date_updated = '' + # extra fields + cve_status = '' + vi_status = '' + vi_outcome = '' ################################# -# Import Jira states +# import Jira states # -# if too slow, change to check update times and ignore those that need nothing -# can also move parsing JSON so that it doesnt happen if record is up to date +#if too slow, change to check update times and ignore those that need nothing (should do anyway to be honest...) +#can also move parsing JSON so that it doesnt happen if record is up to date def do_update_jira(): try: jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) except Exception as e: print("CONNECTION TO JIRA FAILED") - return + return 1 conn = sqlite3.connect(srtDbName) c = conn.cursor() @@ -175,14 +179,14 @@ def do_update_jira(): weeknum = today.strftime("%W") weekday = today.isoweekday() - log = open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a") + update_log = open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a") - log.write("BEGINNING JIRA UPDATES\n") + update_log.write("BEGINNING JIRA UPDATES\n") pre_update_time = datetime.now() products = c.execute('''SELECT * FROM orm_product''').fetchall() for i,product in enumerate(products): -# print("FOO1:%s,%s" % (product[ORM.PRODUCT_DEFECT_TAGS],get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key'))) +# print("JIRA_PRODUCT_SCAN:%s,%s" % (product[ORM.PRODUCT_DEFECT_TAGS],get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key'))) product_defect_prefix = get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key') if get_override('SRTDBG_MINIMAL_DB') and (i > 1): @@ -190,14 +194,13 @@ def do_update_jira(): #specify which fields to get in order to speed up request! #print("\tupdating ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\tloading " + spinner[block_num % 3], end='\r', flush=True) - log.write("\tUPDATING ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\n") + update_log.write("\tUPDATING ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\n") block_size = 500 block_num = 0 spinner = [' ', '. ', '.. ', '...'] while True: print("\tloading" + spinner[block_num % 4] + "\t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], end='\r', flush=True) -# print("\tloading" + spinner[block_num % 4] + "\t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], flush=True) start_idx = block_num*block_size #searches current project's bug issues that contain "cve" in their text issues = jira.search_issues('project=%s AND text ~ "cve*" AND type = Bug' % product_defect_prefix, start_idx, block_size, False, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD)) @@ -206,26 +209,28 @@ def do_update_jira(): break # Development support block_num += 1 - update_project_issues(product, issues, conn, log) + update_project_issues(product, issues, conn, update_log) +# sleep(1.0) # give time for Sqlite to sync conn.commit() #commit to db after each block +# sleep(1.0) # give time for Sqlite to sync +# conn.commit() #commit to db after each product print("\tfinished \t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], flush=True) conn.commit() - log.write("began updates: %s\n" % str(pre_update_time)) - log.write("finished updates: %s\n" % str(datetime.now())) - log.write("=============================================================================\n") - log.write("\n") + update_log.write("began updates: %s\n" % str(pre_update_time)) + update_log.write("finished updates: %s\n" % str(datetime.now())) + update_log.write("=============================================================================\n") + update_log.write("\n") # Reset datasource's lastModifiedDate as today sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE name='Jira'" date_string = datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT) - c.execute(sql, (date_string,) ) + ret = c.execute(sql, (date_string,) ) conn.commit() c.close() conn.close() #############################################################################3 -### Update the defect status from the Jira database ### def new_vulnerability_name(cur): @@ -254,10 +259,6 @@ def new_investigation_name(cur): cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID])) return "INV-%05d" % index -# -# Translate Jira values to SRTool values -# - def translate_priority(j,p): NONE = 0 MINOR = 1 @@ -279,6 +280,21 @@ def translate_priority(j,p): srt_error_log("ERROR: unknown priority string '%s=%s'" % (j,p)) return '0' +def translate_priority_srt_to_jira(s): + Priority = ( + ('Undefined', 'None'), + ('Minor', 'P4'), + ('Low', 'P3'), + ('Medium', 'P2'), + ('High', 'P1'), + ) + for i in range(len(Priority)): + if s == Priority[i][0]: + return str(Priority[i][1]) + print("ERROR: unknown priority string '%s'" % (s)) + srt_error_log("ERROR: unknown priority string '%s'" % (s)) + return 'None' + def translate_status(j,s): OPEN = 0 IN_PROGRESS = 1 @@ -301,7 +317,7 @@ def translate_status(j,s): srt_error_log("ERROR: unknown status string '%s=%s'" % (j,s)) return '0' -def translate_resolution(j,r): +def translate_resolution(j,r,log): Resolution = ( (ORM.DEFECT_UNRESOLVED, 'Unresolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_OPEN), (ORM.DEFECT_RESOLVED, 'Resolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_FIXED), @@ -322,10 +338,6 @@ def translate_resolution(j,r): log.write("ERROR: unknown resolution string '%s=%s'" % (j,r)) return 0,0,0,0 -# -# Update the defect status from the Jira database -# - #handles updating a list of issues for a single product/project #DOES NOT CALL COMMIT (should change this?) def update_project_issues(project, issues, conn, log): @@ -347,11 +359,10 @@ def update_project_issues(project, issues, conn, log): USER_SRTOOL_ID = "SRTool" - d = Defect() - d.project = project[ORM.PRODUCT_NAME] + " " + project[ORM.PRODUCT_VERSION] + " " + project[ORM.PRODUCT_PROFILE] - d.product_id = project[ORM.PRODUCT_ID] + project_str = project[ORM.PRODUCT_NAME] + " " + project[ORM.PRODUCT_VERSION] + " " + project[ORM.PRODUCT_PROFILE] + product_id = project[ORM.PRODUCT_ID] - cve_regex = re.compile("CVE-\d+-\d+") + cve_regex = re.compile(r"CVE-\d+-\d+") srtool_today = datetime.today().strftime('%Y-%m-%d') c = conn.cursor() @@ -362,7 +373,11 @@ def update_project_issues(project, issues, conn, log): break issue_json = issue.raw - d.reset() + + d = Defect() + d.project = project_str + d.product_id = product_id + d.name = issue_json['key'] d.date_updated = issue_json['fields']['updated'] d.date_created = issue_json['fields']['created'] @@ -372,17 +387,13 @@ def update_project_issues(project, issues, conn, log): d.status = translate_status(d.name,issue_json['fields']['status']['name']) if issue_json['fields']['resolution'] is not None: - d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name']) + d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log) else: - d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,'Unresolved') + d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,'Unresolved',log) if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None: d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value'] - else: - d.publish = '' if JIRA_FIX_VERSION_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_FIX_VERSION_FIELD] is not None: d.release_version = issue_json['fields'][JIRA_FIX_VERSION_FIELD]['name'] - else: - d.release_version = '' sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name defect = c.execute(sql).fetchone() @@ -444,6 +455,7 @@ def update_project_issues(project, issues, conn, log): # Also create CVE history entry sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)''' c.execute(sql, (c_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) +# sleep(0.1) else: c_id = cve[ORM.CVE_ID] @@ -467,6 +479,7 @@ def update_project_issues(project, issues, conn, log): # Also create Vulnerability history entry sql = '''INSERT INTO orm_vulnerabilityhistory (vulnerability_id, comment, date, author) VALUES (?,?,?,?)''' c.execute(sql, (v_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) +# sleep(0.1) else: print("FOUND VULNERABILITY ID for %s" % (cve_name)) v_id = c2v[ORM.CVETOVULNERABLILITY_VULNERABILITY_ID] @@ -493,6 +506,7 @@ def update_project_issues(project, issues, conn, log): # Also create Investigation history entry sql = '''INSERT INTO orm_investigationhistory (investigation_id, comment, date, author) VALUES (?,?,?,?)''' c.execute(sql, (i_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) +# sleep(0.1) else: print("FOUND INVESTIGATION ID for %s" % (cve_name)) i_id = investigation[ORM.INVESTIGATION_ID] @@ -506,6 +520,7 @@ def update_project_issues(project, issues, conn, log): log.write("\tINSERTING INVESTIGATION to DEFECT for %s\n" % i_name) sql = '''INSERT INTO orm_investigationtodefect (investigation_id, product_id, defect_id) VALUES (?,?,?)''' c.execute(sql, (i_id,d.product_id,defect_id)) +# sleep(0.1) #print("=========================================================================================\n") #print("\n") @@ -520,13 +535,13 @@ def jira_update_list(jira_list): jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) except Exception as e: print("CONNECTION TO JIRA FAILED") - return + return 1 conn = sqlite3.connect(srtDbName) c = conn.cursor() products = c.execute('''SELECT * FROM orm_product''').fetchall() - log = sys.stdout # open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a") + log = sys.stdout for jira_name in jira_list.split(','): print("Updating:\t" + jira_name, flush=True) @@ -546,6 +561,7 @@ def jira_update_list(jira_list): # Development support block_num += 1 update_project_issues(product, issues, conn, log) +# sleep(1.0) # give time for Sqlite to sync conn.commit() #commit to db after each block ################################# @@ -554,20 +570,19 @@ def jira_update_list(jira_list): #Gets all JIRA Enhancement Requests for the specified project def get_jira_er(project_prefix): - #log.write("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix) print ("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix) try: jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) except Exception as e: print("CONNECTION TO JIRA FAILED") - return + return 1 block_size = 100 block_num = 0 while True: start_idx = block_num*block_size #searches current project's bug issues that contain "cve" in their text - issues = jira.search_issues('project=%s AND type = "Enhancement Request"' % project_prefix, start_idx, block_size, False) #project argument could be better written I guess :) + issues = jira.search_issues('project=%s AND type = "Enhancement Request"' % project_prefix, start_idx, block_size, False) if len(issues) == 0: # Retrieve issues until there are no more to come break @@ -583,6 +598,9 @@ def jira_add_to_defect_db(jira_name): print("Jira_name=%s" % (jira_name)) jira_name = jira_name.strip().upper() + # Just error errors to terminal + log = sys.stdout + #try connecting to jira try: jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) @@ -610,11 +628,10 @@ def jira_add_to_defect_db(jira_name): d.priority = translate_priority(d.name,issue_json['fields']['priority']['name']) d.status = translate_status(d.name,issue_json['fields']['status']['name']) - if issue_json['fields']['resolution'] is not None: - d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name']) + d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log) else: - d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved') + d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved',log) if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None: d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value'] else: @@ -632,7 +649,7 @@ def jira_add_to_defect_db(jira_name): d.product_id = product[ORM.PRODUCT_ID] break - #log.write("\tINSERTING %s\n" % d.name) + #_log("\tINSERTING %s\n" % d.name) sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)''' c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated)) conn.commit() @@ -648,20 +665,15 @@ def jira_add_to_defect_db(jira_name): # Not yet implemented def jira_del_from_defect_db(jira_name): - pass + return 1 ################################# # New defect # -### -### You will need to fill in any required custom fields required -### to create new organization defects in Jira -### - # Use "jiratest" for development testing -IS_TEST = True -IS_SIMULATE = True +JIRA_IS_TEST = True +JIRA_IS_SIMULATE = True def simulate_new_defect_name(product_prefix): conn = sqlite3.connect(srtDbName) @@ -688,32 +700,17 @@ def simulate_new_defect(product_defect_tags,summary,cve_list,description,reason, defect_name = simulate_new_defect_name(product_defect_prefix) print("CREATED:%s,%s/browse/%s" % (defect_name,jira_url,defect_name)) -def translate_priority_srt_to_jira(s): - Priority = ( - ('Undefined', 'None'), - ('Minor', 'P4'), - ('Low', 'P3'), - ('Medium', 'P2'), - ('High', 'P1'), - ) - for i in range(len(Priority)): - if s == Priority[i][0]: - return str(Priority[i][1]) - print("ERROR: unknown priority string '%s'" % (s)) - srt_error_log("ERROR: unknown priority string '%s'" % (s)) - return 'None' - -def jira_new_defect(product_defect_tags,summary,description,priority,components,link): - #srt_error_log("NEW DEFECT:%s|%s|%s|%s|%s|%s|%s|%s" % (product_defect_tags,summary,cve_list,description,reason,priority,components,link)) +def jira_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link): + srt_error_log("NEW DEFECT:%s|%s|%s|%s|%s|%s|%s|%s" % (product_defect_tags,summary,cve_list,description,reason,priority,components,link)) - if IS_TEST: + if JIRA_IS_TEST: # Test URL jira_url = JIRA_TESTSERVER_LINK else: # Production URL jira_url = JIRA_PRODUCTION_LINK - if IS_SIMULATE: + if JIRA_IS_SIMULATE: return simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link,jira_url) # Connect to Jira server @@ -721,7 +718,8 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components, jira = JIRA(jira_url, auth=(srt_user, srt_passwd)) except Exception as e: print("CONNECTION TO JIRA FAILED") - return + return 1 + #srt_error_log("Jira connection made") conn = sqlite3.connect(srtDbName) c = conn.cursor() @@ -734,7 +732,7 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components, # Translate the SRTool priority to Jira priority priority = translate_priority_srt_to_jira(priority) - #print("JIRA_NEW_DEFECT:%s,%s,%s" % (product_defect_prefix,summary,description)) + #print("FOO1:%s,%s,%s" % (product_defect_prefix,summary,description)) jira_components=list() for component in components.split(' '): @@ -745,19 +743,19 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components, 'summary': summary, 'description': description, 'issuetype': {'name': 'Bug'}, - 'priority': {'name': priority}, - 'components': jira_components, # Components by name - 'assignee': {"name":SRT_USER }, # assign to the SRTool developer Jira account -### 'customfield_11000': 'unknown', # EXAMPLE: "Found In Versions" = "unknown" -### 'customfield_11001': {"value":"Review"}, # EXAMPLE: "Where Found" = "Review" + 'components': jira_components, # Components by name + 'assignee': {"name":JIRA_NEW_ASSIGNEE}, # assign to the SRTool developer for now } - if True: - new_issue = jira.create_issue(fields=issue_dict) - else: - print("CREATE:%s" % issue_dict) - new_issue = '%s-%s' % (product_defect_prefix,datetime.now().strftime('%H%M%S')) +## +## Add custom fields here, using the format... +## issue_dict['custom_field_10001'] = value +## + + new_issue = jira.create_issue(fields=issue_dict) + #print("CREATE:%s" % issue_dict) + #new_issue = '%s-%s' % (product_defect_prefix,datetime.now().strftime('%H%M%S')) new_issue = str(new_issue.key) if not new_issue.startswith(product_defect_prefix): @@ -767,12 +765,11 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components, def get_projects(): - try: jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) except Exception as e: print("CONNECTION TO JIRA FAILED") - return + return 1 # Get all projects viewable by anonymous users. projects = jira.projects() @@ -784,9 +781,6 @@ def get_projects(): # def update_jira(is_init): - if get_override('SRTDBG_SKIP_DEFECT_IMPORT'): - print("...Skipping Defect import") - exit(0) try: print("BEGINNING JIRA UPDATES PLEASE WAIT ... this can take some time") do_update_jira() @@ -795,6 +789,8 @@ def update_jira(is_init): except Exception as e: master_log.write("SRTOOL:%s:DEFECT TABLE & JIRA ISSUES:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) print("DATABASE UPDATES FAILED ... %s" % e) + return(1) + return(0) ################################# # main loop @@ -820,13 +816,17 @@ def main(argv): parser.add_argument('--add', nargs=1, help='Add an existing defect to SRTool defect database') parser.add_argument('--delete', nargs=1, help='Delete an existing defect from SRTool defect database') parser.add_argument('--new', action='store_const', const='new', dest='command', help='Create a new defect "--new --product-tags --summary --cve --description --reason --priority --components --link"') - parser.add_argument('--jira-projects', '-P', action='store_const', const='jira-projects', dest='command', help='Jira projects') + parser.add_argument('--jira-projects', '-j', action='store_const', const='jira-projects', dest='command', help='Jira projects') # Be flexible with arguments to support sub-parse trees args, argv = parser.parse_known_args() master_log = open("./update_logs/master_log.txt", "a") + if get_override('SRTDBG_SKIP_DEFECT_IMPORT'): + print("...Skipping Defect import") + exit(0) + # Authorization if args.user: srt_user = args.user @@ -854,12 +854,13 @@ def main(argv): if None != args.jira_update_list: jira_list = args.jira_update_list + ret = 0 if args.jira_er: - get_jira_er(args.jira_er[0]) + ret = get_jira_er(args.jira_er[0]) elif args.add: - jira_add_to_defect_db(args.add[0]) + ret = jira_add_to_defect_db(args.add[0]) elif args.delete: - jira_del_from_defect_db(args.add[0]) + ret = jira_del_from_defect_db(args.add[0]) elif 'new' == args.command: # Instantiate a sub-parse tree for "new" specific arguments # Example: @@ -877,7 +878,7 @@ def main(argv): new_parser.add_argument('--link','-L', help='Link to upstream CVE') args, argv = new_parser.parse_known_args() if verbose: - srt_error_log("SRTool_Jira_New: NOTE unprocessed arguments: %s" % argv) + srt_error_log("SRTool_Jira:NEW: NOTE unprocessed arguments: %s" % argv) jira_new_defect( args.tags if args.tags else '{}', args.summary if args.summary else '', @@ -889,23 +890,26 @@ def main(argv): args.link if args.link else '', ) elif 'init_jira' == args.command: - update_jira(True) + ret = update_jira(True) elif 'update_jira' == args.command: - update_jira(False) + ret = update_jira(False) elif jira_list: - jira_update_list(jira_list) + ret = jira_update_list(jira_list) elif 'jira-projects' == args.command: - get_projects() + ret = get_projects() else: print("Command not found") -if __name__ == '__main__': - global srtool_basepath + if 0 != ret: + exit(ret) + +if __name__ == '__main__': if verbose: print("srtool_jira(%s)" % sys.argv[1:]) # fetch any environment overrides - #set_override('SRTDBG_MINIMAL_DB') + set_override('SRTDBG_SKIP_DEFECT_IMPORT') + set_override('SRTDBG_MINIMAL_DB') srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/common/srtool_patcher.py b/bin/common/srtool_patcher.py new file mode 100755 index 00000000..6bf65755 --- /dev/null +++ b/bin/common/srtool_patcher.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Security Response Tool Implementation +# +# Copyright (C) 2019 Wind River Systems +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# +# Theory of operation +# +# * This script runs quick sanity tests +# * The '--init-test' option will: +# * Report the host package versions +# * Report errors if any required tables are empty +# * Report the running instance of the SRTool +# + +import os +import sys +import argparse +import json + +# Setup: +verbose = False + +################################# +# extract_custom_patch +# + +def extract_custom_patch(custom_file,clean_file,patch_file,label,patcher_dir): + tag_begin = "%s_EXTENSION_BEGIN" % label + tag_end = "%s_EXTENSION_END" % label + ret = 0 + + # Insure patcher working directory is ready + if not patcher_dir: + patcher_dir = os.path.dirname(custom_file) + try: + os.makedirs(patcher_dir) + except: + pass + # Prepare defaults + if not clean_file: + clean_file = os.path.join(patcher_dir,os.path.basename(custom_file) + '.clean') + if not patch_file: + patch_file = os.path.join(patcher_dir,os.path.basename(custom_file) + '.patch') + + with open(custom_file, 'r') as fs: + with open(clean_file, 'w') as fd: + state = "find" + for line in fs: + #print("LINE(%s):%s" % (state,line.strip())) + if state == "find": + if 0 < line.find(tag_begin): + #print("START:$line") + state = "found" + continue + elif state == "found": + if 0 < line.find(tag_end): + #print("STOP:$line") + state = "blank" + continue + elif state == "blank": + state = "find" + # if next line after stop is blank, hide that also + if not line.strip(): + continue + # Normal line? + if state != "found": + fd.write('%s' % line) + # Did we end cleanly? + if state != "find": + print("ERROR: START not STOPPED (%s)" % state) + os.system("diff -u %s %s > %s" % (clean_file,custom_file,patch_file)) + print("Custom File: %s" % (custom_file)) + print("Clean File: %s" % (clean_file)) + print("Patch File: %s" % (patch_file)) + return(ret,clean_file,patch_file) + +################################# +# merge_original +# + +def merge_original(custom_file,original_file,patch_file,label,patcher_dir): + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + custom_saved = os.path.join(os.path.dirname(patch_file),os.path.basename(custom_file) + '.saved') + if 0 == ret: + print("* Preserving custom file as '%s'" % custom_saved) + cmd = "cp %s %s" % (custom_file,custom_saved) + print(cmd) + os.system(cmd) + print("* Merging original file '%s' into custom file" % original_file) + cmd = "cp %s %s" % (original_file,custom_file) + print(cmd) + os.system(cmd) + cmd = "patch %s %s" % (custom_file,patch_file) + print(cmd) + ret = os.system(cmd) + if 0 != ret: + print("* ERROR: Merge failed, restoring previous custom file") + cmd = "cp %s %s" % (custom_saved,custom_file) + print(cmd) + os.system(cmd) + return(ret) + +################################# +# merge_custom +# + +def merge_custom(custom_file,original_file,patch_file,label,patcher_dir): + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + if 0 == ret: + print("== Copying clean version of custom file to '%s' ===" % original_file) + cmd = "cp -f %s %s" % (clean_file, original_file) + print(cmd) + os.system(cmd) + return(ret) + +################################# +# diff_original +# + +def diff_original(custom_file,original_file,patch_file,label,patcher_dir): + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + if 0 == ret: + print("== DIFF from '%s' to clean version of custom file ===" % original_file) + os.system("diff -u %s %s" % (original_file,clean_file)) + return(ret) + +################################# +# load_json_list +# + +def load_json_list(json_file): + label = 'CUSTOM' + patcher_dir = '' + file_list = [] + + with open(json_file) as json_data: + dct = json.load(json_data) + + if 'label' in dct: + label = dct['label'] + if 'patcher_dir' in dct: + patcher_dir = dct['patcher_dir'] + if 'patch_set' in dct: + for patch in dct['patch_set']: + if 'DISABLE' in patch['options']: + continue + file_list.append([patch['custom'],patch['original'],patch['patch'],patch['options']]) + return file_list,patcher_dir,label + +################################# +# main loop +# + +def main(argv): + global verbose + + # setup + parser = argparse.ArgumentParser(description='srtool_sanity_test.py: SRTool common sanity tests') + + parser.add_argument('--merge-original', '-O', action='store_const', const='merge_original', dest='command', help='Copy the (updated) original file, merge the custom patches') + parser.add_argument('--merge-custom', '-C', action='store_const', const='merge_custom', dest='command', help='Copy the (updated) file to the original, without custom patches') + + parser.add_argument('--custom', '-c', help='Custom file') + parser.add_argument('--original', '-o', help='Original file') + parser.add_argument('--label', '-l', help='Custom label tag (default="CUSTOM")') + parser.add_argument('--json', '-j', help='Use JSON file for file list') + + parser.add_argument('--extract-custom-patch', '-e', action='store_const', const='extract_custom_patch', dest='command', help='Extract a patch of the custom content') + parser.add_argument('--apply-custom-patch', '-a', action='store_const', const='extract_custom_patch', dest='command', help='Apply the custom patch to the (reset) custom file') + parser.add_argument('--patch', '-p', help='Patch file') + parser.add_argument('--diff-original', '-d', action='store_const', const='diff_original', dest='command', help='Show how the file compares to original, ignoring custom patches') + + parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output') + args = parser.parse_args() + + def validate_file(filename,msg): + if not (filename or os.path.isfile(filename)): + print("ERROR: %s file not found '%s'" % (msg,filename)) + exit(1) + + # Extract provided values + verbose = args.verbose + file_list = [] + if args.json: + validate_file(args.json,"JSON") + file_list,patcher_dir,label = load_json_list(args.json) + else: + custom_file = '' + if args.custom: + custom_file = args.custom + original_file = '' + if args.original: + original_file = args.original + patch_file = '' + if args.patch: + patch_file = args.patch + options = '' + file_list.append([custom_file,original_file,patch_file,options]) + label = 'CUSTOM' + if args.label: + label = args.label + patcher_dir = os.path.join(os.path.dirname(custom_file),'patcher') + + ret = 0 + for custom_file,original_file,patch_file,options in file_list: + #print("PATCH_FILE:%s,%s,%s,%s,%s" % (custom_file,original_file,patch_file,options,patcher_dir)) + if 'merge_original' == args.command: + validate_file(custom_file,'Custom') + validate_file(original_file,'Original') + ret = merge_original(custom_file,original_file,patch_file,label,patcher_dir) + elif 'merge_custom' == args.command: + validate_file(custom_file,'Custom') + validate_file(original_file,'Original') + ret = merge_custom(custom_file,original_file,patch_file,label,patcher_dir) + elif 'extract_custom_patch' == args.command: + validate_file(custom_file,'Custom') + ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir) + elif 'diff_original' == args.command: + validate_file(custom_file,'Custom') + validate_file(original_file,'Original') + ret = diff_original(custom_file,original_file,patch_file,label,patcher_dir) + else: + print("Command not found '%s'" % args.command) + ret = 1 + if 0 != ret: + exit(ret) + exit(ret) + +if __name__ == '__main__': + srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) + main(sys.argv[1:]) + + + |