aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Reyna <David.Reyna@windriver.com>2019-01-17 18:02:49 -0800
committerDavid Reyna <David.Reyna@windriver.com>2019-01-17 18:02:49 -0800
commitec4e3d12a6ff4b8e0ed9db3e9c94bbba15d0ea20 (patch)
tree3541d92ffc4820ca835b5a8f2e612bd45f3ae76d
parenta305be5e7e6c3e70b091226715d5d64c41dcf711 (diff)
downloadsrtool-ec4e3d12a6ff4b8e0ed9db3e9c94bbba15d0ea20.tar.gz
srtool-ec4e3d12a6ff4b8e0ed9db3e9c94bbba15d0ea20.tar.bz2
srtool-ec4e3d12a6ff4b8e0ed9db3e9c94bbba15d0ea20.zip
srtool: patcher, Jira template, Jira ACME
Add formal support for extending and customizing shared template code files. * The initial use case is a shared Jira integration file that partners can extend to their particular installation. A working example is provided in the ACME directory: upstream: "bin/common/srtool_jira_template.py" custom : "bin/acme/srtool_jira_acme.py" * The custom sections are blocked off with comment tags (e.g. ACME) ### ACME_EXTENSION_BEGIN ### ... ### ACME_EXTENSION_END ### * The './bin/common/srtool_patcher.py' tool provides the support, for example: To merge shared upstream code into the custom Jira script, run: $ ./bin/common/srtool_patcher.py -j bin/acme/patcher.json --merge-original To merge edits in script's common areas back to upstream, run: $ ./bin/common/srtool_patcher.py -j bin/acme/patcher.json --merge-custom Signed-off-by: David Reyna <David.Reyna@windriver.com>
-rwxr-xr-xbin/acme/patcher.json20
-rwxr-xr-xbin/acme/srtool_jira_acme.py944
-rw-r--r--[-rwxr-xr-x]bin/common/srtool_jira_template.py (renamed from bin/acme/srtool_jira.py)346
-rwxr-xr-xbin/common/srtool_patcher.py252
4 files changed, 1391 insertions, 171 deletions
diff --git a/bin/acme/patcher.json b/bin/acme/patcher.json
new file mode 100755
index 00000000..365990fa
--- /dev/null
+++ b/bin/acme/patcher.json
@@ -0,0 +1,20 @@
+{
+ "_comments_" : "Blank values indicate defaults",
+ "label" : "ACME",
+ "patcher_dir" : "bin/acme/patcher",
+ "patch_set" : [
+ {
+ "_comments_" : "The ACME custom version of the Jira integration script",
+ "original" : "bin/common/srtool_jira_template.py",
+ "custom" : "bin/acme/srtool_jira_acme.py",
+ "patch" : "",
+ "options" : ""
+ },
+ {
+ "original" : "bin/srt",
+ "custom" : "bin/srt",
+ "patch" : "",
+ "options" : "DISABLE"
+ }
+ ]
+}
diff --git a/bin/acme/srtool_jira_acme.py b/bin/acme/srtool_jira_acme.py
new file mode 100755
index 00000000..d7ef0507
--- /dev/null
+++ b/bin/acme/srtool_jira_acme.py
@@ -0,0 +1,944 @@
+#!/usr/bin/env python3
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Security Response Tool Commandline Tool
+#
+# Copyright (C) 2018-2019 Wind River Systems
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+### Usage Examples (run from top level directory)
+# Updating Jira Issues: ./bin/<app>/srtool_jira<_app>.py -u
+
+##
+## THIS IS A JIRA INTEGRATION SCRIPT FOR
+## MANAGING AN ORGANIZATION'S SRTOOL INTEGRATION
+##
+## You can use the "bin/common/srtool_patcher.py" to extend this template
+## with your custom changes yet keep in sync with the shared upstream file.
+##
+## See the example: "bin/acme/srtool_jira_acme.py"
+##
+## INSTALLATION INSTRUCTIONS AND DOCUMENATION OF THE JIRA PYTHON
+## LIBRARIES CAN BE FOUND HERE:
+## https://jira.readthedocs.io
+##
+
+### ACME_EXTENSION_BEGIN ###
+#
+# This is the ACME 'patcher' extension of:
+# 'bin/common/srtool_jira_template.py'
+#
+# To merge edits in common areas to upstream, run:
+# $ ./bin/common/srtool_patcher.py -j bin/acme/patcher.json --merge-custom
+# To merge upstream into this custom extension, run:
+# $ ./bin/common/srtool_patcher.py -j bin/acme/patcher.json --merge-original
+#
+### ACME_EXTENSION_END ###
+
+import os
+import sys
+import re
+import argparse
+import sqlite3
+import json
+from datetime import datetime, date
+
+# load the srt.sqlite schema indexes
+dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+sys.path.insert(0, dir_path)
+from common.srt_schema import ORM
+
+# Setup:
+master_log = ''
+srt_user = ''
+srt_passwd = ''
+force_update = False
+
+srtDbName = 'srt.sqlite'
+srtErrorLog = 'srt_errors.txt'
+
+#
+# Load Jira test
+#
+
+try:
+ from jira import JIRA
+except:
+ # could be SRTDBG_SKIP_DEFECT_IMPORT
+ pass
+
+##
+## THESE ARE JIRA INTEGRATION LINKS:
+## The 'production' link is to the main Jira server
+## The 'test' link is to a test Jira server, for example to pre-test defect creation
+## The 'browse' link is how to format the Jira defect lookup link that is passed to the SRTool users
+##
+
+# Jira constants: examples
+JIRA_PRODUCTION_LINK = 'https://jira.sample.com'
+JIRA_BROWSE_LINK_FORMAT = 'http://jira.sample.com/browse/%s'
+JIRA_TESTSERVER_LINK = 'https://jira.sampletest.com'
+JIRA_NEW_ASSIGNEE = 'kilroy'
+
+### ACME_EXTENSION_BEGIN ###
+JIRA_PRODUCTION_LINK = 'https://jira.wrs.com'
+JIRA_BROWSE_LINK_FORMAT = 'http://jira.wrs.com/browse/%s'
+JIRA_TESTSERVER_LINK = 'http://jiratest.wrs.com'
+JIRA_NEW_ASSIGNEE = 'dreyna'
+### ACME_EXTENSION_END ###
+
+##
+## THESE ARE SAMPLE JIRA CUSTOM FIELD LINKS:
+## These are example links that you can use to define and populate the respective fields for the SRTool
+## These are not defined in the standard Jira schema, and would presumably be custom added
+## You can extend and cusomize the Jira integration to SRTool by added additional custom fields using this model
+## The two indicated fields below are part of the SRTool design. They can be present as empty strings if they
+## do not exist in your schema:
+## The 'PUBLISHED' field is used to indicate if and when a Jira defect has been published to the customers
+## The 'FIX_VERSION' field is used to indicate which version/release/update/service pack of the product has the fix
+##
+
+# Custom Jira fields
+# Custom Jira fields
+JIRA_PUBLISHED_FIELD = 'customfield_10001' ### REPLACE WITH YOUR CUSTOM FIELD VALUE
+JIRA_FIX_VERSION_FIELD = 'customfield_10002' ### REPLACE WITH YOUR CUSTOM FIELD VALUE
+# ...
+
+### ACME_EXTENSION_BEGIN ###
+JIRA_PUBLISHED_FIELD = 'customfield_10010'
+JIRA_FIX_VERSION_FIELD = 'customfield_11002'
+### ACME_EXTENSION_END ###
+
+#################################
+# Helper methods
+#
+
+verbose = False
+
+def debugMsg(msg):
+ if verbose:
+ print(msg)
+
+overrides = {}
+
+def set_override(key,value=None,quite=False):
+ if not value is None:
+ overrides[key] = value
+ elif key in os.environ.keys():
+ overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no'
+ else:
+ overrides[key] = 'no'
+ if 'yes' == overrides[key] and not quite:
+ print("OVERRIDE: %s = %s" % (key,overrides[key]))
+
+def get_override(key):
+ if key in overrides.keys():
+ return 'yes' == overrides[key]
+ return False
+
+def srt_error_log(msg):
+ f1=open(srtErrorLog, 'a')
+ f1.write("|" + msg + "|\n" )
+ f1.close()
+
+def get_tag_key(tag,key):
+ d = json.loads(tag)
+ return d[key]
+
+#################################
+# class to hold fields of a Jira issues
+#
+
+class Defect():
+ project = ''
+ product_id = ''
+ id = -1
+ name = ''
+ summary = ''
+ url = ''
+ priority = -1
+ status = ''
+ resolution = 'Unresolved'
+ publish = ''
+ # RCPL
+ release_version = ''
+ date_created = ''
+ date_updated = ''
+ # extra fields
+ cve_status = ''
+ vi_status = ''
+ vi_outcome = ''
+
+#################################
+# import Jira states
+#
+#if too slow, change to check update times and ignore those that need nothing (should do anyway to be honest...)
+#can also move parsing JSON so that it doesnt happen if record is up to date
+
+def do_update_jira():
+ try:
+ jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
+ except Exception as e:
+ print("CONNECTION TO JIRA FAILED")
+ return 1
+
+ conn = sqlite3.connect(srtDbName)
+ c = conn.cursor()
+
+ today = datetime.today()
+ weeknum = today.strftime("%W")
+ weekday = today.isoweekday()
+
+ update_log = open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a")
+
+ update_log.write("BEGINNING JIRA UPDATES\n")
+ pre_update_time = datetime.now()
+
+ products = c.execute('''SELECT * FROM orm_product''').fetchall()
+ for i,product in enumerate(products):
+# print("JIRA_PRODUCT_SCAN:%s,%s" % (product[ORM.PRODUCT_DEFECT_TAGS],get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')))
+ product_defect_prefix = get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')
+
+ if get_override('SRTDBG_MINIMAL_DB') and (i > 1):
+ break
+
+ #specify which fields to get in order to speed up request!
+ #print("\tupdating ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\tloading " + spinner[block_num % 3], end='\r', flush=True)
+ update_log.write("\tUPDATING ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\n")
+
+ block_size = 500
+ block_num = 0
+ spinner = [' ', '. ', '.. ', '...']
+ while True:
+ print("\tloading" + spinner[block_num % 4] + "\t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], end='\r', flush=True)
+ start_idx = block_num*block_size
+ #searches current project's bug issues that contain "cve" in their text
+ issues = jira.search_issues('project=%s AND text ~ "cve*" AND type = Bug' % product_defect_prefix, start_idx, block_size, False, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD))
+ if len(issues) == 0:
+ # Retrieve issues until there are no more to come
+ break
+ # Development support
+ block_num += 1
+ update_project_issues(product, issues, conn, update_log)
+# sleep(1.0) # give time for Sqlite to sync
+ conn.commit() #commit to db after each block
+# sleep(1.0) # give time for Sqlite to sync
+# conn.commit() #commit to db after each product
+ print("\tfinished \t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], flush=True)
+ conn.commit()
+ update_log.write("began updates: %s\n" % str(pre_update_time))
+ update_log.write("finished updates: %s\n" % str(datetime.now()))
+ update_log.write("=============================================================================\n")
+ update_log.write("\n")
+
+ # Reset datasource's lastModifiedDate as today
+ sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE name='Jira'"
+ date_string = datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT)
+ ret = c.execute(sql, (date_string,) )
+ conn.commit()
+
+ c.close()
+ conn.close()
+
+#############################################################################3
+###
+
+def new_vulnerability_name(cur):
+ sql = "SELECT * FROM orm_srtsetting WHERE name='current_vulnerability_index'"
+ cvi = cur.execute(sql).fetchone()
+ if not cvi:
+ index = 100
+ sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)'''
+ cur.execute(sql, ('current_vulnerability_index', '', index))
+ else:
+ index = int(cvi[ORM.SRTSETTING_VALUE]) + 1
+ sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?'''
+ cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID]))
+ return "VUL-%05d" % index
+
+def new_investigation_name(cur):
+ sql = "SELECT * FROM orm_srtsetting WHERE name='current_investigation_index'"
+ cvi = cur.execute(sql).fetchone()
+ if not cvi:
+ index = 100
+ sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)'''
+ cur.execute(sql, ('current_investigation_index', '', index))
+ else:
+ index = int(cvi[ORM.SRTSETTING_VALUE]) + 1
+ sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?'''
+ cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID]))
+ return "INV-%05d" % index
+
+def translate_priority(j,p):
+ NONE = 0
+ MINOR = 1
+ LOW = 2
+ MEDIUM = 3
+ HIGH = 4
+ Priority = (
+ (NONE, 'None'),
+ (MINOR, 'P4'),
+ (LOW, 'P3'),
+ (MEDIUM, 'P2'),
+ (HIGH, 'P1'),
+ )
+
+ for i in range(len(Priority)):
+ if p == Priority[i][1]:
+ return str(Priority[i][0])
+ print("ERROR: unknown priority string '%s=%s'" % (j,p))
+ srt_error_log("ERROR: unknown priority string '%s=%s'" % (j,p))
+ return '0'
+
+def translate_priority_srt_to_jira(s):
+ Priority = (
+ ('Undefined', 'None'),
+ ('Minor', 'P4'),
+ ('Low', 'P3'),
+ ('Medium', 'P2'),
+ ('High', 'P1'),
+ )
+ for i in range(len(Priority)):
+ if s == Priority[i][0]:
+ return str(Priority[i][1])
+ print("ERROR: unknown priority string '%s'" % (s))
+ srt_error_log("ERROR: unknown priority string '%s'" % (s))
+ return 'None'
+
+def translate_status(j,s):
+ OPEN = 0
+ IN_PROGRESS = 1
+ ON_HOLD = 2
+ CHECKED_IN = 3
+ RESOLVED = 4
+ CLOSED = 5
+ Status = (
+ (OPEN, 'Open'),
+ (IN_PROGRESS, 'In progress'),
+ (ON_HOLD, 'On Hold'),
+ (CHECKED_IN, 'Checked In'),
+ (RESOLVED, 'Resolved'),
+ (CLOSED, 'Closed'),
+ )
+ for i in range(len(Status)):
+ if s == Status[i][1]:
+ return str(Status[i][0])
+ print("ERROR: unknown status string '%s=%s'" % (j,s))
+ srt_error_log("ERROR: unknown status string '%s=%s'" % (j,s))
+ return '0'
+
+def translate_resolution(j,r,log):
+ Resolution = (
+ (ORM.DEFECT_UNRESOLVED, 'Unresolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_OPEN),
+ (ORM.DEFECT_RESOLVED, 'Resolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_FIXED),
+ (ORM.DEFECT_FIXED, 'Fixed', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_FIXED),
+ (ORM.DEFECT_WILL_NOT_FIX, 'Won\'t Fix', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_NOT_FIX),
+ (ORM.DEFECT_WITHDRAWN, 'Withdrawn', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED),
+ (ORM.DEFECT_REJECTED, 'Rejected', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED),
+ (ORM.DEFECT_DUPLICATE, 'Duplicate', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED),
+ (ORM.DEFECT_NOT_APPLICABLE, 'Not Applicable', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED),
+ (ORM.DEFECT_REPLACED_BY_REQUIREMENT, 'Replaced By Requirement',ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_CLOSED),
+ (ORM.DEFECT_CANNOT_REPRODUCE, 'Cannot Reproduce', ORM.STATUS_NOT_VULNERABLE,ORM.STATUS_VULNERABLE,ORM.OUTCOME_CLOSED),
+ (ORM.DEFECT_DONE, 'Done', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_CLOSED),
+ )
+ for i in range(len(Resolution)):
+ if r == Resolution[i][1]:
+ return Resolution[i][0],Resolution[i][2],Resolution[i][3],Resolution[i][4],
+ print("ERROR: unknown resolution string '%s=%s'" % (j,r))
+ log.write("ERROR: unknown resolution string '%s=%s'" % (j,r))
+ return 0,0,0,0
+
+#handles updating a list of issues for a single product/project
+#DOES NOT CALL COMMIT (should change this?)
+def update_project_issues(project, issues, conn, log):
+ global force_update
+
+ #CREATE TABLE "orm_defect" (
+ #0 "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT
+ #1 "name" varchar(50) NOT NULL
+ #2 "summary" text NOT NULL
+ #3 "url" text NOT NULL
+ #4 "priority" integer NOT NULL
+ #5 "status" integer NOT NULL
+ #6 "resolution" integer NOT NULL
+ #7 "publish" text NOT NULL
+ #8 "release_version" varchar(50) NOT NULL
+ #9 "product_id" integer NULL REFERENCES "orm_product" ("id")
+ #10 "date_created" varchar(50) NOT NULL
+ #11 "date_updated" varchar(50) NOT NULL);
+
+ USER_SRTOOL_ID = "SRTool"
+
+ project_str = project[ORM.PRODUCT_NAME] + " " + project[ORM.PRODUCT_VERSION] + " " + project[ORM.PRODUCT_PROFILE]
+ product_id = project[ORM.PRODUCT_ID]
+
+ cve_regex = re.compile(r"CVE-\d+-\d+")
+ srtool_today = datetime.today().strftime('%Y-%m-%d')
+
+ c = conn.cursor()
+ d_cursor = conn.cursor()
+ for i,issue in enumerate(issues):
+
+ if get_override('SRTDBG_MINIMAL_DB') and (i > 10):
+ break
+
+ issue_json = issue.raw
+
+ d = Defect()
+ d.project = project_str
+ d.product_id = product_id
+
+ d.name = issue_json['key']
+ d.date_updated = issue_json['fields']['updated']
+ d.date_created = issue_json['fields']['created']
+ d.summary = issue_json['fields']['summary']
+ d.url = JIRA_BROWSE_LINK_FORMAT % d.name
+ d.priority = translate_priority(d.name,issue_json['fields']['priority']['name'])
+ d.status = translate_status(d.name,issue_json['fields']['status']['name'])
+
+ if issue_json['fields']['resolution'] is not None:
+ d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log)
+ else:
+ d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,'Unresolved',log)
+ if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None:
+ d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value']
+ if JIRA_FIX_VERSION_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_FIX_VERSION_FIELD] is not None:
+ d.release_version = issue_json['fields'][JIRA_FIX_VERSION_FIELD]['name']
+
+ sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name
+ defect = c.execute(sql).fetchone()
+
+ #if defect does not exists then create it, if defect is out of date then update the database record, else ignore
+ if defect is None:
+ log.write("\tINSERTING %s\n" % d.name)
+ sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated, srt_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)'''
+ c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated, datetime.now()))
+ # Get the new id
+ sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name
+ defect = c.execute(sql).fetchone()
+ defect_id = defect[ORM.DEFECT_ID]
+ elif force_update or (d.date_updated > defect[ORM.DEFECT_DATE_UPDATED]):
+ log.write("\tUPDATING %s\n" % d.name)
+ sql = '''UPDATE orm_defect SET summary=?, priority=?, status=?, resolution=?, publish=?, release_version=?, date_updated=? WHERE id = ?'''
+ c.execute(sql, (d.summary, d.priority, d.status, d.resolution, d.publish, d.release_version, d.date_updated, defect[ORM.DEFECT_ID]))
+ defect_id = defect[ORM.DEFECT_ID]
+ else:
+ log.write("\tSKIPPING %s\n" % d.name)
+ continue
+
+ #
+ # Update CVE -> Vulnerability -> Investigation -> this Defect chain
+ #
+
+ # V/I priority and status minimum from defect
+ # Add audit lines
+
+ # Find parent CVE
+ m = cve_regex.search(d.summary)
+ if m:
+ cve_name = m.group(0)
+ else:
+ print("WARNING: Missing CVE in defect name '%s'" % (d.summary))
+ continue
+ sql = "SELECT * FROM orm_cve WHERE name='%s'" % cve_name
+ cve = c.execute(sql).fetchone()
+ if not cve:
+ # create the placeholder CVE
+ log.write("\tINSERTING CVE for %s\n" % cve_name)
+ print("INSERTING CVE for %s,%s" % (cve_name,d.name))
+
+ try:
+ a = cve_name.split('-')
+ cve_name_sort = '%s-%s-%07d' % (a[0],a[1],int(a[2]))
+ except:
+ cve_name_sort = cve.name
+
+ sql = ''' INSERT into orm_cve (name, name_sort, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity, packages, srt_updated)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'''
+ c.execute(sql, (cve_name, cve_name_sort, d.priority, cve_status, '', '', '', '', '', 1, 0, '', 'Created from defect %s' % d.name, '', '', 0, '', '', '', '', '', '', datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT)))
+
+ # Find the new id
+ print("FINDING CVE ID for %s\n" % cve_name)
+ c.execute("SELECT * FROM orm_cve where name = '%s'" % cve_name)
+ cve = c.fetchone()
+ c_id = cve[ORM.CVE_ID]
+ # Also create CVE history entry
+ sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)'''
+ c.execute(sql, (c_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID))
+# sleep(0.1)
+ else:
+ c_id = cve[ORM.CVE_ID]
+
+ # Find CVE's vulnerability, else create
+ d_cursor.execute("SELECT * FROM orm_cvetovulnerablility where cve_id = '%s'" % c_id)
+ c2v = d_cursor.fetchone()
+ if not c2v:
+ # Create Vulnerability
+ v_name = new_vulnerability_name(d_cursor)
+ log.write("\tINSERTING VULNERABILITY %s for %s\n" % (v_name,cve_name))
+ print("INSERTING VULNERABILITY for (%s,%s)" % (cve_name,v_name))
+ sql = '''INSERT INTO orm_vulnerability (name,description,cve_primary_name,public,comments,comments_private,status,outcome,priority) VALUES (?,?,?,?,?,?,?,?,?)'''
+ c.execute(sql, (v_name, cve[ORM.CVE_DESCRIPTION], cve_name, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome ,d.priority))
+ # Find the new id
+ d_cursor.execute("SELECT * FROM orm_vulnerability where name = '%s'" % v_name)
+ v = d_cursor.fetchone()
+ v_id = v[ORM.VULNERABILITY_ID]
+ # Also create CVE to Vulnerability
+ sql = '''INSERT INTO orm_cvetovulnerablility (vulnerability_id, cve_id) VALUES (?,?)'''
+ c.execute(sql, (v_id,c_id))
+ # Also create Vulnerability history entry
+ sql = '''INSERT INTO orm_vulnerabilityhistory (vulnerability_id, comment, date, author) VALUES (?,?,?,?)'''
+ c.execute(sql, (v_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID))
+# sleep(0.1)
+ else:
+ print("FOUND VULNERABILITY ID for %s" % (cve_name))
+ v_id = c2v[ORM.CVETOVULNERABLILITY_VULNERABILITY_ID]
+
+ # Find CVE's investigation, else create
+ sql = "SELECT * FROM orm_investigation where vulnerability_id = '%s' AND product_id = '%s';" % (v_id,d.product_id)
+# print("I_TEST:%s" % sql)
+ d_cursor.execute(sql)
+ investigation = d_cursor.fetchone()
+ if not investigation:
+ # Create Investigation
+ i_name = new_investigation_name(d_cursor)
+ log.write("\tINSERTING INVESTIGATION for %s\n" % cve_name)
+ print("INSERTING INVESTIGATION for %s,%s" % (i_name,d.name))
+ sql = '''INSERT INTO orm_investigation (name,vulnerability_id,product_id,public,comments,comments_private,status,outcome,priority) VALUES (?,?,?,?,?,?,?,?,?)'''
+ c.execute(sql, (i_name, v_id, d.product_id, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome, d.priority))
+ # Find the new id
+ d_cursor.execute("SELECT * FROM orm_investigation where name = '%s'" % i_name)
+ investigation = d_cursor.fetchone()
+ i_id = investigation[ORM.INVESTIGATION_ID]
+ # Also create Vulnerability to Investigation
+ sql = '''INSERT INTO orm_vulnerabilitytoinvestigation (vulnerability_id, investigation_id) VALUES (?,?)'''
+ c.execute(sql, (v_id,i_id))
+ # Also create Investigation history entry
+ sql = '''INSERT INTO orm_investigationhistory (investigation_id, comment, date, author) VALUES (?,?,?,?)'''
+ c.execute(sql, (i_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID))
+# sleep(0.1)
+ else:
+ print("FOUND INVESTIGATION ID for %s" % (cve_name))
+ i_id = investigation[ORM.INVESTIGATION_ID]
+
+ # Add this defect to the investigation
+ d_cursor.execute("SELECT * FROM orm_investigationtodefect where investigation_id = '%s' and product_id = '%s' and defect_id = '%s'" % (i_id,d.product_id,defect_id))
+ i2d = d_cursor.fetchone()
+ if not i2d:
+ # Create Investigation
+ i_name = new_investigation_name(d_cursor)
+ log.write("\tINSERTING INVESTIGATION to DEFECT for %s\n" % i_name)
+ sql = '''INSERT INTO orm_investigationtodefect (investigation_id, product_id, defect_id) VALUES (?,?,?)'''
+ c.execute(sql, (i_id,d.product_id,defect_id))
+# sleep(0.1)
+
+ #print("=========================================================================================\n")
+ #print("\n")
+ c.close()
+
+#################################
+# Jira list update
+#
+
+def jira_update_list(jira_list):
+ try:
+ jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
+ except Exception as e:
+ print("CONNECTION TO JIRA FAILED")
+ return 1
+
+ conn = sqlite3.connect(srtDbName)
+ c = conn.cursor()
+
+ products = c.execute('''SELECT * FROM orm_product''').fetchall()
+ log = sys.stdout
+
+ for jira_name in jira_list.split(','):
+ print("Updating:\t" + jira_name, flush=True)
+ # find the matching parent project
+ for i,product in enumerate(products):
+ product_defect_prefix = get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')
+ print("\tfrom:\t%s %s %s %s" %(product[ORM.PRODUCT_NAME],product[ORM.PRODUCT_VERSION],product[ORM.PRODUCT_PROFILE],product_defect_prefix), flush=True)
+ block_size = 500
+ block_num = 0
+ while True:
+ start_idx = block_num*block_size
+ #searches current project's bug issues that contain "cve" in their text
+ issues = jira.search_issues('project=%s AND text ~ "%s" AND type = Bug' % (product_defect_prefix,jira_name), start_idx, block_size, False, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD)) #project argument could be better written I guess :)
+ if len(issues) == 0:
+ # Retrieve issues until there are no more to come
+ break
+ # Development support
+ block_num += 1
+ update_project_issues(product, issues, conn, log)
+# sleep(1.0) # give time for Sqlite to sync
+ conn.commit() #commit to db after each block
+
+#################################
+# Jira Enhancement Requests
+#
+
+#Gets all JIRA Enhancement Requests for the specified project
+def get_jira_er(project_prefix):
+ print ("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix)
+ try:
+ jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
+ except Exception as e:
+ print("CONNECTION TO JIRA FAILED")
+ return 1
+
+ block_size = 100
+ block_num = 0
+ while True:
+ start_idx = block_num*block_size
+ #searches current project's bug issues that contain "cve" in their text
+ issues = jira.search_issues('project=%s AND type = "Enhancement Request"' % project_prefix, start_idx, block_size, False)
+ if len(issues) == 0:
+ # Retrieve issues until there are no more to come
+ break
+ block_num += 1
+ for issue in issues:
+ print (issue.raw['key'])
+
+#################################
+# Jira add to investigation
+#
+
+def jira_add_to_defect_db(jira_name):
+ print("Jira_name=%s" % (jira_name))
+ jira_name = jira_name.strip().upper()
+
+ # Just error errors to terminal
+ log = sys.stdout
+
+ #try connecting to jira
+ try:
+ jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
+ conn = sqlite3.connect(srtDbName)
+ c = conn.cursor()
+ except Exception as e:
+ print("xhr_investigation_commit:CONNECTION TO JIRA FAILED:(%s)\n" % e, file=sys.stderr)
+ return 1
+
+ #Import the issue into the SRTool
+ try:
+ issue = jira.issue(jira_name, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD))
+ except Exception as e:
+ print("ERROR:key '%s' does not exist(%s)" % (jira_name,e), file=sys.stderr)
+ return 1
+
+ d = Defect()
+ try:
+ issue_json = issue.raw
+ d.name = issue_json['key']
+ d.date_updated = issue_json['fields']['updated']
+ d.date_created = issue_json['fields']['created']
+ d.summary = issue_json['fields']['summary']
+ d.url = JIRA_BROWSE_LINK_FORMAT % d.name
+ d.priority = translate_priority(d.name,issue_json['fields']['priority']['name'])
+ d.status = translate_status(d.name,issue_json['fields']['status']['name'])
+
+ if issue_json['fields']['resolution'] is not None:
+ d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log)
+ else:
+ d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved',log)
+ if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None:
+ d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value']
+ else:
+ d.publish=''
+ if JIRA_FIX_VERSION_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_FIX_VERSION_FIELD] is not None:
+ d.release_version = issue_json['fields'][JIRA_FIX_VERSION_FIELD]['name']
+ else:
+ d.release_version = ''
+
+ # Get the product ID
+ products = c.execute('''SELECT * FROM orm_product''').fetchall()
+ d.product_id = 1
+ for product in products:
+ if d.name.startswith(get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')):
+ d.product_id = product[ORM.PRODUCT_ID]
+ break
+
+ #_log("\tINSERTING %s\n" % d.name)
+ sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)'''
+ c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated))
+ conn.commit()
+ c.close()
+ conn.close()
+ except Exception as e:
+ print("ERROR:could not find/import defect(%s)" % e, file=sys.stderr)
+ return 1
+
+#################################
+# jira_del_from_defect_db
+#
+
+# Not yet implemented
+def jira_del_from_defect_db(jira_name):
+ return 1
+
+#################################
+# New defect
+#
+
+# Use "jiratest" for development testing
+JIRA_IS_TEST = True
+JIRA_IS_SIMULATE = True
+
+def simulate_new_defect_name(product_prefix):
+ conn = sqlite3.connect(srtDbName)
+ cur = conn.cursor()
+
+ sql = "SELECT * FROM orm_srtsetting WHERE name='current_defect_simulation_index'"
+ cvi = cur.execute(sql).fetchone()
+ if not cvi:
+ index = 100
+ sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)'''
+ cur.execute(sql, ('current_defect_simulation_index', '', index))
+ else:
+ index = int(cvi[ORM.SRTSETTING_VALUE]) + 1
+ sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?'''
+ cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID]))
+ conn.commit() #commit to db
+ conn.close()
+
+ defect_name = "%s-%05d" % (product_prefix,index)
+ return defect_name
+
+def simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link,jira_url):
+ product_defect_prefix = get_tag_key(product_defect_tags,'key')
+ defect_name = simulate_new_defect_name(product_defect_prefix)
+ print("CREATED:%s,%s/browse/%s" % (defect_name,jira_url,defect_name))
+
+def jira_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link):
+ srt_error_log("NEW DEFECT:%s|%s|%s|%s|%s|%s|%s|%s" % (product_defect_tags,summary,cve_list,description,reason,priority,components,link))
+
+ if JIRA_IS_TEST:
+ # Test URL
+ jira_url = JIRA_TESTSERVER_LINK
+ else:
+ # Production URL
+ jira_url = JIRA_PRODUCTION_LINK
+
+ if JIRA_IS_SIMULATE:
+ return simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link,jira_url)
+
+ # Connect to Jira server
+ try:
+ jira = JIRA(jira_url, auth=(srt_user, srt_passwd))
+ except Exception as e:
+ print("CONNECTION TO JIRA FAILED")
+ return 1
+ #srt_error_log("Jira connection made")
+
+ conn = sqlite3.connect(srtDbName)
+ c = conn.cursor()
+
+ # append the jira link to description
+ description += "\n\n\n%s" % link
+
+ product_defect_prefix = get_tag_key(product_defect_tags,'key')
+
+ # Translate the SRTool priority to Jira priority
+ priority = translate_priority_srt_to_jira(priority)
+
+ #print("FOO1:%s,%s,%s" % (product_defect_prefix,summary,description))
+
+ jira_components=list()
+ for component in components.split(' '):
+ jira_components.append({'name' : component})
+
+ issue_dict = {
+ 'project': {'key': product_defect_prefix},
+ 'summary': summary,
+ 'description': description,
+ 'issuetype': {'name': 'Bug'},
+ 'priority': {'name': priority},
+ 'components': jira_components, # Components by name
+ 'assignee': {"name":JIRA_NEW_ASSIGNEE}, # assign to the SRTool developer for now
+ }
+
+##
+## Add custom fields here, using the format...
+## issue_dict['custom_field_10001'] = value
+##
+
+ ### ACME_EXTENSION_BEGIN ###
+ ## HERE IS AN EXAMPLE OF ADJUSTMENTS TO THE JIRA QUERY
+ issue_dict['customfield_13304'] = {"value":"Review"} # "Where Found" = "Review"
+ ### ACME_EXTENSION_END ###
+
+ new_issue = jira.create_issue(fields=issue_dict)
+ #print("CREATE:%s" % issue_dict)
+ #new_issue = '%s-%s' % (product_defect_prefix,datetime.now().strftime('%H%M%S'))
+
+ new_issue = str(new_issue.key)
+ if not new_issue.startswith(product_defect_prefix):
+ print("ERROR:%s" % new_issue)
+ else:
+ print("CREATED:%s,%s/browse/%s" % (new_issue,jira_url,new_issue))
+
+
+def get_projects():
+ try:
+ jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
+ except Exception as e:
+ print("CONNECTION TO JIRA FAILED")
+ return 1
+
+ # Get all projects viewable by anonymous users.
+ projects = jira.projects()
+ print("Projects:%s" % projects)
+
+
+#################################
+# Init/Update from Jira status
+#
+
+def update_jira(is_init):
+ try:
+ print("BEGINNING JIRA UPDATES PLEASE WAIT ... this can take some time")
+ do_update_jira()
+ master_log.write("SRTOOL:%s:DEFECT TABLE & JIRA ISSUES:\t\t\t...\t\t\tUPDATED\n" % (date.today()))
+ print("DATABASE UPDATE FINISHED\n")
+ except Exception as e:
+ master_log.write("SRTOOL:%s:DEFECT TABLE & JIRA ISSUES:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
+ print("DATABASE UPDATES FAILED ... %s" % e)
+ return(1)
+ return(0)
+
+#################################
+# main loop
+#
+
+def main(argv):
+ global force_update
+ global verbose
+ global master_log
+ global srt_user
+ global srt_passwd
+
+ parser = argparse.ArgumentParser(description='srtool_jira.py: Manage the SRTool to Jira connection')
+ parser.add_argument('--init-jira', '-i', action='store_const', const='init_jira', dest='command', help='Init and import Jira states and update defects')
+ parser.add_argument('--update-jira', '-u', action='store_const', const='update_jira', dest='command', help='Import Jira states and update defects')
+ parser.add_argument('--update-jira-list', '-l', dest='jira_update_list', help='List of Jira defects to update in SRTool database')
+ parser.add_argument('--jira_er', '-e', nargs=1, help='Query list of pending ERs under a project, review them, and assign to Rally themes')
+ parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force updates')
+ parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Verbose debugging')
+ parser.add_argument('--user', dest='user', help='User name for Jira access')
+ parser.add_argument('--passwd', dest='passwd', help='User password for Jira access')
+
+ parser.add_argument('--add', nargs=1, help='Add an existing defect to SRTool defect database')
+ parser.add_argument('--delete', nargs=1, help='Delete an existing defect from SRTool defect database')
+ parser.add_argument('--new', action='store_const', const='new', dest='command', help='Create a new defect "--new --product-tags --summary --cve --description --reason --priority --components --link"')
+ parser.add_argument('--jira-projects', '-j', action='store_const', const='jira-projects', dest='command', help='Jira projects')
+
+ # Be flexible with arguments to support sub-parse trees
+ args, argv = parser.parse_known_args()
+
+ master_log = open("./update_logs/master_log.txt", "a")
+
+ if get_override('SRTDBG_SKIP_DEFECT_IMPORT'):
+ print("...Skipping Defect import")
+ exit(0)
+
+ # Authorization
+ if args.user:
+ srt_user = args.user
+ else:
+ srt_user = os.environ.get('SRT_USER')
+ if args.passwd:
+ srt_passwd = args.passwd
+ else:
+ srt_passwd = os.environ.get('SRT_PASSWD')
+ if not srt_user or not srt_passwd:
+ msg = "FATAL ERROR: Missing user/password for Jira access"
+ print(msg)
+ srt_error_log(msg)
+ return 1
+
+ force_update = False
+ if None != args.force_update:
+ force_update = args.force_update
+ if None != args.verbose:
+ verbose = True
+ if verbose:
+ srt_error_log("srtool_jira: NOTE unprocessed arguments: %s" % argv)
+
+ jira_list = ''
+ if None != args.jira_update_list:
+ jira_list = args.jira_update_list
+
+ ret = 0
+ if args.jira_er:
+ ret = get_jira_er(args.jira_er[0])
+ elif args.add:
+ ret = jira_add_to_defect_db(args.add[0])
+ elif args.delete:
+ ret = jira_del_from_defect_db(args.add[0])
+ elif 'new' == args.command:
+ # Instantiate a sub-parse tree for "new" specific arguments
+ # Example:
+ # $ ./bin/<app>/srtool_jira.py --new --product-tags '{"key":"MERRIE"}' --summary 'User error' \
+ # --cve 'CVE-2019-0000' --description 'Incorrect equipement use' --reason 'beep' \
+ # --priority High --components "acme" --link 'www.acme.com/cve/CVE-2019-0000'
+ new_parser = argparse.ArgumentParser(parents=[parser],add_help=False)
+ new_parser.add_argument('--product-tags','-T', dest='tags', help='Product tags for defect tool integration')
+ new_parser.add_argument('--summary','-S', help='Defect summary')
+ new_parser.add_argument('--cve','-C', help='CVE list')
+ new_parser.add_argument('--description','-D', help='Defect summary')
+ new_parser.add_argument('--reason','-R', help='Defect reason hint')
+ new_parser.add_argument('--priority','-P', help='Defect priority')
+ new_parser.add_argument('--components','-O', help='Affected components')
+ new_parser.add_argument('--link','-L', help='Link to upstream CVE')
+ args, argv = new_parser.parse_known_args()
+ if verbose:
+ srt_error_log("SRTool_Jira:NEW: NOTE unprocessed arguments: %s" % argv)
+ jira_new_defect(
+ args.tags if args.tags else '{}',
+ args.summary if args.summary else '',
+ args.cve if args.cve else '',
+ args.description if args.description else '',
+ args.reason if args.reason else '',
+ args.priority if args.priority else '',
+ args.components if args.components else '',
+ args.link if args.link else '',
+ )
+ elif 'init_jira' == args.command:
+ ret = update_jira(True)
+ elif 'update_jira' == args.command:
+ ret = update_jira(False)
+ elif jira_list:
+ ret = jira_update_list(jira_list)
+ elif 'jira-projects' == args.command:
+ ret = get_projects()
+ else:
+ print("Command not found")
+
+ if 0 != ret:
+ exit(ret)
+
+
+if __name__ == '__main__':
+ if verbose: print("srtool_jira(%s)" % sys.argv[1:])
+
+ # fetch any environment overrides
+ set_override('SRTDBG_SKIP_DEFECT_IMPORT')
+ set_override('SRTDBG_MINIMAL_DB')
+
+ srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
+ main(sys.argv[1:])
diff --git a/bin/acme/srtool_jira.py b/bin/common/srtool_jira_template.py
index f9b8103b..5f29355d 100755..100644
--- a/bin/acme/srtool_jira.py
+++ b/bin/common/srtool_jira_template.py
@@ -5,7 +5,7 @@
#
# Security Response Tool Commandline Tool
#
-# Copyright (C) 2018 Wind River Systems
+# Copyright (C) 2018-2019 Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
@@ -21,33 +21,29 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
### Usage Examples (run from top level directory)
-# Updating Jira Issues: ./bin/srtool_jira.py -U
-
-
-###
-### THIS IS A SAMPLE JIRA INTEGRATION SCRIPT FOR
-### MANAGING AN ORGANIZATION'S SRTOOL INTEGRATION
-###
-### INSTALLATION INSTRUCTIONS AND DOCUMENATION OF THE JIRA PYTHON
-### LIBRARIES CAN BE FOUND HERE:
-### https://jira.readthedocs.io
-###
-
+# Updating Jira Issues: ./bin/<app>/srtool_jira<_app>.py -u
+
+##
+## THIS IS A JIRA INTEGRATION SCRIPT FOR
+## MANAGING AN ORGANIZATION'S SRTOOL INTEGRATION
+##
+## You can use the "bin/common/srtool_patcher.py" to extend this template
+## with your custom changes yet keep in sync with the shared upstream file.
+##
+## See the example: "bin/acme/srtool_jira_acme.py"
+##
+## INSTALLATION INSTRUCTIONS AND DOCUMENATION OF THE JIRA PYTHON
+## LIBRARIES CAN BE FOUND HERE:
+## https://jira.readthedocs.io
+##
import os
import sys
import re
-import csv
-import xml.etree.ElementTree as ET
import argparse
import sqlite3
-import subprocess
import json
-from jira import JIRA
-from time import sleep
from datetime import datetime, date
-from urllib.request import urlopen, URLError
-from urllib.parse import urlparse
# load the srt.sqlite schema indexes
dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -55,35 +51,52 @@ sys.path.insert(0, dir_path)
from common.srt_schema import ORM
# Setup:
+master_log = ''
+srt_user = ''
+srt_passwd = ''
+force_update = False
+
srtDbName = 'srt.sqlite'
srtErrorLog = 'srt_errors.txt'
-###
-### THESE ARE SAMPLE JIRA INTEGRATION LINKS:
-### The 'production' link is to the main Jira server
-### The 'test' link is to a test Jira server, for example to pre-test defect creation
-### The 'browse' link is how to format the Jira defect lookup link that is passed to the SRTool users
-###
+#
+# Load Jira test
+#
-# Jira constants
-JIRA_PRODUCTION_LINK = 'https://jira.acme.com'
-JIRA_BROWSE_LINK_FORMAT = 'http://jira.acme.com/browse/%s'
-JIRA_TESTSERVER_LINK = 'https://jira.acmetest.com'
+try:
+ from jira import JIRA
+except:
+ # could be SRTDBG_SKIP_DEFECT_IMPORT
+ pass
-###
-### THESE ARE SAMPLE JIRA CUSTOM FIELD LINKS:
-### These are example links that you can use to define and populate the respective fields for the SRTool
-### These are not defined in the standard Jira schema, and would presumably be custom added
-### You can extend and cusomize the Jira integration to SRTool by added additional custom fields using this model
-### The two indicated fields below are part of the SRTool design. They can be present as empty strings if they
-### do not exist in your schema:
-### The 'PUBLISHED' field is used to indicate if and when a Jira defect has been published to the customers
-### The 'FIX_VERSION' field is used to indicate which version/release/update/service pack of the product has the fix
-###
+##
+## THESE ARE JIRA INTEGRATION LINKS:
+## The 'production' link is to the main Jira server
+## The 'test' link is to a test Jira server, for example to pre-test defect creation
+## The 'browse' link is how to format the Jira defect lookup link that is passed to the SRTool users
+##
+
+# Jira constants: examples
+JIRA_PRODUCTION_LINK = 'https://jira.sample.com'
+JIRA_BROWSE_LINK_FORMAT = 'http://jira.sample.com/browse/%s'
+JIRA_TESTSERVER_LINK = 'https://jira.sampletest.com'
+JIRA_NEW_ASSIGNEE = 'kilroy'
+
+##
+## THESE ARE SAMPLE JIRA CUSTOM FIELD LINKS:
+## These are example links that you can use to define and populate the respective fields for the SRTool
+## These are not defined in the standard Jira schema, and would presumably be custom added
+## You can extend and cusomize the Jira integration to SRTool by added additional custom fields using this model
+## The two indicated fields below are part of the SRTool design. They can be present as empty strings if they
+## do not exist in your schema:
+## The 'PUBLISHED' field is used to indicate if and when a Jira defect has been published to the customers
+## The 'FIX_VERSION' field is used to indicate which version/release/update/service pack of the product has the fix
+##
# Custom Jira fields
-JIRA_PUBLISHED_FIELD = 'customfield_10010' ### REPLACE WITH YOUR CUSTOM FIELD VALUE
-JIRA_FIX_VERSION_FIELD = 'customfield_10011' ### REPLACE WITH YOUR CUSTOM FIELD VALUE
+# Custom Jira fields
+JIRA_PUBLISHED_FIELD = 'customfield_10001' ### REPLACE WITH YOUR CUSTOM FIELD VALUE
+JIRA_FIX_VERSION_FIELD = 'customfield_10002' ### REPLACE WITH YOUR CUSTOM FIELD VALUE
# ...
#################################
@@ -98,14 +111,14 @@ def debugMsg(msg):
overrides = {}
-def set_override(key,value=None):
+def set_override(key,value=None,quite=False):
if not value is None:
overrides[key] = value
elif key in os.environ.keys():
overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no'
else:
overrides[key] = 'no'
- if 'yes' == overrides[key]:
+ if 'yes' == overrides[key] and not quite:
print("OVERRIDE: %s = %s" % (key,overrides[key]))
def get_override(key):
@@ -126,47 +139,38 @@ def get_tag_key(tag,key):
# class to hold fields of a Jira issues
#
-class Defect:
+class Defect():
project = ''
product_id = ''
-
- def __init__(self,*args, **kwargs):
- self.reset()
-
- # reset all but persistent project info
- def reset(self):
- self.id = -1
- self.name = ''
- self.summary = ''
- self.url = ''
- self.priority = -1
- self.status = ''
- self.resolution = 'Unresolved'
- self.publish = ''
-
- # RCPL
- self.release_version = ''
-
- self.date_created = ''
- self.date_updated = ''
-
- # extra fields
- self.cve_status = ''
- self.vi_status = ''
- self.vi_outcome = ''
+ id = -1
+ name = ''
+ summary = ''
+ url = ''
+ priority = -1
+ status = ''
+ resolution = 'Unresolved'
+ publish = ''
+ # RCPL
+ release_version = ''
+ date_created = ''
+ date_updated = ''
+ # extra fields
+ cve_status = ''
+ vi_status = ''
+ vi_outcome = ''
#################################
-# Import Jira states
+# import Jira states
#
-# if too slow, change to check update times and ignore those that need nothing
-# can also move parsing JSON so that it doesnt happen if record is up to date
+#if too slow, change to check update times and ignore those that need nothing (should do anyway to be honest...)
+#can also move parsing JSON so that it doesnt happen if record is up to date
def do_update_jira():
try:
jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
except Exception as e:
print("CONNECTION TO JIRA FAILED")
- return
+ return 1
conn = sqlite3.connect(srtDbName)
c = conn.cursor()
@@ -175,14 +179,14 @@ def do_update_jira():
weeknum = today.strftime("%W")
weekday = today.isoweekday()
- log = open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a")
+ update_log = open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a")
- log.write("BEGINNING JIRA UPDATES\n")
+ update_log.write("BEGINNING JIRA UPDATES\n")
pre_update_time = datetime.now()
products = c.execute('''SELECT * FROM orm_product''').fetchall()
for i,product in enumerate(products):
-# print("FOO1:%s,%s" % (product[ORM.PRODUCT_DEFECT_TAGS],get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')))
+# print("JIRA_PRODUCT_SCAN:%s,%s" % (product[ORM.PRODUCT_DEFECT_TAGS],get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')))
product_defect_prefix = get_tag_key(product[ORM.PRODUCT_DEFECT_TAGS],'key')
if get_override('SRTDBG_MINIMAL_DB') and (i > 1):
@@ -190,14 +194,13 @@ def do_update_jira():
#specify which fields to get in order to speed up request!
#print("\tupdating ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\tloading " + spinner[block_num % 3], end='\r', flush=True)
- log.write("\tUPDATING ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\n")
+ update_log.write("\tUPDATING ... " + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE] + "\n")
block_size = 500
block_num = 0
spinner = [' ', '. ', '.. ', '...']
while True:
print("\tloading" + spinner[block_num % 4] + "\t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], end='\r', flush=True)
-# print("\tloading" + spinner[block_num % 4] + "\t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], flush=True)
start_idx = block_num*block_size
#searches current project's bug issues that contain "cve" in their text
issues = jira.search_issues('project=%s AND text ~ "cve*" AND type = Bug' % product_defect_prefix, start_idx, block_size, False, fields='key,summary,priority,status,resolution,project,updated,created,%s,%s' % (JIRA_PUBLISHED_FIELD, JIRA_FIX_VERSION_FIELD))
@@ -206,26 +209,28 @@ def do_update_jira():
break
# Development support
block_num += 1
- update_project_issues(product, issues, conn, log)
+ update_project_issues(product, issues, conn, update_log)
+# sleep(1.0) # give time for Sqlite to sync
conn.commit() #commit to db after each block
+# sleep(1.0) # give time for Sqlite to sync
+# conn.commit() #commit to db after each product
print("\tfinished \t" + product[ORM.PRODUCT_NAME] + " " + product[ORM.PRODUCT_VERSION] + " " + product[ORM.PRODUCT_PROFILE], flush=True)
conn.commit()
- log.write("began updates: %s\n" % str(pre_update_time))
- log.write("finished updates: %s\n" % str(datetime.now()))
- log.write("=============================================================================\n")
- log.write("\n")
+ update_log.write("began updates: %s\n" % str(pre_update_time))
+ update_log.write("finished updates: %s\n" % str(datetime.now()))
+ update_log.write("=============================================================================\n")
+ update_log.write("\n")
# Reset datasource's lastModifiedDate as today
sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE name='Jira'"
date_string = datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT)
- c.execute(sql, (date_string,) )
+ ret = c.execute(sql, (date_string,) )
conn.commit()
c.close()
conn.close()
#############################################################################3
-### Update the defect status from the Jira database
###
def new_vulnerability_name(cur):
@@ -254,10 +259,6 @@ def new_investigation_name(cur):
cur.execute(sql, (index, cvi[ORM.SRTSETTING_ID]))
return "INV-%05d" % index
-#
-# Translate Jira values to SRTool values
-#
-
def translate_priority(j,p):
NONE = 0
MINOR = 1
@@ -279,6 +280,21 @@ def translate_priority(j,p):
srt_error_log("ERROR: unknown priority string '%s=%s'" % (j,p))
return '0'
+def translate_priority_srt_to_jira(s):
+ Priority = (
+ ('Undefined', 'None'),
+ ('Minor', 'P4'),
+ ('Low', 'P3'),
+ ('Medium', 'P2'),
+ ('High', 'P1'),
+ )
+ for i in range(len(Priority)):
+ if s == Priority[i][0]:
+ return str(Priority[i][1])
+ print("ERROR: unknown priority string '%s'" % (s))
+ srt_error_log("ERROR: unknown priority string '%s'" % (s))
+ return 'None'
+
def translate_status(j,s):
OPEN = 0
IN_PROGRESS = 1
@@ -301,7 +317,7 @@ def translate_status(j,s):
srt_error_log("ERROR: unknown status string '%s=%s'" % (j,s))
return '0'
-def translate_resolution(j,r):
+def translate_resolution(j,r,log):
Resolution = (
(ORM.DEFECT_UNRESOLVED, 'Unresolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_OPEN),
(ORM.DEFECT_RESOLVED, 'Resolved', ORM.STATUS_VULNERABLE,ORM.STATUS_NOT_VULNERABLE,ORM.OUTCOME_FIXED),
@@ -322,10 +338,6 @@ def translate_resolution(j,r):
log.write("ERROR: unknown resolution string '%s=%s'" % (j,r))
return 0,0,0,0
-#
-# Update the defect status from the Jira database
-#
-
#handles updating a list of issues for a single product/project
#DOES NOT CALL COMMIT (should change this?)
def update_project_issues(project, issues, conn, log):
@@ -347,11 +359,10 @@ def update_project_issues(project, issues, conn, log):
USER_SRTOOL_ID = "SRTool"
- d = Defect()
- d.project = project[ORM.PRODUCT_NAME] + " " + project[ORM.PRODUCT_VERSION] + " " + project[ORM.PRODUCT_PROFILE]
- d.product_id = project[ORM.PRODUCT_ID]
+ project_str = project[ORM.PRODUCT_NAME] + " " + project[ORM.PRODUCT_VERSION] + " " + project[ORM.PRODUCT_PROFILE]
+ product_id = project[ORM.PRODUCT_ID]
- cve_regex = re.compile("CVE-\d+-\d+")
+ cve_regex = re.compile(r"CVE-\d+-\d+")
srtool_today = datetime.today().strftime('%Y-%m-%d')
c = conn.cursor()
@@ -362,7 +373,11 @@ def update_project_issues(project, issues, conn, log):
break
issue_json = issue.raw
- d.reset()
+
+ d = Defect()
+ d.project = project_str
+ d.product_id = product_id
+
d.name = issue_json['key']
d.date_updated = issue_json['fields']['updated']
d.date_created = issue_json['fields']['created']
@@ -372,17 +387,13 @@ def update_project_issues(project, issues, conn, log):
d.status = translate_status(d.name,issue_json['fields']['status']['name'])
if issue_json['fields']['resolution'] is not None:
- d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'])
+ d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log)
else:
- d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,'Unresolved')
+ d.resolution,cve_status,vi_status,vi_outcome = translate_resolution(d.name,'Unresolved',log)
if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None:
d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value']
- else:
- d.publish = ''
if JIRA_FIX_VERSION_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_FIX_VERSION_FIELD] is not None:
d.release_version = issue_json['fields'][JIRA_FIX_VERSION_FIELD]['name']
- else:
- d.release_version = ''
sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name
defect = c.execute(sql).fetchone()
@@ -444,6 +455,7 @@ def update_project_issues(project, issues, conn, log):
# Also create CVE history entry
sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)'''
c.execute(sql, (c_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID))
+# sleep(0.1)
else:
c_id = cve[ORM.CVE_ID]
@@ -467,6 +479,7 @@ def update_project_issues(project, issues, conn, log):
# Also create Vulnerability history entry
sql = '''INSERT INTO orm_vulnerabilityhistory (vulnerability_id, comment, date, author) VALUES (?,?,?,?)'''
c.execute(sql, (v_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID))
+# sleep(0.1)
else:
print("FOUND VULNERABILITY ID for %s" % (cve_name))
v_id = c2v[ORM.CVETOVULNERABLILITY_VULNERABILITY_ID]
@@ -493,6 +506,7 @@ def update_project_issues(project, issues, conn, log):
# Also create Investigation history entry
sql = '''INSERT INTO orm_investigationhistory (investigation_id, comment, date, author) VALUES (?,?,?,?)'''
c.execute(sql, (i_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID))
+# sleep(0.1)
else:
print("FOUND INVESTIGATION ID for %s" % (cve_name))
i_id = investigation[ORM.INVESTIGATION_ID]
@@ -506,6 +520,7 @@ def update_project_issues(project, issues, conn, log):
log.write("\tINSERTING INVESTIGATION to DEFECT for %s\n" % i_name)
sql = '''INSERT INTO orm_investigationtodefect (investigation_id, product_id, defect_id) VALUES (?,?,?)'''
c.execute(sql, (i_id,d.product_id,defect_id))
+# sleep(0.1)
#print("=========================================================================================\n")
#print("\n")
@@ -520,13 +535,13 @@ def jira_update_list(jira_list):
jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
except Exception as e:
print("CONNECTION TO JIRA FAILED")
- return
+ return 1
conn = sqlite3.connect(srtDbName)
c = conn.cursor()
products = c.execute('''SELECT * FROM orm_product''').fetchall()
- log = sys.stdout # open("./update_logs/update_jira_log_%s_%s.txt" % (weeknum, weekday), "a")
+ log = sys.stdout
for jira_name in jira_list.split(','):
print("Updating:\t" + jira_name, flush=True)
@@ -546,6 +561,7 @@ def jira_update_list(jira_list):
# Development support
block_num += 1
update_project_issues(product, issues, conn, log)
+# sleep(1.0) # give time for Sqlite to sync
conn.commit() #commit to db after each block
#################################
@@ -554,20 +570,19 @@ def jira_update_list(jira_list):
#Gets all JIRA Enhancement Requests for the specified project
def get_jira_er(project_prefix):
- #log.write("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix)
print ("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix)
try:
jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
except Exception as e:
print("CONNECTION TO JIRA FAILED")
- return
+ return 1
block_size = 100
block_num = 0
while True:
start_idx = block_num*block_size
#searches current project's bug issues that contain "cve" in their text
- issues = jira.search_issues('project=%s AND type = "Enhancement Request"' % project_prefix, start_idx, block_size, False) #project argument could be better written I guess :)
+ issues = jira.search_issues('project=%s AND type = "Enhancement Request"' % project_prefix, start_idx, block_size, False)
if len(issues) == 0:
# Retrieve issues until there are no more to come
break
@@ -583,6 +598,9 @@ def jira_add_to_defect_db(jira_name):
print("Jira_name=%s" % (jira_name))
jira_name = jira_name.strip().upper()
+ # Just error errors to terminal
+ log = sys.stdout
+
#try connecting to jira
try:
jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
@@ -610,11 +628,10 @@ def jira_add_to_defect_db(jira_name):
d.priority = translate_priority(d.name,issue_json['fields']['priority']['name'])
d.status = translate_status(d.name,issue_json['fields']['status']['name'])
-
if issue_json['fields']['resolution'] is not None:
- d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'])
+ d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,issue_json['fields']['resolution']['name'],log)
else:
- d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved')
+ d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved',log)
if JIRA_PUBLISHED_FIELD in issue_json['fields'] and issue_json['fields'][JIRA_PUBLISHED_FIELD] is not None:
d.publish = issue_json['fields'][JIRA_PUBLISHED_FIELD]['value']
else:
@@ -632,7 +649,7 @@ def jira_add_to_defect_db(jira_name):
d.product_id = product[ORM.PRODUCT_ID]
break
- #log.write("\tINSERTING %s\n" % d.name)
+ #_log("\tINSERTING %s\n" % d.name)
sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)'''
c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated))
conn.commit()
@@ -648,20 +665,15 @@ def jira_add_to_defect_db(jira_name):
# Not yet implemented
def jira_del_from_defect_db(jira_name):
- pass
+ return 1
#################################
# New defect
#
-###
-### You will need to fill in any required custom fields required
-### to create new organization defects in Jira
-###
-
# Use "jiratest" for development testing
-IS_TEST = True
-IS_SIMULATE = True
+JIRA_IS_TEST = True
+JIRA_IS_SIMULATE = True
def simulate_new_defect_name(product_prefix):
conn = sqlite3.connect(srtDbName)
@@ -688,32 +700,17 @@ def simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,
defect_name = simulate_new_defect_name(product_defect_prefix)
print("CREATED:%s,%s/browse/%s" % (defect_name,jira_url,defect_name))
-def translate_priority_srt_to_jira(s):
- Priority = (
- ('Undefined', 'None'),
- ('Minor', 'P4'),
- ('Low', 'P3'),
- ('Medium', 'P2'),
- ('High', 'P1'),
- )
- for i in range(len(Priority)):
- if s == Priority[i][0]:
- return str(Priority[i][1])
- print("ERROR: unknown priority string '%s'" % (s))
- srt_error_log("ERROR: unknown priority string '%s'" % (s))
- return 'None'
-
-def jira_new_defect(product_defect_tags,summary,description,priority,components,link):
- #srt_error_log("NEW DEFECT:%s|%s|%s|%s|%s|%s|%s|%s" % (product_defect_tags,summary,cve_list,description,reason,priority,components,link))
+def jira_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link):
+ srt_error_log("NEW DEFECT:%s|%s|%s|%s|%s|%s|%s|%s" % (product_defect_tags,summary,cve_list,description,reason,priority,components,link))
- if IS_TEST:
+ if JIRA_IS_TEST:
# Test URL
jira_url = JIRA_TESTSERVER_LINK
else:
# Production URL
jira_url = JIRA_PRODUCTION_LINK
- if IS_SIMULATE:
+ if JIRA_IS_SIMULATE:
return simulate_new_defect(product_defect_tags,summary,cve_list,description,reason,priority,components,link,jira_url)
# Connect to Jira server
@@ -721,7 +718,8 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components,
jira = JIRA(jira_url, auth=(srt_user, srt_passwd))
except Exception as e:
print("CONNECTION TO JIRA FAILED")
- return
+ return 1
+ #srt_error_log("Jira connection made")
conn = sqlite3.connect(srtDbName)
c = conn.cursor()
@@ -734,7 +732,7 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components,
# Translate the SRTool priority to Jira priority
priority = translate_priority_srt_to_jira(priority)
- #print("JIRA_NEW_DEFECT:%s,%s,%s" % (product_defect_prefix,summary,description))
+ #print("FOO1:%s,%s,%s" % (product_defect_prefix,summary,description))
jira_components=list()
for component in components.split(' '):
@@ -745,19 +743,19 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components,
'summary': summary,
'description': description,
'issuetype': {'name': 'Bug'},
-
'priority': {'name': priority},
- 'components': jira_components, # Components by name
- 'assignee': {"name":SRT_USER }, # assign to the SRTool developer Jira account
-### 'customfield_11000': 'unknown', # EXAMPLE: "Found In Versions" = "unknown"
-### 'customfield_11001': {"value":"Review"}, # EXAMPLE: "Where Found" = "Review"
+ 'components': jira_components, # Components by name
+ 'assignee': {"name":JIRA_NEW_ASSIGNEE}, # assign to the SRTool developer for now
}
- if True:
- new_issue = jira.create_issue(fields=issue_dict)
- else:
- print("CREATE:%s" % issue_dict)
- new_issue = '%s-%s' % (product_defect_prefix,datetime.now().strftime('%H%M%S'))
+##
+## Add custom fields here, using the format...
+## issue_dict['custom_field_10001'] = value
+##
+
+ new_issue = jira.create_issue(fields=issue_dict)
+ #print("CREATE:%s" % issue_dict)
+ #new_issue = '%s-%s' % (product_defect_prefix,datetime.now().strftime('%H%M%S'))
new_issue = str(new_issue.key)
if not new_issue.startswith(product_defect_prefix):
@@ -767,12 +765,11 @@ def jira_new_defect(product_defect_tags,summary,description,priority,components,
def get_projects():
-
try:
jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd))
except Exception as e:
print("CONNECTION TO JIRA FAILED")
- return
+ return 1
# Get all projects viewable by anonymous users.
projects = jira.projects()
@@ -784,9 +781,6 @@ def get_projects():
#
def update_jira(is_init):
- if get_override('SRTDBG_SKIP_DEFECT_IMPORT'):
- print("...Skipping Defect import")
- exit(0)
try:
print("BEGINNING JIRA UPDATES PLEASE WAIT ... this can take some time")
do_update_jira()
@@ -795,6 +789,8 @@ def update_jira(is_init):
except Exception as e:
master_log.write("SRTOOL:%s:DEFECT TABLE & JIRA ISSUES:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e))
print("DATABASE UPDATES FAILED ... %s" % e)
+ return(1)
+ return(0)
#################################
# main loop
@@ -820,13 +816,17 @@ def main(argv):
parser.add_argument('--add', nargs=1, help='Add an existing defect to SRTool defect database')
parser.add_argument('--delete', nargs=1, help='Delete an existing defect from SRTool defect database')
parser.add_argument('--new', action='store_const', const='new', dest='command', help='Create a new defect "--new --product-tags --summary --cve --description --reason --priority --components --link"')
- parser.add_argument('--jira-projects', '-P', action='store_const', const='jira-projects', dest='command', help='Jira projects')
+ parser.add_argument('--jira-projects', '-j', action='store_const', const='jira-projects', dest='command', help='Jira projects')
# Be flexible with arguments to support sub-parse trees
args, argv = parser.parse_known_args()
master_log = open("./update_logs/master_log.txt", "a")
+ if get_override('SRTDBG_SKIP_DEFECT_IMPORT'):
+ print("...Skipping Defect import")
+ exit(0)
+
# Authorization
if args.user:
srt_user = args.user
@@ -854,12 +854,13 @@ def main(argv):
if None != args.jira_update_list:
jira_list = args.jira_update_list
+ ret = 0
if args.jira_er:
- get_jira_er(args.jira_er[0])
+ ret = get_jira_er(args.jira_er[0])
elif args.add:
- jira_add_to_defect_db(args.add[0])
+ ret = jira_add_to_defect_db(args.add[0])
elif args.delete:
- jira_del_from_defect_db(args.add[0])
+ ret = jira_del_from_defect_db(args.add[0])
elif 'new' == args.command:
# Instantiate a sub-parse tree for "new" specific arguments
# Example:
@@ -877,7 +878,7 @@ def main(argv):
new_parser.add_argument('--link','-L', help='Link to upstream CVE')
args, argv = new_parser.parse_known_args()
if verbose:
- srt_error_log("SRTool_Jira_New: NOTE unprocessed arguments: %s" % argv)
+ srt_error_log("SRTool_Jira:NEW: NOTE unprocessed arguments: %s" % argv)
jira_new_defect(
args.tags if args.tags else '{}',
args.summary if args.summary else '',
@@ -889,23 +890,26 @@ def main(argv):
args.link if args.link else '',
)
elif 'init_jira' == args.command:
- update_jira(True)
+ ret = update_jira(True)
elif 'update_jira' == args.command:
- update_jira(False)
+ ret = update_jira(False)
elif jira_list:
- jira_update_list(jira_list)
+ ret = jira_update_list(jira_list)
elif 'jira-projects' == args.command:
- get_projects()
+ ret = get_projects()
else:
print("Command not found")
-if __name__ == '__main__':
- global srtool_basepath
+ if 0 != ret:
+ exit(ret)
+
+if __name__ == '__main__':
if verbose: print("srtool_jira(%s)" % sys.argv[1:])
# fetch any environment overrides
- #set_override('SRTDBG_MINIMAL_DB')
+ set_override('SRTDBG_SKIP_DEFECT_IMPORT')
+ set_override('SRTDBG_MINIMAL_DB')
srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
main(sys.argv[1:])
diff --git a/bin/common/srtool_patcher.py b/bin/common/srtool_patcher.py
new file mode 100755
index 00000000..6bf65755
--- /dev/null
+++ b/bin/common/srtool_patcher.py
@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Security Response Tool Implementation
+#
+# Copyright (C) 2019 Wind River Systems
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+#
+# Theory of operation
+#
+# * This script runs quick sanity tests
+# * The '--init-test' option will:
+# * Report the host package versions
+# * Report errors if any required tables are empty
+# * Report the running instance of the SRTool
+#
+
+import os
+import sys
+import argparse
+import json
+
+# Setup:
+verbose = False
+
+#################################
+# extract_custom_patch
+#
+
+def extract_custom_patch(custom_file,clean_file,patch_file,label,patcher_dir):
+ tag_begin = "%s_EXTENSION_BEGIN" % label
+ tag_end = "%s_EXTENSION_END" % label
+ ret = 0
+
+ # Insure patcher working directory is ready
+ if not patcher_dir:
+ patcher_dir = os.path.dirname(custom_file)
+ try:
+ os.makedirs(patcher_dir)
+ except:
+ pass
+ # Prepare defaults
+ if not clean_file:
+ clean_file = os.path.join(patcher_dir,os.path.basename(custom_file) + '.clean')
+ if not patch_file:
+ patch_file = os.path.join(patcher_dir,os.path.basename(custom_file) + '.patch')
+
+ with open(custom_file, 'r') as fs:
+ with open(clean_file, 'w') as fd:
+ state = "find"
+ for line in fs:
+ #print("LINE(%s):%s" % (state,line.strip()))
+ if state == "find":
+ if 0 < line.find(tag_begin):
+ #print("START:$line")
+ state = "found"
+ continue
+ elif state == "found":
+ if 0 < line.find(tag_end):
+ #print("STOP:$line")
+ state = "blank"
+ continue
+ elif state == "blank":
+ state = "find"
+ # if next line after stop is blank, hide that also
+ if not line.strip():
+ continue
+ # Normal line?
+ if state != "found":
+ fd.write('%s' % line)
+ # Did we end cleanly?
+ if state != "find":
+ print("ERROR: START not STOPPED (%s)" % state)
+ os.system("diff -u %s %s > %s" % (clean_file,custom_file,patch_file))
+ print("Custom File: %s" % (custom_file))
+ print("Clean File: %s" % (clean_file))
+ print("Patch File: %s" % (patch_file))
+ return(ret,clean_file,patch_file)
+
+#################################
+# merge_original
+#
+
+def merge_original(custom_file,original_file,patch_file,label,patcher_dir):
+ ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir)
+ custom_saved = os.path.join(os.path.dirname(patch_file),os.path.basename(custom_file) + '.saved')
+ if 0 == ret:
+ print("* Preserving custom file as '%s'" % custom_saved)
+ cmd = "cp %s %s" % (custom_file,custom_saved)
+ print(cmd)
+ os.system(cmd)
+ print("* Merging original file '%s' into custom file" % original_file)
+ cmd = "cp %s %s" % (original_file,custom_file)
+ print(cmd)
+ os.system(cmd)
+ cmd = "patch %s %s" % (custom_file,patch_file)
+ print(cmd)
+ ret = os.system(cmd)
+ if 0 != ret:
+ print("* ERROR: Merge failed, restoring previous custom file")
+ cmd = "cp %s %s" % (custom_saved,custom_file)
+ print(cmd)
+ os.system(cmd)
+ return(ret)
+
+#################################
+# merge_custom
+#
+
+def merge_custom(custom_file,original_file,patch_file,label,patcher_dir):
+ ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir)
+ if 0 == ret:
+ print("== Copying clean version of custom file to '%s' ===" % original_file)
+ cmd = "cp -f %s %s" % (clean_file, original_file)
+ print(cmd)
+ os.system(cmd)
+ return(ret)
+
+#################################
+# diff_original
+#
+
+def diff_original(custom_file,original_file,patch_file,label,patcher_dir):
+ ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir)
+ if 0 == ret:
+ print("== DIFF from '%s' to clean version of custom file ===" % original_file)
+ os.system("diff -u %s %s" % (original_file,clean_file))
+ return(ret)
+
+#################################
+# load_json_list
+#
+
+def load_json_list(json_file):
+ label = 'CUSTOM'
+ patcher_dir = ''
+ file_list = []
+
+ with open(json_file) as json_data:
+ dct = json.load(json_data)
+
+ if 'label' in dct:
+ label = dct['label']
+ if 'patcher_dir' in dct:
+ patcher_dir = dct['patcher_dir']
+ if 'patch_set' in dct:
+ for patch in dct['patch_set']:
+ if 'DISABLE' in patch['options']:
+ continue
+ file_list.append([patch['custom'],patch['original'],patch['patch'],patch['options']])
+ return file_list,patcher_dir,label
+
+#################################
+# main loop
+#
+
+def main(argv):
+ global verbose
+
+ # setup
+ parser = argparse.ArgumentParser(description='srtool_sanity_test.py: SRTool common sanity tests')
+
+ parser.add_argument('--merge-original', '-O', action='store_const', const='merge_original', dest='command', help='Copy the (updated) original file, merge the custom patches')
+ parser.add_argument('--merge-custom', '-C', action='store_const', const='merge_custom', dest='command', help='Copy the (updated) file to the original, without custom patches')
+
+ parser.add_argument('--custom', '-c', help='Custom file')
+ parser.add_argument('--original', '-o', help='Original file')
+ parser.add_argument('--label', '-l', help='Custom label tag (default="CUSTOM")')
+ parser.add_argument('--json', '-j', help='Use JSON file for file list')
+
+ parser.add_argument('--extract-custom-patch', '-e', action='store_const', const='extract_custom_patch', dest='command', help='Extract a patch of the custom content')
+ parser.add_argument('--apply-custom-patch', '-a', action='store_const', const='extract_custom_patch', dest='command', help='Apply the custom patch to the (reset) custom file')
+ parser.add_argument('--patch', '-p', help='Patch file')
+ parser.add_argument('--diff-original', '-d', action='store_const', const='diff_original', dest='command', help='Show how the file compares to original, ignoring custom patches')
+
+ parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Debugging: verbose output')
+ args = parser.parse_args()
+
+ def validate_file(filename,msg):
+ if not (filename or os.path.isfile(filename)):
+ print("ERROR: %s file not found '%s'" % (msg,filename))
+ exit(1)
+
+ # Extract provided values
+ verbose = args.verbose
+ file_list = []
+ if args.json:
+ validate_file(args.json,"JSON")
+ file_list,patcher_dir,label = load_json_list(args.json)
+ else:
+ custom_file = ''
+ if args.custom:
+ custom_file = args.custom
+ original_file = ''
+ if args.original:
+ original_file = args.original
+ patch_file = ''
+ if args.patch:
+ patch_file = args.patch
+ options = ''
+ file_list.append([custom_file,original_file,patch_file,options])
+ label = 'CUSTOM'
+ if args.label:
+ label = args.label
+ patcher_dir = os.path.join(os.path.dirname(custom_file),'patcher')
+
+ ret = 0
+ for custom_file,original_file,patch_file,options in file_list:
+ #print("PATCH_FILE:%s,%s,%s,%s,%s" % (custom_file,original_file,patch_file,options,patcher_dir))
+ if 'merge_original' == args.command:
+ validate_file(custom_file,'Custom')
+ validate_file(original_file,'Original')
+ ret = merge_original(custom_file,original_file,patch_file,label,patcher_dir)
+ elif 'merge_custom' == args.command:
+ validate_file(custom_file,'Custom')
+ validate_file(original_file,'Original')
+ ret = merge_custom(custom_file,original_file,patch_file,label,patcher_dir)
+ elif 'extract_custom_patch' == args.command:
+ validate_file(custom_file,'Custom')
+ ret,clean_file,patch_file = extract_custom_patch(custom_file,'',patch_file,label,patcher_dir)
+ elif 'diff_original' == args.command:
+ validate_file(custom_file,'Custom')
+ validate_file(original_file,'Original')
+ ret = diff_original(custom_file,original_file,patch_file,label,patcher_dir)
+ else:
+ print("Command not found '%s'" % args.command)
+ ret = 1
+ if 0 != ret:
+ exit(ret)
+ exit(ret)
+
+if __name__ == '__main__':
+ srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))))
+ main(sys.argv[1:])
+
+
+