diff options
| -rwxr-xr-x | bin/acme/srtool_acme.py | 18 | ||||
| -rwxr-xr-x | bin/acme/srtool_defect.py | 19 | ||||
| -rwxr-xr-x | bin/acme/srtool_jira.py | 3 | ||||
| -rwxr-xr-x | bin/common/srtool_backup.py | 56 | ||||
| -rwxr-xr-x | bin/common/srtool_common.py | 35 | ||||
| -rwxr-xr-x | bin/common/srtool_defect.py | 614 | ||||
| -rwxr-xr-x | bin/common/srtool_email.py | 14 | ||||
| -rwxr-xr-x | bin/common/srtool_sanity_test.py | 11 | ||||
| -rwxr-xr-x | bin/common/srtool_update.py | 23 | ||||
| -rwxr-xr-x | bin/common/srtool_utils.py | 21 | ||||
| -rwxr-xr-x | bin/debian/srtool_debian.py | 15 | ||||
| -rwxr-xr-x | bin/mitre/srtool_mitre.py | 16 | ||||
| -rwxr-xr-x | bin/nist/srtool_nist.py | 54 | ||||
| -rwxr-xr-x | bin/redhat/srtool_redhat.py | 15 | ||||
| -rwxr-xr-x | bin/yp/srtool_defect.py | 23 | ||||
| -rwxr-xr-x | bin/yp/srtool_yp.py | 14 |
16 files changed, 132 insertions, 819 deletions
diff --git a/bin/acme/srtool_acme.py b/bin/acme/srtool_acme.py index b3453dff..0e33de0e 100755 --- a/bin/acme/srtool_acme.py +++ b/bin/acme/srtool_acme.py @@ -32,34 +32,23 @@ import os import sys -import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -import urllib -from datetime import datetime # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -from datetime import datetime, date -from urllib.request import urlopen, URLError -from urllib.parse import urlparse - +# Setup: +verbose = False srtDbName = 'srt.sqlite' - ################################# # Helper methods # -verbose = False - def debugMsg(msg): if verbose: print(msg) @@ -153,9 +142,6 @@ def main(argv): print("Command not found") if __name__ == '__main__': - global script_pathname - global srtool_basepath - srtool_scriptpath = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) srtool_basepath = os.path.dirname(srtool_scriptpath) main(sys.argv[1:]) diff --git a/bin/acme/srtool_defect.py b/bin/acme/srtool_defect.py index 2cf58375..77f8e030 100755 --- a/bin/acme/srtool_defect.py +++ b/bin/acme/srtool_defect.py @@ -21,25 +21,26 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. ### Usage Examples (run from top level directory) -# Updating Defect System Issues: ./bin/acme/srtool_defect.py -U - +# Updating Defect System Issues: ./bin/<app>/srtool_defect.py -U import os import sys -import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -from time import sleep # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM +# Setup: +master_log = '' +force_update = False +verbose = False +srt_user = '' +srt_passwd = '' + srtDbName = 'srt.sqlite' srtErrorLog = 'srt_errors.txt' @@ -47,8 +48,6 @@ srtErrorLog = 'srt_errors.txt' # Helper methods # -verbose = False - def debugMsg(msg): if verbose: print(msg) @@ -296,8 +295,6 @@ def main(argv): sys.exit("Command '%s' not found" % args.command) if __name__ == '__main__': - global srtool_basepath - if verbose: print("srtool_defect(%s)" % sys.argv[1:]) # fetch any environment overrides diff --git a/bin/acme/srtool_jira.py b/bin/acme/srtool_jira.py index dc72404b..f9b8103b 100755 --- a/bin/acme/srtool_jira.py +++ b/bin/acme/srtool_jira.py @@ -54,6 +54,7 @@ dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM +# Setup: srtDbName = 'srt.sqlite' srtErrorLog = 'srt_errors.txt' @@ -556,7 +557,7 @@ def get_jira_er(project_prefix): #log.write("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix) print ("GETTING ENHANCEMENT REQUESTS FOR " + project_prefix) try: - jira = JIRA(JIRA_PRODUCTION_LINK', auth=(srt_user, srt_passwd)) + jira = JIRA(JIRA_PRODUCTION_LINK, auth=(srt_user, srt_passwd)) except Exception as e: print("CONNECTION TO JIRA FAILED") return diff --git a/bin/common/srtool_backup.py b/bin/common/srtool_backup.py index ec252cdb..6cd62573 100755 --- a/bin/common/srtool_backup.py +++ b/bin/common/srtool_backup.py @@ -40,30 +40,23 @@ import os import sys -import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -import urllib +from datetime import datetime, date # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) -from srt_schema import ORM - -from datetime import datetime, date -from pprint import pprint -from urllib.request import urlopen, URLError -from urllib.parse import urlparse +from common.srt_schema import ORM -# setup +# Setup: lookupTable = [] cveIndex = {} db_change = False -is_verbose = False +verbose = False +cmd_skip = 0 +cmd_count = 0 srtDbName = 'srt.sqlite' # Save the "whole" table contents @@ -233,17 +226,27 @@ def restore_db_json(weeknum, weekday): #print("Inserting value %d" % i) #print("Restoring %s" % table + spinner[i % 4], end='\r', flush=True) #create sql query for updating subset of a record - "%s = ?," * len(dic) sql = "UPDATE %s SET " % table - sql += "%s = ?," * len(dic) % tuple(dic) - sql = sql[:-1] - sql += " WHERE %s=?" % primary_key + # Build member assignment string and values list + names = '' + values = [] + for n,v in dic: + if n == primary_key: + continue + names += "%s = ?," % n + values.append(v) + names = names[:-1] + sql += "%s WHERE %s=?" % (names,primary_key) + values.append(dic[primary_key]) + cur.execute(sql, values) + + #sql += ("%s = ?," * len(dic)) % tuple(dic) + #sql = sql[:-1] + #sql += " WHERE %s=?" % primary_key #add primary key at the end for WHERE clause - - temp = list(dic.values()) - temp.append(dic[primary_key]) - - cur.execute(sql, temp) + #temp = list(dic.values()) + #temp.append(dic[primary_key]) + #cur.execute(sql, temp) conn.commit() print("Finished %s " % table, end='\r', flush=True) print(" " * 80, end='\r') @@ -255,8 +258,10 @@ def restore_db_json(weeknum, weekday): def simulate_corruption(): conn = sqlite3.connect(srtDbName) cur = conn.cursor() - [cur.execute("DELETE FROM %s" % table) for table in WHOLE_TABLES] - #[cur.execute("DELETE FROM %s" % table[0]) for table in SUBSET_TABLES] + for table in WHOLE_TABLES: + cur.execute("DELETE FROM %s" % table) + #for table in SUBSET_TABLES: + # cur.execute("DELETE FROM %s" % table[0]) conn.commit() conn.close() @@ -268,6 +273,7 @@ def simulate_corruption(): def backup_cve_packages(): global cmd_skip + global cmd_count conn = sqlite3.connect(srtDbName) cur_cve = conn.cursor() @@ -495,7 +501,5 @@ def main(argv): master_log.close() if __name__ == '__main__': - global script_pathname - from os.path import abspath script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/common/srtool_common.py b/bin/common/srtool_common.py index 4cba2dc2..a098fc7e 100755 --- a/bin/common/srtool_common.py +++ b/bin/common/srtool_common.py @@ -31,13 +31,11 @@ import sys import re import csv import json -import xml.etree.ElementTree as ET import argparse import sqlite3 import subprocess from time import sleep -from datetime import datetime, date, timedelta -import pytz +from datetime import datetime # Load the srt.sqlite schema index file # Since it is generated from this script @@ -49,6 +47,11 @@ except: print("Warning: srt_schema not yet created or bad format") pass +# Setup: +verbose = False +cmd_skip = 0 +cmd_count = 0 + srtDbName = 'srt.sqlite' packageKeywordsFile = 'data/package_keywords.csv' notifyCategoriesFile = 'data/notify-categories.json' @@ -86,6 +89,7 @@ def get_name_sort(cve_name): # Load the package keyword source into the database # +# CSV database offsets KEYWORDS_MODE = 0 KEYWORDS_NAME = 1 KEYWORDS_REALNAME = 2 @@ -93,9 +97,6 @@ KEYWORDS_INVALIDNAME = 3 KEYWORDS_WEIGHT = 4 def init_package_keywords(filename): - global pkglistDbName - global pkglistTable - if not os.path.exists(filename): print("ERROR: DB NOT FOUND '%s'" % filename) return @@ -217,10 +218,9 @@ def attach_packages(cur, cve, recommend_list): AGAINST = 1 cve_id = cve[ORM.CVE_ID] # Bootstrap... - if False: - cve_packages = cve[ORM.CVE_PACKAGES] - else: - cve_packages = '' + #cve_packages = cve[ORM.CVE_PACKAGES] + cve_packages = '' + for pkg_name in recommend_list.split(','): if '-' == pkg_name[0:1]: mode = AGAINST @@ -246,7 +246,7 @@ def attach_packages(cur, cve, recommend_list): pkg_id = package[ORM.PACKAGE_ID] else: # Create Package - if verbose: print("INSERTING PACKAGE for %s,%s" % (cve_name,pkg_name)) + if verbose: print("INSERTING PACKAGE for %s,%s" % (cve[ORM.CVE_NAME],pkg_name)) sql = '''INSERT INTO orm_package (mode, name, realname, invalidname, weight, cve_count, vulnerability_count, investigation_count,defect_count ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)''' cur.execute(sql, (mode, pkg_name, pkg_name, '', 1 if FOR==mode else -1),0,0,0,0) pkg_id = cur.lastrowid @@ -283,8 +283,8 @@ def score_new_cves(cve_filter): cur.execute(sql) for package in cur: # Fixup notation not intended to be regex - name = package[ORM.PACKAGE_NAME].replace('++','\+\+') - realname = package[ORM.PACKAGE_REALNAME].replace('++','\+\+') + name = package[ORM.PACKAGE_NAME].replace('++',r'\+\+') + realname = package[ORM.PACKAGE_REALNAME].replace('++',r'\+\+') recommends.append([package[ORM.PACKAGE_MODE],name,realname,package[ORM.PACKAGE_INVALIDNAME],package[ORM.PACKAGE_WEIGHT]]) # Scan the open CVEs @@ -349,10 +349,8 @@ def score_new_cves(cve_filter): if recommend_list: # Go ahead and create/attach packages to CVEs - if True: - cve_packages = attach_packages(cur_write, cve, recommend_list) - else: - cve_packages = cve[ORM.CVE_PACKAGES] + cve_packages = attach_packages(cur_write, cve, recommend_list) + #cve_packages = cve[ORM.CVE_PACKAGES] sql = ''' UPDATE orm_cve SET recommend = ?, @@ -556,7 +554,7 @@ def fix_cve_recommend(): # Progress indicator support if 0 == i % 10: - print('%05d: %20s to %20s\r' % (i,cve[ORM.CVE_NAME],name_sort), end='') + print('%05d: %20s\r' % (i,cve[ORM.CVE_NAME]), end='') if (0 == i % 200): conn.commit() @@ -618,7 +616,6 @@ def main(argv): print("Command not found") if __name__ == '__main__': - global srtool_basepath srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/common/srtool_defect.py b/bin/common/srtool_defect.py deleted file mode 100755 index d2a6878f..00000000 --- a/bin/common/srtool_defect.py +++ /dev/null @@ -1,614 +0,0 @@ -#!/usr/bin/env python3 -# -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- -# -# Security Response Tool Commandline Tool -# -# Copyright (C) 2018 Wind River Systems -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -### Usage Examples (run from top level directory) -# Updating defect Issues: ./bin/srtool_defect.py --update - - -import os -import sys -import re -import csv -import xml.etree.ElementTree as ET -import argparse -import sqlite3 -import subprocess -import json -import urllib -from time import sleep - -try: - from datetime import datetime, date - from urllib.request import urlopen, URLError - from urllib.parse import urlparse -except ImportError: - from urllib2 import urlopen, URLError - from urlparse import urlparse - -srtDbName = 'srt.sqlite' - -# load the srt.sqlite schema indexes -dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -sys.path.insert(0, dir_path) -from common.srt_schema import ORM - -################################# -# Helper methods -# - -overrides = {} - -def set_override(key,value=None): - if not value is None: - overrides[key] = value - elif key in os.environ.keys(): - overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no' - else: - overrides[key] = '' - print("OVERRIDE: %s = %s" % (key,overrides[key])) - -def get_override(key): - if key in overrides.keys(): - return overrides[key] - return '' - -################################# -# Access Rights -# -srt_user = os.environ.get('SRT_USER') -srt_passwd = os.environ.get('SRT_PASSWD') -if not srt_user: - srt_user = 'user' -if not srt_passwd: - srt_user = 'passwd' - -################################# -# class to hold fields of a defect -# -class Defect: - id = -1 - name = '' - summary = '' - url = '' - priority = -1 - status = 0 - resolution = 0 - publish = 'Unpublished' - - # Fixed release - release_version = '' - - product_id = 1 - date_created = '' - date_updated = '' - - project = '' - - # extra fields - cve_status = 0 - vi_status = 0 - vi_outcome = 0 - -################################# -# import defect states -# -#if too slow, change to check update times and ignore those that need nothing (should do anyway to be honest...) -#can also move parsing JSON so that it doesnt happen if record is up to date - -def update_defects(): - try: - ### TODO: open connection to defect database - # defect_db = ... - pass - except Exception as e: - print("CONNECTION TO DEFECT DATABASE FAILED") - return - - conn = sqlite3.connect(srtDbName) - c = conn.cursor() - - today = datetime.today() - weeknum = today.strftime("%W") - weekday = today.isoweekday() - - log = open("./update_logs/update_defects_log_%s_%s.txt" % (weeknum, weekday), "a") - - - PRODUCT_ID = 0 - PRODUCT_NAME = 1 - PRODUCT_VERSION = 2 - PRODUCT_PROFILE = 3 - PRODUCT_PREFIX = 6 - - DEFECT_ID = 0 - DEFECT_DATE_UPDATED = 10 - - PUBLISHED_FIELD = 'customfield_10010' - FIX_VERSION_FIELD = 'customfield_11002' - - log.write("BEGINNING DEFECT UPDATES\n") - pre_update_time = datetime.now() - - products = c.execute('''SELECT * FROM orm_product''').fetchall() - for i,product in enumerate(products): - - if get_override('SRTDBG_MINIMAL_DB') and (i > 1): - break - - #specify which fields to get in order to speed up request! - #print("\tupdating ... " + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE] + "\tloading " + spinner[block_num % 3], end='\r', flush=True) - log.write("\tUPDATING ... " + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE] + "\n") - - block_size = 500 - block_num = 0 - spinner = [' ', '. ', '.. ', '...'] - while True: - print("\tloading" + spinner[block_num % 4] + "\t" + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE], flush=True) - start_idx = block_num*block_size - - #searches current project's bug issues that contain "cve" in their text - ### TODO: FETCH DEFECT RECORDS FOR THIS PRODUCT FROM DEFECT DATABASE - - if len(issues) == 0: - # Retrieve issues until there are no more to come - break - # Development support - block_num += 1 - update_project_issues(product, issues, conn, log) - conn.commit() #commit to db after each block - sleep(1.0) # give time for Sqlite to sync - print("\tfinished \t" + product[PRODUCT_NAME] + " " + product[PRODUCT_VERSION] + " " + product[PRODUCT_PROFILE], flush=True) - conn.commit() - log.write("began updates: %s\n" % str(pre_update_time)) - log.write("finished updates: %s\n" % str(datetime.now())) - log.write("=============================================================================\n") - log.write("\n") - - # Reset datasource's lastModifiedDate as today - sql = "UPDATE orm_datasource SET lastModifiedDate=? WHERE name='Defect'" - date_string = datetime.today().strftime(ORM.DATASOURCE_DATETIME_FORMAT) - c.execute(sql, (date_string,) ) - conn.commit() - - c.close() - conn.close() - -#############################################################################3 -### - -def new_vulnerability_name(c): - CVI_ID = 0 - CVI_VALUE = 3 - sql = "SELECT * FROM orm_srtsetting WHERE name='current_vulnerability_index'" - cvi = c.execute(sql).fetchone() - if not cvi: - index = 100 - sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' - c.execute(sql, ('current_vulnerability_index', '', index)) - else: - index = int(cvi[CVI_VALUE]) + 1 - sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' - c.execute(sql, (index, cvi[CVI_ID])) - return "V%05d" % index - -def new_investigation_name(c): - CVI_ID = 0 - CVI_VALUE = 3 - sql = "SELECT * FROM orm_srtsetting WHERE name='current_investigation_index'" - cvi = c.execute(sql).fetchone() - if not cvi: - index = 100 - sql = '''INSERT INTO orm_srtsetting (name, helptext, value) VALUES (?,?,?)''' - c.execute(sql, ('current_investigation_index', '', index)) - else: - index = int(cvi[CVI_VALUE]) + 1 - sql = '''UPDATE orm_srtsetting SET value=? WHERE id = ?''' - c.execute(sql, (index, cvi[CVI_ID])) - return "I%05d" % index - -def translate_priority(j,p): - MINOR = 0 - LOW = 1 - MEDIUM = 2 - HIGH = 3 - Priority = ( - (MINOR, 'P4'), - (LOW, 'P3'), - (MEDIUM, 'P2'), - (HIGH, 'P1'), - ) - for i in range(len(Priority)): - if p == Priority[i][1]: - return str(Priority[i][0]) - print("ERROR: unknown priority string '%s=%s'" % (j,p)) - log.write("ERROR: unknown priority string '%s=%s'" % (j,p)) - return '0' - -def translate_status(j,s): - OPEN = 0 - IN_PROGRESS = 1 - ON_HOLD = 2 - CHECKED_IN = 3 - RESOLVED = 4 - CLOSED = 5 - Status = ( - (OPEN, 'Open'), - (IN_PROGRESS, 'In progress'), - (ON_HOLD, 'On Hold'), - (CHECKED_IN, 'Checked In'), - (RESOLVED, 'Resolved'), - (CLOSED, 'Closed'), - ) - for i in range(len(Status)): - if s == Status[i][1]: - return str(Status[i][0]) - print("ERROR: unknown status string '%s=%s'" % (j,s)) - log.write("ERROR: unknown status string '%s=%s'" % (j,s)) - return '0' - -def translate_resolution(j,r): - UNRESOLVED = 0 - RESOLVED = 1 - FIXED = 2 - WILL_NOT_FIX = 3 - WITHDRAWN = 4 - REJECTED = 5 - DUPLICATE = 6 - NOT_APPLICABLE = 7 - REPLACED_BY_REQUIREMENT = 8 - CANNOT_REPRODUCE = 9 - DONE = 10 - - CVE_NEW = 0 - CVE_INVESTIGATE = 1 - CVE_VULNERABLE = 2 - CVE_NOT_VULNERABLE = 3 - VI_INVESTIGATE = 0 - VI_NOT_VULNERABLE = 1 - VI_VULNERABLE = 2 - VI_OPEN = 0 - VI_CLOSED = 1 - VI_FIXED = 2 - VI_NOT_FIX = 3 - - Resolution = ( - (UNRESOLVED, 'Unresolved', CVE_VULNERABLE,VI_VULNERABLE,VI_OPEN), - (RESOLVED, 'Resolved', CVE_VULNERABLE,VI_VULNERABLE,VI_FIXED), - (FIXED, 'Fixed', CVE_VULNERABLE,VI_VULNERABLE,VI_FIXED), - (WILL_NOT_FIX, 'Won\'t Fix', CVE_VULNERABLE,VI_VULNERABLE,VI_NOT_FIX), - (WITHDRAWN, 'Withdrawn', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), - (REJECTED, 'Rejected', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), - (DUPLICATE, 'Duplicate', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), - (NOT_APPLICABLE, 'Not Applicable', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), - (REPLACED_BY_REQUIREMENT, 'Replaced By Requirement',CVE_VULNERABLE,VI_VULNERABLE,VI_CLOSED), - (CANNOT_REPRODUCE, 'Cannot Reproduce', CVE_NOT_VULNERABLE,VI_NOT_VULNERABLE,VI_CLOSED), - (DONE, 'Done', CVE_VULNERABLE,VI_VULNERABLE,VI_CLOSED), - ) - for i in range(len(Resolution)): - if r == Resolution[i][1]: - return Resolution[i][0],Resolution[i][2],Resolution[i][3],Resolution[i][4] - print("ERROR: unknown resolution string '%s=%s'" % (j,r)) - log.write("ERROR: unknown resolution string '%s=%s'" % (j,r)) - return 0,0,0,0 - -#handles updating a list of issues for a single product/project -#DOES NOT CALL COMMIT (should change this?) -def update_project_issues(project, issues, conn, log): - global force_update - - PRODUCT_ID = 0 - PRODUCT_NAME = 1 - PRODUCT_VERSION = 2 - PRODUCT_PROFILE = 3 - PRODUCT_PREFIX = 6 - - #CREATE TABLE "orm_defect" ( - #0 "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT - #1 "name" varchar(50) NOT NULL - #2 "summary" text NOT NULL - #3 "url" text NOT NULL - #4 "priority" integer NOT NULL - #5 "status" integer NOT NULL - #6 "resolution" integer NOT NULL - #7 "publish" text NOT NULL - #8 "release_version" varchar(50) NOT NULL - #9 "product_id" integer NULL REFERENCES "orm_product" ("id") - #10 "date_created" varchar(50) NOT NULL - #11 "date_updated" varchar(50) NOT NULL); - - # ORM record indexes - DEFECT_ID = 0 - DEFECT_DATE_UPDATED = 10 - ID = 0 - NAME = 1 - CVE_DESCRIPTION = 13 - CV_V_ID = 1 - CV_C_ID = 2 - I_VULNERABILITY_ID = 2 - I_PRODUCT_ID = 3 - - USER_SRTOOL_ID = "SRTool" - - PUBLISHED_FIELD = 'customfield_10010' - FIX_VERSION_FIELD = 'customfield_11002' - - d = Defect() - d.project = project[PRODUCT_NAME] + " " + project[PRODUCT_VERSION] + " " + project[PRODUCT_PROFILE] - d.product_id = project[PRODUCT_ID] - - cve_regex = re.compile("CVE-\d+-\d+") - - c = conn.cursor() - d_cursor = conn.cursor() - for i,issue in enumerate(issues): - - if get_override('SRTDBG_MINIMAL_DB') and (i > 10): - break - - ### TODO: fill in values from defect database record - #d.name = ... - #d.date_updated = ... - #d.date_created = ... - #d.summary = ... - #d.url = ... - #d.priority = ... - #d.status = ... - #d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name, ...) - #d.publish = ... - #d.release_version = ... - - sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name - defect = c.execute(sql).fetchone() - - #if defect does not exists then create it, if defect is out of date then update the database record, else ignore - if defect is None: - log.write("\tINSERTING %s\n" % d.name) - sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)''' - c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated)) - # Get the new id - sql = "SELECT * FROM orm_defect WHERE name='%s'" % d.name - defect = c.execute(sql).fetchone() - defect_id = defect[DEFECT_ID] - elif force_update or (d.date_updated > defect[DEFECT_DATE_UPDATED]): - log.write("\tUPDATING %s\n" % d.name) - sql = '''UPDATE orm_defect SET summary=?, priority=?, status=?, resolution=?, publish=?, release_version=?, date_updated=? WHERE id = ?''' - c.execute(sql, (d.summary, d.priority, d.status, d.resolution, d.publish, d.release_version, d.date_updated, defect[DEFECT_ID])) - defect_id = defect[DEFECT_ID] - else: - log.write("\tSKIPPING %s\n" % d.name) - continue - - # - # Update CVE -> Vulnerability -> Investigation -> this Defect chain - # - - # V/I severity and status minimum from defect - # Add audit lines - - # Find parent CVE - m = cve_regex.search(d.summary) - if m: - cve_name = m.group(0) - else: - print("WARNING: Missing CVE in defect name '%s'" % (d.summary)) - continue - sql = "SELECT * FROM orm_cve WHERE name='%s'" % cve_name - cve = c.execute(sql).fetchone() - if not cve: - # create the placeholder CVE - log.write("\tINSERTING CVE for %s\n" % cve_name) - print("INSERTING CVE for %s,%s" % (cve_name,d.name)) - sql = ''' INSERT into orm_cve (name, source, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cpe_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV3_vectorString, cvssV3_exploitabilityScore, cvssV3_impactScore, cvssV3_attackVector, cvssV3_attackComplexity, cvssV3_privilegesRequired, cvssV3_userInteraction, cvssV3_scope, cvssV3_confidentialityImpact, cvssV3_integrityImpact, cvssV3_availabilityImpact, cvssV2_baseScore, cvssV2_severity, cvssV2_vectorString, cvssV2_exploitabilityScore, cvssV2_impactScore, cvssV2_accessVector, cvssV2_accessComplexity, cvssV2_authentication, cvssV2_confidentialityImpact, cvssV2_integrityImpact) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' - c.execute(sql, (cve_name, '', d.priority, cve_status, '', '', '', '', '', 1, 0, '', 'Created from defect %s' % d.name, '', '', 0, '', '', - '','','','','','','','','','','','','','','','','','','','','','','')) - # Find the new id - print("FINDING CVE ID for %s\n" % cve_name) - c.execute("SELECT * FROM orm_cve where name = '%s'" % cve_name) - cve = c.fetchone() - c_id = cve[ID] - # Also create CVE history entry - sql = '''INSERT INTO orm_cvehistory (cve_id, comment, date, author) VALUES (?,?,?,?)''' - c.execute(sql, (c_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) -# sleep(0.1) - else: - c_id = cve[ID] - - # Find CVE's vulnerability, else create - d_cursor.execute("SELECT * FROM orm_cvetovulnerablility where cve_id = '%s'" % c_id) - c2v = d_cursor.fetchone() - if not c2v: - # Create Vulnerability - v_name = new_vulnerability_name(d_cursor) - log.write("\tINSERTING VULNERABILITY %s for %s\n" % (v_name,cve_name)) - print("INSERTING VULNERABILITY for (%s,%s)" % (cve_name,v_name)) - sql = '''INSERT INTO orm_vulnerability (name,description,cve_primary_name,public,comments,comments_private,status,outcome,severity) VALUES (?,?,?,?,?,?,?,?,?)''' - c.execute(sql, (v_name, cve[CVE_DESCRIPTION], cve_name, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome ,d.priority)) - # Find the new id - d_cursor.execute("SELECT * FROM orm_vulnerability where name = '%s'" % v_name) - v = d_cursor.fetchone() - v_id = v[ID] - # Also create CVE to Vulnerability - sql = '''INSERT INTO orm_cvetovulnerablility (vulnerability_id, cve_id) VALUES (?,?)''' - c.execute(sql, (v_id,c_id)) - # Also create Vulnerability history entry - sql = '''INSERT INTO orm_vulnerabilityhistory (vulnerability_id, comment, date, author) VALUES (?,?,?,?)''' - c.execute(sql, (v_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) -# sleep(0.1) - else: - print("FOUND VULNERABILITY ID for %s" % (cve_name)) - v_id = c2v[CV_V_ID] - - # Find CVE's investigation, else create - sql = "SELECT * FROM orm_investigation where vulnerability_id = '%s' AND product_id = '%s';" % (v_id,d.product_id) - print("I_TEST:%s" % sql) - d_cursor.execute(sql) - investigation = d_cursor.fetchone() - if not investigation: - # Create Investigation - i_name = new_investigation_name(d_cursor) - log.write("\tINSERTING INVESTIGATION for %s\n" % cve_name) - print("INSERTING INVESTIGATION for %s,%s" % (i_name,d.name)) - sql = '''INSERT INTO orm_investigation (name,vulnerability_id,product_id,public,comments,comments_private,status,outcome,severity) VALUES (?,?,?,?,?,?,?,?,?)''' - c.execute(sql, (i_name, v_id, d.product_id, True, 'Created from defect %s' % d.name, '', vi_status, vi_outcome, d.priority)) - # Find the new id - d_cursor.execute("SELECT * FROM orm_investigation where name = '%s'" % i_name) - investigation = d_cursor.fetchone() - i_id = investigation[ID] - # Also create Investigation to Vulnerability - AFFECTED = 0 - sql = '''INSERT INTO orm_vulnerabilityproduct (vulnerability_id, investigation_id, product_id, relation) VALUES (?,?,?,?)''' - c.execute(sql, (v_id,i_id,d.product_id,AFFECTED)) - # Also create Investigation history entry - sql = '''INSERT INTO orm_investigationhistory (investigation_id, comment, date, author) VALUES (?,?,?,?)''' - c.execute(sql, (i_id,'Created from defect %s' % d.name,srtool_today,USER_SRTOOL_ID)) -# sleep(0.1) - else: - print("FOUND INVESTIGATION ID for %s" % (cve_name)) - i_id = investigation[ID] - - # Add this defect to the investigation - d_cursor.execute("SELECT * FROM orm_investigationtodefect where investigation_id = '%s' and product_id = '%s' and defect_id = '%s'" % (i_id,d.product_id,defect_id)) - i2d = d_cursor.fetchone() - if not i2d: - # Create Investigation - i_name = new_investigation_name(d_cursor) - log.write("\tINSERTING INVESTIGATION to DEFECT for %s\n" % i_name) - sql = '''INSERT INTO orm_investigationtodefect (investigation_id, product_id, defect_id) VALUES (?,?,?)''' - c.execute(sql, (i_id,d.product_id,defect_id)) -# sleep(0.1) - - #print("=========================================================================================\n") - #print("\n") - c.close() - - -################################# -# Add defect record summary to SRTool defect table -# - -def add_to_defect_db(defect_name): - defect_name = defect_name.strip().upper() - - #try connecting to defect database - try: - ### TODO: open connection to defect database - # defect_db = ... - conn = sqlite3.connect(srtDbName) - c = conn.cursor() - except Exception as e: - print("xhr_investigation_commit:CONNECTION TO DATABASE FAILED:(%s)\n" % e, file=sys.stderr) - return 1 - - srtool_today = datetime.today().strftime('%Y-%m-%d') - - #Import the issue into the SRTool - try: - ### TODO: read defect record from defect database - # issue = ... - pass - except Exception as e: - print("ERROR:key '%s' does not exist(%s)" % (defect_name,e), file=sys.stderr) - return 1 - - d = Defect() - try: - ### TODO: fill in values from defect database record - d.name = defect_name - d.date_updated = srtool_today # ... - d.date_created = srtool_today # ... - d.summary = 'temp defect record' # ... - #d.url = ... - d.priority = translate_priority(d.name,'P3') # ... - d.status = translate_status(d.name,'Open') # ... - d.resolution,d.cve_status,d.vi_status,d.vi_outcome = translate_resolution(d.name,'Unresolved') # ... - #d.publish = ... - #d.release_version = ... - - # Get the product ID - PRODUCT_ID = 0 - PRODUCT_NAME = 1 - PRODUCT_VERSION = 2 - PRODUCT_PROFILE = 3 - PRODUCT_PREFIX = 6 - products = c.execute('''SELECT * FROM orm_product''').fetchall() - d.product_id = 1 - for product in products: - if d.name.startswith(product[PRODUCT_PREFIX]): - d.product_id = product[PRODUCT_ID] - break - - #log.write("\tINSERTING %s\n" % d.name) - sql = '''INSERT INTO orm_defect (name, summary, url, priority, status, resolution, publish, release_version, product_id, date_created, date_updated) VALUES (?,?,?,?,?,?,?,?,?,?,?)''' - c.execute(sql, (d.name, d.summary, d.url, d.priority, d.status, d.resolution, str(d.publish), d.release_version, d.product_id, d.date_created, d.date_updated)) - conn.commit() - c.close() - conn.close() - except Exception as e: - print("ERROR:could not find/import defect(%s)" % e, file=sys.stderr) - return 1 - -################################# -# main loop -# - -def main(argv): - global force_update - - parser = argparse.ArgumentParser(description='srtool_defect.py: manage the SRTool defect database') - parser.add_argument('--update', '-U', action='store_const', const='update', dest='command', help='Import defect states and update SRTool defect table') - parser.add_argument('--force', '-f', action='store_true', dest='force_update', help='Force updates') - parser.add_argument('--add', nargs=1, help='Add an existing defect to SRTool defect database') - - args = parser.parse_args() - - master_log = open("./update_logs/master_log.txt", "a") - - force_update = False - if None != args.force_update: - force_update = args.force_update - - if args.add: - add_to_defect_db(args.add[0]) - elif 'update' == args.command: - try: - print("BEGINNING DEFECT UPDATES PLEASE WAIT ... this can take some time") - update() - master_log.write("SRTOOL:%s:DEFECT TABLE & DEFECT ISSUES:\t\t\t...\t\t\tUPDATED\n" % (date.today())) - print("DATABASE UPDATE FINISHED\n") - except Exception as e: - master_log.write("SRTOOL:%s:DEFECT TABLE & DEFECT ISSUES:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) - print("DATABASE UPDATES FAILED ... %s" % e) - else: - print("Command not found") - -if __name__ == '__main__': - global script_pathname - - # fetch any environment overrides - set_override('SRTDBG_MINIMAL_DB') - set_override('SRTDBG_SKIP_DEFECT_IMPORT') - if get_override('SRTDBG_SKIP_DEFECT_IMPORT'): - exit(0) - - script_pathname=os.path.dirname(sys.argv[0]) - main(sys.argv[1:]) diff --git a/bin/common/srtool_email.py b/bin/common/srtool_email.py index e254a5ee..103c8be0 100755 --- a/bin/common/srtool_email.py +++ b/bin/common/srtool_email.py @@ -32,6 +32,18 @@ import argparse import smtplib from email.mime.text import MIMEText +# Setup: +toaddrs = '' +fromaddr = '' +smtpserver = '' +smtpencryption = '' +srt_user = '' +srt_passwd = '' +subject = '' +msg = '' +verbose = False +test = False + ################################# # Send the email # @@ -183,8 +195,6 @@ def main(argv): if __name__ == '__main__': - global script_pathname - script_pathname=os.path.dirname(sys.argv[0]) main(sys.argv[1:]) diff --git a/bin/common/srtool_sanity_test.py b/bin/common/srtool_sanity_test.py index bb5c3de5..4bd116a7 100755 --- a/bin/common/srtool_sanity_test.py +++ b/bin/common/srtool_sanity_test.py @@ -32,11 +32,9 @@ import os import sys -import re import argparse import sqlite3 import subprocess -from datetime import datetime, date, timedelta from django import VERSION as DJANGO_VERSION @@ -48,11 +46,13 @@ try: except ImportError: pass -srtDbName = 'srt.sqlite' - -# Values +# Setup: +verbose = False +cmd_skip = 0 +cmd_count = 0 table_counts = {} +srtDbName = 'srt.sqlite' ################################# # Helper methods @@ -210,7 +210,6 @@ def main(argv): exit(1) if __name__ == '__main__': - global srtool_basepath srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/common/srtool_update.py b/bin/common/srtool_update.py index c282f40b..131918ca 100755 --- a/bin/common/srtool_update.py +++ b/bin/common/srtool_update.py @@ -26,19 +26,16 @@ import argparse import sqlite3 import json import time +from datetime import datetime, timedelta # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) -from srt_schema import ORM +from common.srt_schema import ORM -from datetime import datetime, timedelta, date -from pprint import pprint -from urllib.request import urlopen, URLError -from urllib.parse import urlparse - -# setup -is_verbose = False +# Setup: +verbose = False +master_log = '' srtDbName = 'srt.sqlite' UPDATE_STATUS_LOG = 'update_logs/update_status.log' @@ -321,19 +318,19 @@ def main(argv): try: print("BEGINNING UPDATING DATASOURCES... this MAY take a long time") run_updates(args.force,name_filter,args.is_trial) - master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(date.today())) + master_log.write("SRTOOL:%s:UPDATING DATASOURCES:\t\t\t...\t\t\tSUCCESS\n" %(datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT))) print("FINISHED UPDATING ALL DATASOURCES\n") except Exception as e: print("FAILED UPDATING ALL DATASOURCES (%s)" % e) - master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), e)) + master_log.write("SRTOOL:%s:UPDATING DATASOURCES\t\t\t...\t\t\tFAILED ... %s\n" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT), e)) elif args.configure_ds_update: try: print("CHANGING UPDATE CONFIGURATION FOR %s" % args.configure_ds_update[0]) configure_ds_update(args.configure_ds_update[0], args.configure_ds_update[1], args.configure_ds_update[2]) - master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (date.today(), args.configure_ds_update[0])) + master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tCONFIGURED" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT), args.configure_ds_update[0])) except Exception as e: print("FAILED TO CONFIGURE UPDATE SETTINGS FOR %s" % args.configure_ds_update[0]) - master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (date.today(), args.configure_ds_update[0], e)) + master_log.write("SRTOOL:%s:%s\t\t\t...\t\t\tFAILED ... %s" % (datetime.now().strftime(ORM.DATASOURCE_DATETIME_FORMAT), args.configure_ds_update[0], e)) elif 'cron_start' == args.command: cron_start() elif 'cron_stop' == args.command: @@ -343,7 +340,5 @@ def main(argv): master_log.close() if __name__ == '__main__': - global script_pathname - from os.path import abspath script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/common/srtool_utils.py b/bin/common/srtool_utils.py index d5e66d54..e411c413 100755 --- a/bin/common/srtool_utils.py +++ b/bin/common/srtool_utils.py @@ -23,29 +23,16 @@ import os import sys -import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess -import json -import urllib # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) -from srt_schema import ORM +from common.srt_schema import ORM -from datetime import datetime, date -from pprint import pprint -from urllib.request import urlopen, URLError -from urllib.parse import urlparse - - -# setup - -is_verbose = False +# Setup: +verbose = False srtDbName = 'srt.sqlite' @@ -163,7 +150,5 @@ def main(argv): master_log.close() if __name__ == '__main__': - global script_pathname - from os.path import abspath script_pathname = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/debian/srtool_debian.py b/bin/debian/srtool_debian.py index 91092afe..a8d8b3d4 100755 --- a/bin/debian/srtool_debian.py +++ b/bin/debian/srtool_debian.py @@ -28,22 +28,16 @@ import os import sys -import re -import json import argparse import shutil -from datetime import datetime, date - -from urllib.request import urlopen, URLError, Request -from urllib.parse import urlparse +from urllib.request import urlopen # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -# Constants - +# Setup: srtDbName = 'srt.sqlite' #debian_cve_url_file = 'https://salsa.debian.org/security-tracker-team/security-tracker/blob/master/data/CVE/list' @@ -80,13 +74,11 @@ def get_override(key): # REST API, cache the results # -import collections - summary = {} def extract_text(k,parent,maxdepth): global summary - #print("FOO:%s,%s" % (type(k),parent)) + #print("EXTRACT_TEXT:%s,%s" % (type(k),parent)) if isinstance(k,dict): for l in k.keys(): if parent: @@ -245,6 +237,5 @@ def main(argv): print("Command not found") if __name__ == '__main__': - global srtool_basepath srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/mitre/srtool_mitre.py b/bin/mitre/srtool_mitre.py index 7504100e..2ac8bc08 100755 --- a/bin/mitre/srtool_mitre.py +++ b/bin/mitre/srtool_mitre.py @@ -28,24 +28,21 @@ import os import sys -import re import xml.etree.ElementTree as ET import argparse import shutil import sqlite3 -from datetime import datetime, date, timedelta +from datetime import datetime, timedelta import pytz -from urllib.request import urlopen, URLError, Request -from urllib.parse import urlparse +from urllib.request import urlopen # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -# Constants - +# Setup: srtDbName = 'srt.sqlite' srtErrorLog = 'srt_errors.txt' @@ -267,7 +264,7 @@ def append_cve_database(is_init,file_xml): root = tree.getroot() # Max count for development cycle - cmd_count = 20 if get_override('SRTDBG_MINIMAL_DB') else 0; + cmd_count = 20 if get_override('SRTDBG_MINIMAL_DB') else 0 conn = sqlite3.connect(srtDbName) cur = conn.cursor() @@ -275,7 +272,9 @@ def append_cve_database(is_init,file_xml): cur_ds = conn.cursor() datasource_id = 0 - for i,child in enumerate(root): + i = 0 + for child in root: + i += 1 if not 'Vulnerability' in child.tag: continue summary = _extract_text(child) @@ -455,6 +454,5 @@ def main(argv): print("Command not found") if __name__ == '__main__': - global srtool_basepath srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/nist/srtool_nist.py b/bin/nist/srtool_nist.py index e69420c7..37116140 100755 --- a/bin/nist/srtool_nist.py +++ b/bin/nist/srtool_nist.py @@ -27,25 +27,19 @@ import os import sys import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -import urllib from datetime import datetime, date, timedelta import pytz - from urllib.request import urlopen, URLError -from urllib.parse import urlparse # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -# setup +# Setup: lookupTable = [] cveIndex = {} db_change = False @@ -179,24 +173,24 @@ def update_nist(is_init,datasource_description, url_file, url_meta, cve_file, in log.write("BEGINNING NIST %s\n" % ('INITS' if is_init else 'UPDATES')) #determine if we are out of date and apply updates if true if (date_new > date_past) or force_update: - pre_update_time = datetime.now() #used for logging purposes only - - nist_json(is_init,nist_cve_url, ds[ORM.DATASOURCE_ID], nist_file, log, date_new, incremental) - log.write("began %s: %s\n" % ( 'init' if is_init else 'updates', str(pre_update_time) )) - log.write("finished %s: %s\n" % ( 'init' if is_init else 'updates', str(datetime.now()) )) - log.write("=============================================================================\n") - log.write("\n") - - #update datasource's lastModifiedDate after successsfuly updating it - sql = "UPDATE orm_datasource SET lastModifiedDate = ? WHERE id='%s'" % ds[ORM.DATASOURCE_ID] - c.execute(sql, (str(date_new),)) - conn.commit() + pre_update_time = datetime.now() #used for logging purposes only + + nist_json(is_init,nist_cve_url, ds[ORM.DATASOURCE_ID], nist_file, log, date_new, incremental) + log.write("began %s: %s\n" % ( 'init' if is_init else 'updates', str(pre_update_time) )) + log.write("finished %s: %s\n" % ( 'init' if is_init else 'updates', str(datetime.now()) )) + log.write("=============================================================================\n") + log.write("\n") + + #update datasource's lastModifiedDate after successsfuly updating it + sql = "UPDATE orm_datasource SET lastModifiedDate = ? WHERE id='%s'" % ds[ORM.DATASOURCE_ID] + c.execute(sql, (str(date_new),)) + conn.commit() else: - log.write("No %s needed\n" % ('init' if is_init else 'update')) - log.write("Checked: %s\n" % datetime.now()) - log.write("=============================================================================\n") - log.write("\n") - print("NO %s NEEDED" % ('INIT' if is_init else 'UPDATE')) + log.write("No %s needed\n" % ('init' if is_init else 'update')) + log.write("Checked: %s\n" % datetime.now()) + log.write("=============================================================================\n") + log.write("\n") + print("NO %s NEEDED" % ('INIT' if is_init else 'UPDATE')) # Reset datasource's lastModifiedDate as today sql = "UPDATE orm_datasource SET lastModifiedDate = ? WHERE id='%s'" % ds[ORM.DATASOURCE_ID] @@ -215,7 +209,6 @@ def update_nist(is_init,datasource_description, url_file, url_meta, cve_file, in f.close() except URLError as e: raise Exception("Failed to open %s: %s" % (nist_meta_url, e.reason)) - continue log.close() c.close() conn.close() @@ -290,7 +283,7 @@ def nist_json(is_init,summary_json_url, datasource_id, datasource_file, log, dat v.public = True # Always true since NIST is public source # We do not know yet if this has been published to the SRTool management - v.publish = ORM.PUBLISH_UNPUBLISHED + v.publish_state = ORM.PUBLISH_UNPUBLISHED v.publish_date = '' if ('impact' in CVE_Item) and ('baseMetricV3' in CVE_Item['impact']): @@ -507,7 +500,7 @@ def fetch_cve(cve_name,cve_source_file): f = open(cve_cache_path, 'r') CVE_Item = json.load(f) except Exception as e: - print("Description=ERROR reading CVE summary file '%s':e" % (cve_cache_path,e)) + print("Description=ERROR reading CVE summary file '%s':%s" % (cve_cache_path,e)) return elif cve_source_file: try: @@ -531,11 +524,11 @@ def fetch_cve(cve_name,cve_source_file): cve_cache_file.write(json.dumps(CVE_Item)) break except Exception as e: - print("Description=ERROR creating CVE cache file '%s':e" % (cve_source_file,e)) + print("Description=ERROR creating CVE cache file '%s':%s" % (cve_source_file,e)) return else: # No data source for details - return v + return if not CVE_Item: print("description=There is no CVE record for %s in the loaded NIST public CVE database." % cve_name) @@ -603,7 +596,7 @@ def fetch_cve(cve_name,cve_source_file): elif "OR" == config['operator']: summary['cpe_list'] += nist_scan_configuration_or(config, cve_name, 0) else: - print("ERROR CONFIGURE:OP?:%s" % config_rec['operator']) + print("ERROR CONFIGURE:OP?:%s" % config['operator']) summary['cpe_list'] += '[/and]|' summary['cpe_list'] += '[/config]|' @@ -728,6 +721,5 @@ def main(argv): exit(ret) if __name__ == '__main__': - global srtool_basepath srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/redhat/srtool_redhat.py b/bin/redhat/srtool_redhat.py index 27a568a7..f570965c 100755 --- a/bin/redhat/srtool_redhat.py +++ b/bin/redhat/srtool_redhat.py @@ -31,14 +31,10 @@ import sys import re import json import argparse -import subprocess -from datetime import datetime, date -from urllib.request import urlopen, URLError, Request -from urllib.parse import urlparse - -# Constants +from urllib.request import urlopen, Request +# Setup: srtDbName = 'srt.sqlite' redhat_cache_dir = 'data/cache/redhat' @@ -73,8 +69,6 @@ def get_override(key): # REST API, cache the results # -import collections - summary = {} def extract_json(k,parent,maxdepth): @@ -132,7 +126,7 @@ def fetch_cve(cve_name): msg = e # Cache the error result so that we do not keep hitting that server with open(datasource_file, 'w') as fp: - json.dump({'bugzilla': {'description':msg}}, fp) + json.dump({'bugzilla': {'description':msg}}, fp) print("description=%s" % msg) return @@ -147,7 +141,7 @@ def fetch_cve(cve_name): extract_json(dct,'',10) # for key in summary.keys(): -# print("FOO %s=%s" % (key,summary[key])) +# print("KEY_LIST: %s=%s" % (key,summary[key])) # Translate the content to NIST schema results = {} @@ -199,6 +193,5 @@ def main(argv): print("Command not found") if __name__ == '__main__': - global srtool_basepath srtool_basepath = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0])))) main(sys.argv[1:]) diff --git a/bin/yp/srtool_defect.py b/bin/yp/srtool_defect.py index 56a50591..77f8e030 100755 --- a/bin/yp/srtool_defect.py +++ b/bin/yp/srtool_defect.py @@ -21,30 +21,25 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. ### Usage Examples (run from top level directory) -# Updating Defect System Issues: ./bin/acme/srtool_defect.py -U - +# Updating Defect System Issues: ./bin/<app>/srtool_defect.py -U import os import sys -import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -import urllib -from time import sleep -from datetime import datetime # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -from datetime import datetime, date -from urllib.request import urlopen, URLError -from urllib.parse import urlparse +# Setup: +master_log = '' +force_update = False +verbose = False +srt_user = '' +srt_passwd = '' srtDbName = 'srt.sqlite' srtErrorLog = 'srt_errors.txt' @@ -53,8 +48,6 @@ srtErrorLog = 'srt_errors.txt' # Helper methods # -verbose = False - def debugMsg(msg): if verbose: print(msg) @@ -302,8 +295,6 @@ def main(argv): sys.exit("Command '%s' not found" % args.command) if __name__ == '__main__': - global srtool_basepath - if verbose: print("srtool_defect(%s)" % sys.argv[1:]) # fetch any environment overrides diff --git a/bin/yp/srtool_yp.py b/bin/yp/srtool_yp.py index 552eadf1..6caa7863 100755 --- a/bin/yp/srtool_yp.py +++ b/bin/yp/srtool_yp.py @@ -26,25 +26,16 @@ import os import sys -import re -import csv -import xml.etree.ElementTree as ET import argparse import sqlite3 -import subprocess import json -import urllib -from datetime import datetime # load the srt.sqlite schema indexes dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, dir_path) from common.srt_schema import ORM -from datetime import datetime, date -from urllib.request import urlopen, URLError -from urllib.parse import urlparse - +# Setup: srtDbName = 'srt.sqlite' @@ -142,9 +133,6 @@ def main(argv): print("Command not found") if __name__ == '__main__': - global script_pathname - global srtool_basepath - srtool_scriptpath = os.path.dirname(os.path.dirname(os.path.abspath(sys.argv[0]))) srtool_basepath = os.path.dirname(srtool_scriptpath) main(sys.argv[1:]) |