diff options
Diffstat (limited to 'bin/srtool_cve.py')
| -rwxr-xr-x | bin/srtool_cve.py | 608 |
1 files changed, 608 insertions, 0 deletions
diff --git a/bin/srtool_cve.py b/bin/srtool_cve.py new file mode 100755 index 00000000..343b8138 --- /dev/null +++ b/bin/srtool_cve.py @@ -0,0 +1,608 @@ +#!/usr/bin/env python3 +# +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# Security Response Tool Commandline Tool +# +# Copyright (C) 2018 Wind River Systems +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +### Usage Examples (run from top level directory) +# Updating a specific NIST feed: ./bin/srtool.py -u "NIST JSON Data 2017" +# Updating with the NIST incremental feed: ./bin/srtool.py -U + +import os +import sys +import re +import csv +import xml.etree.ElementTree as ET +import argparse +import sqlite3 +import subprocess +import json +import urllib + +try: + from datetime import datetime, date + from urllib.request import urlopen, URLError + from urllib.parse import urlparse +except ImportError: + from urllib2 import urlopen, URLError + from urlparse import urlparse + +# setup +lookupTable = [] +cveIndex = {} +db_change = False + +srtDbName = 'srt.sqlite' +is_verbose = False + +keywords_for = [] +keywords_against = [] + +################################# +# Helper methods +# + +overrides = {} + +def set_override(key,value=None): + if not value is None: + overrides[key] = value + elif key in os.environ.keys(): + overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no' + else: + overrides[key] = '' + print("OVERRIDE: %s = %s" % (key,overrides[key])) + +def get_override(key): + if key in overrides.keys(): + return overrides[key] + return '' + + +################################# +# check for updates and apply if any +# +# Change orm_datasource schema to make LastModifiedDate a datetime object +# datetime and urllib imports may be in an inappropriate location (top of file currently) + +#gets CVE-Modified feed, determines if we are out of date, and applies updates if true +#tracks history in update_log.txt +#incremental argument is boolean that idicates if bulk updating or incremental updating. +def update_nist(datasource_description, incremental): + DS_ID = 0 + DS_DATA = 1 + DS_SOURCE = 2 + DS_TYPE = 3 + DS_DESCRIPTION = 4 + DS_FILE_PATH = 5 + DS_URL = 6 + DS_LOADED = 7 + DS_META_URL = 8 + DS_LAST_MODIFIED_DATE = 9 + + #update log (1=Monday, 7= Sunday) + today = datetime.today() + weeknum = today.strftime("%W") + weekday = today.isoweekday() + log = open(os.path.join(SRT_BASE_DIR,"update_logs/update_nist_log_%s_%s.txt" % (weeknum, weekday)), "a") + + + #ensure cache folder exists (clear cache during "run_all_updates()" from "srtool_utils.py") + path = os.path.join(SRT_BASE_DIR, "data/cache") + try: + os.makedirs(path) + except: + pass + + # Set up database connection + conn = sqlite3.connect(srtDbName) + c = conn.cursor() + + # Fetch the keywords used to generate recommendation scores + global keywords_for + global keywords_against + sql = "SELECT * FROM orm_srtsetting WHERE name=?" + keywords_for = c.execute(sql, ("keywords_for",)).fetchone()[3].split("|") + keywords_against = c.execute(sql, ("keywords_against",)).fetchone()[3].split("|") + + sql = "SELECT * FROM orm_datasource WHERE description='%s'" % datasource_description + c.execute(sql) + for ds in c: + try: + f = urlopen(ds[DS_META_URL]) #Note: meta files are not in json format, hence manual parse + content = f.readline().decode('UTF-8') + date_new = datetime.strptime(content, 'lastModifiedDate:%Y-%m-%dT%H:%M:%S-04:00\r\n') + date_past = datetime.strptime(ds[DS_LAST_MODIFIED_DATE], '%Y-%m-%d %H:%M:%S') + + log.write("BEGINNING NIST UPDATES\n") + #determine if we are out of date and apply updates if true + if (date_new > date_past): + pre_update_time = datetime.now() #used for logging purposes only + + nist_json(ds[DS_FILE_PATH], ds[DS_URL], ds[DS_ID], ds[DS_FILE_PATH], log, date_past, incremental) + log.write("began updates: %s\n" % str(pre_update_time)) + log.write("finished updates: %s\n" % str(datetime.now()) ) + log.write("=============================================================================\n") + log.write("\n") + + #update datasource's lastModifiedDate after successsfuly updating it + sql = "UPDATE orm_datasource SET lastModifiedDate = ? WHERE id='%s'" % ds[DS_ID] + #c.execute(sql, (str(date_new),)) + conn.commit() + else: + + log.write("No update needed\n") + log.write("Checked: %s\n" % datetime.now()) + log.write("=============================================================================\n") + log.write("\n") + print("NO UPDATE NEEDED") + + # Reset datasource's update_time as today + sql = "UPDATE orm_datasource SET update_time = ? WHERE id='%s'" % ds[DS_ID] + c.execute(sql, (datetime.today().strftime('%Y-%m-%d %H:%M:%S'),) ) + conn.commit() + + ####### + ## TESTING PURPOSES ONLY: reset lastModifiedDate so will always need update! + ####### + # sql = '''UPDATE orm_datasource + # SET lastModifiedDate = "0001-01-01 01:01:01" + # WHERE description="NIST JSON Modified Data 2017" ''' + # c.execute(sql) + # conn.commit() + + f.close() + except URLError as e: + raise Exception("Failed to open %s: %s" % (href, e.reason)) + continue + log.close() + c.close() + conn.close() + +def nist_scan_configuration_or(cve, cpe_or_node, name, and_enum): + cpe_list = '[or]|' + for cpe in cpe_or_node['cpe']: + cpe23Uri = cpe['cpe23Uri'] + if 'cpeMatchString' in cpe: + cpeMatchString = cpe['cpeMatchString'] + else: + cpeMatchString = '' + if 'versionEndIncluding' in cpe: + versionEndIncluding = cpe['versionEndIncluding'] + else: + versionEndIncluding = '' + cpe_list += '%s,%s,%s,%s|' % (cpe['vulnerable'],cpe23Uri,cpeMatchString,versionEndIncluding) + cpe_list += '[/or]|' + return cpe_list + +#parses JSON, creates CVE object, and updates database as necessary. Commits to database on success +#will EITHER create new record in orm_cve if cve does not exist OR overwrite every field if existing cve out-of-date OR ignore cve +#requires json to be formatted with NIST Json schema (https://csrc.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema) +def nist_json(file_path, json_url, datasource_id, datasource_filename, log, date_past, incremental): + import traceback + import gzip + import requests + + response = urlopen(json_url) + dct = json.loads(gzip.decompress(response.read()).decode('utf-8')) #uncompress and decode json.gz + + #save datasource feed to "data" + datasource_file = open(os.path.join(SRT_BASE_DIR, datasource_filename), 'w+') + datasource_file.write(json.dumps(dct)) + + conn = sqlite3.connect(srtDbName) + c = conn.cursor() + + CVE_Items = dct['CVE_Items'] + total = len(CVE_Items) + v = Cve() + + cache_path = os.path.join(SRT_BASE_DIR, "data/cache") + #begin parsing each cve in the JSON data + for i, CVE_Item in enumerate(CVE_Items): + # Development support + if get_override('SRTDBG_MINIMAL_DB') and (i > 10): + break + + cve = CVE_Item['cve'] + references = cve['references']['reference_data'] + CVE_data_meta = cve['CVE_data_meta']['ID'] + + #if cve exists in cache, delete it + cve_path = os.path.join(cache_path, CVE_data_meta + ".json") + if (os.path.isfile(cve_path)): + os.remove(cve_path) + + #if incremental update, save json copy of the cve to cache + if incremental: + file = open(cve_path, 'w+') + file.write(json.dumps(CVE_Item)) + + #print('.', end='', flush=True) + print('[%4d]%30s\r' % ((i * 100)/ total, CVE_data_meta), end='', flush=True) + try: + v.name = CVE_data_meta + v.source = 'NIST' + v.datasource = datasource_id + + v.cve_data_type = cve['data_type'] + v.cve_data_format = cve['data_format'] + v.cve_data_version = cve['data_version'] + + v.description = cve['description']['description_data'][0]['value'] + v.publishedDate = re.sub('T.*','',CVE_Item['publishedDate']) + v.lastModifiedDate = re.sub('T.*','',CVE_Item['lastModifiedDate']) + v.public = True #always true since NIST is public source + v.publish = Cve.PUBLISH_PUBLISHED + v.publish_date = v.publishedDate + + if ('impact' in CVE_Item) and ('baseMetricV3' in CVE_Item['impact']): + baseMetricV3 = CVE_Item['impact']['baseMetricV3'] + v.cvssV3_baseScore = baseMetricV3['cvssV3']['baseScore'] + v.cvssV3_baseSeverity = baseMetricV3['cvssV3']['baseSeverity'] + # v.cvssV3_vectorString = baseMetricV3['cvssV3']['vectorString'] + # v.cvssV3_exploitabilityScore = baseMetricV3['exploitabilityScore'] + # v.cvssV3_impactScore = baseMetricV3['impactScore'] + # v.cvssV3_attackVector = baseMetricV3['cvssV3']['attackVector'] + # v.cvssV3_attackComplexity = baseMetricV3['cvssV3']['attackComplexity'] + # v.cvssV3_privilegesRequired = baseMetricV3['cvssV3']['privilegesRequired'] + # v.cvssV3_userInteraction = baseMetricV3['cvssV3']['userInteraction'] + # v.cvssV3_scope = baseMetricV3['cvssV3']['scope'] + # v.cvssV3_confidentialityImpact = baseMetricV3['cvssV3']['confidentialityImpact'] + # v.cvssV3_integrityImpact = baseMetricV3['cvssV3']['integrityImpact'] + # v.cvssV3_availabilityImpact = baseMetricV3['cvssV3']['availabilityImpact'] + if ('impact' in CVE_Item) and ('baseMetricV2' in CVE_Item['impact']): + baseMetricV2 = CVE_Item['impact']['baseMetricV2'] + v.cvssV2_baseScore = baseMetricV2['cvssV2']['baseScore'] + # v.cvssV2_severity = baseMetricV2['severity'] + # v.cvssV2_vectorString = baseMetricV2['cvssV2']['vectorString'] + # v.cvssV2_exploitabilityScore = baseMetricV2['exploitabilityScore'] + # v.cvssV2_impactScore = baseMetricV2['exploitabilityScore'] + # v.cvssV2_accessVector = baseMetricV2['cvssV2']['accessVector'] + # v.cvssV2_accessComplexity = baseMetricV2['cvssV2']['accessComplexity'] + # v.cvssV2_authentication = baseMetricV2['cvssV2']['authentication'] + # v.cvssV2_confidentialityImpact = baseMetricV2['cvssV2']['confidentialityImpact'] + # v.cvssV2_integrityImpact = baseMetricV2['cvssV2']['integrityImpact'] + + # configurations = CVE_Item['configurations'] + # is_first_and = True + # for i, config in enumerate(configurations['nodes']): + # v.cpe_list += '[config]|' + # v.cpe_list += '[and]|' + # if "AND" == config['operator']: + # # create AND record + # if not is_first_and: + # v.cpe_list += '[/and]|' + # v.cpe_list += '[and]|' + # #is_first_and = False + # if 'children' in config: + # for j, cpe_or_node in enumerate(config['children']): + # if "OR" == cpe_or_node['operator']: + # v.cpe_list += nist_scan_configuration_or(v,cpe_or_node, CVE_data_meta, j) + # else: + # print("ERROR CONFIGURE:OR_OP?:%s" % cpe_or_node['operator']) + # elif "OR" == config['operator']: + # v.cpe_list += nist_scan_configuration_or(v,config, CVE_data_meta, 0) + # else: + # print("ERROR CONFIGURE:OP?:%s" % config_rec['operator']) + # v.cpe_list += '[/and]|' + # v.cpe_list += '[/config]|' + + + #check if cve object `v` need to be uploaded to database (cases: new cve, modified cve, or no changes) + #if true, apply changes. Else ignore and continue + v_id, is_change = sql_cve_query(conn, v, log) + + + #if CVE `v` updates, must check and update associated records (CWEs, references, and CVE2CWE) + #sql_reference_query, sql_cwe_query, and sql_cve2cwe_query require valid CVE record primary key at some point during their execution, therefore must always be after call to sql_cve_query + if is_change: + r = Reference() + for ref in references: + r.cve_id = v_id + r.hyperlink = ref['url'] + sql_reference_query(conn, r) + + problem_list = cve['problemtype']['problemtype_data'] + for problem_Item in problem_list: + description_list = problem_Item['description'] + for description_Item in description_list: + value = description_Item['value'] + cwe_id = sql_cwe_query(conn, value) + sql_cve2cwe_query(conn, v_id, cwe_id) + + except Exception as e: + print(traceback.format_exc()) + print("UPDATE FAILED") + c.close() + conn.close() + return + print() + log.write("total number of CVEs checked: %s\n" % total) + conn.commit() + c.close() + conn.close() + +################################# +# cve class +# +class Cve(): + # WR Status + NEW = 0 + EXCLUDE = 1 + INVESTIGATE = 2 + VULNERABLE = 3 + NOT_VULNERABLE = 4 + STATUS = ( + (NEW, 'New'), + (EXCLUDE, 'Exclude'), + (INVESTIGATE, 'Investigate'), + (VULNERABLE, 'Vulnerable'), + (NOT_VULNERABLE, 'Not Vulnerable'), + ) + + # Publish options + PUBLISH_UNPUBLISHED = 0 + PUBLISH_NOPUBLISH = 1 + PUBLISH_PUBLISHED = 2 + PUBLISH_REQUEST = 3 + PUBLISH_UPDATE = 4 + PUBLISH_SUBMITTED = 5 + + # index - primary key + id = -1 + + name = '' + datasource = 0 + + source = '' + priority = 0 + status = NEW + + comments = '' + comments_private = '' + + cve_data_type = '' + cve_data_format = '' + cve_data_version = '' + + public = False + publish_state = PUBLISH_UNPUBLISHED + publish_date = '' + + description = '' + publishedDate = '' + lastModifiedDate = '' + problemtype = '' + + # cpe_list = '' + + cvssV3_baseScore = '' + cvssV3_baseSeverity = '' + # cvssV3_vectorString = '' + # cvssV3_exploitabilityScore = '' + # cvssV3_impactScore = '' + # cvssV3_attackVector = '' + # cvssV3_attackComplexity = '' + # cvssV3_privilegesRequired = '' + # cvssV3_userInteraction = '' + # cvssV3_scope = '' + # cvssV3_confidentialityImpact = '' + # cvssV3_integrityImpact = '' + # cvssV3_availabilityImpact = '' + + cvssV2_baseScore = '' + cvssV2_severity = '' + # cvssV2_vectorString = '' + # cvssV2_exploitabilityScore = '' + # cvssV2_impactScore = '' + # cvssV2_accessVector = '' + # cvssV2_accessComplexity = '' + # cvssV2_authentication = '' + # cvssV2_confidentialityImpact = '' + # cvssV2_integrityImpact = '' + + recommend = 0 + recommend_list = [] + + #generates importance score based on key-words in description of CVE, higher indicates more important + def recommendation(self): + description = ' '+self.description.lower()+' ' + total = 0 + list = '' + + for keypair in keywords_for: + #print("keypair='%s'" % keypair) + key,w = keypair.split(',') + weight = 1 +# if w: +# weight = int(w) + #if ' '+key+' ' in description: + if re.search(r'\b%s\b' % key, description): + list += ",+%s" % key + total += weight + + for keypair in keywords_against: + #print("keypair='%s'" % keypair) + key,w = keypair.split(',') + weight = 1 +# if w: +# weight = int(w) + # re.search(r'\bis\b', your_string) + #if ' '+key+' ' in description: + #if re.search(re.escape(r'\b%s\b' % key), description): + if re.search(r'\b%s\b' % key, description): + list += ",-%s" % key + total -= weight + + # set filter maximums + if total < -3: + total = -3 + if total > 3: + total = 3 + self.recommend = total + self.recommend_list = list[1:] + +#generates and executes appropriate SQLite query for CVE depending on situation +#new CVE -> INSERT || modified CVE -> UPDATE || no change -> ignore and return +#returns (CVE_ID, BOOL) tuple, True if insert or update executed +### THIS DOES NOT CALL CONNECTION.COMMIT() +def sql_cve_query(conn, cve, log): + CVE_LASTMODIFIEDDATE = 14 + is_change = False + cur = conn.cursor() + sql = '''SELECT * FROM orm_cve WHERE name=?''' + exists = cur.execute(sql, (cve.name,)).fetchone() + cve_id = -1 + if exists is None: + cve.recommendation() + sql = ''' INSERT into orm_cve (name, source, datasource_id, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)''' + cur.execute(sql, (cve.name, cve.source, cve.datasource, cve.priority, cve.status, cve.comments, cve.comments_private, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, 1, cve.publish_state, cve.publish_date, cve.description, cve.publishedDate, cve.lastModifiedDate, cve.recommend, cve.recommend_list, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity)) + is_change = True + cve_id = cur.lastrowid + log.write("\tINSERTED '%s'\n" % cve.name) + elif exists[CVE_LASTMODIFIEDDATE] < cve.lastModifiedDate: + cve.recommendation() + sql = ''' UPDATE orm_cve + SET source = ?, + recommend = ?, + recommend_list = ?, + cve_data_type = ?, + cve_data_format = ?, + cve_data_version = ?, + description = ?, + lastModifiedDate = ?, + cvssV3_baseScore = ?, + cvssV3_baseSeverity = ?, + cvssV2_baseScore = ?, + cvssV2_severity = ? + WHERE id = ?''' + cur.execute(sql, (cve.source, cve.recommend, cve.recommend_list, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, cve.description, cve.lastModifiedDate, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, exists[0])) + is_change = True + log.write("\tUPDATED '%s'\n" % cve.name) + cve_id = exists[0] + else: + is_change = False + log.write("\tSKIPPED '%s'\n" % cve.name) + cur.close() + return (cve_id, is_change) + + +################################# +# reference +# +class Reference(): + cve_id = -1 + hyperlink = '' + resource = '' + type_ = '' + source = '' + name = '' + +#generates and executes appropriate SQLite query for a new reference +### THIS DOES NOT CALL CONNECTION.COMMIT() +def sql_reference_query(conn, ref): + cur = conn.cursor() + sql = '''SELECT 1 FROM orm_cvereference WHERE hyperlink=?''' + record = cur.execute(sql, (ref.hyperlink,)).fetchone() + if record is None: + sql = '''INSERT INTO orm_cvereference (cve_id, hyperlink) VALUES (?, ?)''' + cur.execute(sql, (ref.cve_id, ref.hyperlink)) + cur.close() + +################################# +# cwe and cve2cwe +# + +#generates and executes appropriate SQLite query for a new CWE +#returns CWE_ID +### THIS DOES NOT CALL CONNECTION.COMMIT() +def sql_cwe_query(conn, value): + cur = conn.cursor() + sql = '''SELECT 1 FROM orm_cwetable WHERE name=?''' + cwe = cur.execute(sql, (value,)).fetchone() + if cwe is None: + sql = '''INSERT INTO orm_cwetable (name, name_sort, href, summary, description, vulnerable_count, found) VALUES (?,'','','','',1,1)''' + cur.execute(sql, (value,)) + cwe = cur.lastrowid + cur.close() + return cwe + cur.close() + return cwe[0] + +#generates and executes appropriate SQLite query for new CVE to CWE relation +### THIS DOES NOT CALL CONNECTION.COMMIT() +def sql_cve2cwe_query(conn, cve_id, cwe_id): + cur = conn.cursor() + sql = '''SELECT * FROM orm_cvetocwe WHERE cve_id=? AND cwe_id=?''' + cve2cwe = cur.execute(sql, (cve_id, cwe_id)) + if cve2cwe is None: + sql = '''INSERT INTO orm_cvetocwe (cve_id, cwe_id) VALUES (?, ?)''' + cur.execute(sql, (cve_id, cwe_id)) + cur.close() + +################################# +# main loop +# + +def main(argv): + parser = argparse.ArgumentParser(description='srtool_cve.py: manage the CVEs within SRTool database') + + parser.add_argument('--update_nist', '-n', nargs=1, help='Check nvd.nist.gov/vuln/data-feeds for updates on a specified datasource') + parser.add_argument('--update_nist_incremental', '-i', action='store_const', const='update_nist_incremental', dest='command', help='Check nvd.nist.gov/vuln/data-feeds for updates') + args = parser.parse_args() + + master_log = open(os.path.join(SRT_BASE_DIR, "update_logs/master_log.txt"), "a") + + if args.update_nist: + try: + print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time") + update_nist(args.update_nist[0], False) + master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tUPDATED\n" % (date.today(), args.update_nist[0])) + print("DATABASE UPDATE FINISHED\n") + except Exception as e: + print("DATABASE UPDATED FAILED ... %s" % e) + master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), args.update_nist[0], e)) + elif 'update_nist_incremental' == args.command: + try: + print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time") + update_nist('NIST JSON Modified Data', True) + master_log.write("SRTOOL:%s:'NIST JSON Modified Data':\t\t\t...\t\t\tUPDATED\n" % date.today()) + print("DATABASE UPDATE FINISHED\n") + except Exception as e: + print("DATABASE UPDATED FAILED ... %s" % e) + master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), 'NIST JSON Modified Data', e)) + + else: + print("Command not found") + master_log.close() + +if __name__ == '__main__': + global SRT_BASE_DIR + from os.path import abspath + # fetch any environment overrides + set_override('SRTDBG_SKIP_CVE_IMPORT') + set_override('SRTDBG_MINIMAL_DB') + if get_override('SRTDBG_SKIP_CVE_IMPORT'): + exit(0) + + SRT_BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(sys.argv[0]))) + + main(sys.argv[1:]) |