path: root/bin/srtool_cve.py

#!/usr/bin/env python3
#
# ex:ts=4:sw=4:sts=4:et
# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
#
# Security Response Tool Commandline Tool
#
# Copyright (C) 2018       Wind River Systems
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

### Usage Examples (run from top level directory)
# Updating a specific NIST feed:            ./bin/srtool.py -u "NIST JSON Data 2017"
# Updating with the NIST incremental feed:  ./bin/srtool.py -U

import os
import sys
import re
import csv
import xml.etree.ElementTree as ET
import argparse
import sqlite3
import subprocess
import json
import urllib

try:
    from datetime import datetime, date
    from urllib.request import urlopen, URLError
    from urllib.parse import urlparse
except ImportError:
    from urllib2 import urlopen, URLError
    from urlparse import urlparse

# setup
lookupTable = []
cveIndex = {}
db_change = False

srtDbName = 'srt.sqlite'
is_verbose = False

keywords_for = []
keywords_against = []

#################################
# Helper methods
#

overrides = {}

def set_override(key,value=None):
    if not value is None:
        overrides[key] = value
    elif key in os.environ.keys():
        overrides[key] = 'yes' if os.environ[key].startswith('1') else 'no'
    else:
        overrides[key] = ''
    print("OVERRIDE: %s = %s" % (key,overrides[key]))

def get_override(key):
    if key in overrides.keys():
        return overrides[key]
    return ''


#################################
# check for updates and apply if any
#
# Change orm_datasource schema to make LastModifiedDate a datetime object
# datetime and urllib imports may be in an inappropriate location (top of file currently)

#gets CVE-Modified feed, determines if we are out of date, and applies updates if true
#tracks history in update_log.txt
#incremental argument is boolean that idicates if bulk updating or incremental updating.
def update_nist(datasource_description, incremental):
    DS_ID = 0
    DS_DATA = 1
    DS_SOURCE = 2
    DS_TYPE = 3
    DS_DESCRIPTION = 4
    DS_FILE_PATH = 5
    DS_URL = 6
    DS_LOADED = 7
    DS_META_URL = 8
    DS_LAST_MODIFIED_DATE = 9

    #update log (1=Monday, 7= Sunday)
    today = datetime.today()
    weeknum = today.strftime("%W")
    weekday = today.isoweekday()
    log = open(os.path.join(SRT_BASE_DIR,"update_logs/update_nist_log_%s_%s.txt" % (weeknum, weekday)), "a")


    #ensure cache folder exists (clear cache during "run_all_updates()" from "srtool_utils.py")
    path = os.path.join(SRT_BASE_DIR, "data/cache")
    try:
        os.makedirs(path)
    except:
        pass

    # Set up database connection
    conn = sqlite3.connect(srtDbName)
    c = conn.cursor()

    # Fetch the keywords used to generate recommendation scores
    global keywords_for
    global keywords_against
    sql = "SELECT * FROM orm_srtsetting WHERE name=?"
    keywords_for = c.execute(sql, ("keywords_for",)).fetchone()[3].split("|")
    keywords_against = c.execute(sql, ("keywords_against",)).fetchone()[3].split("|")

    sql = "SELECT * FROM orm_datasource WHERE description='%s'" % datasource_description
    c.execute(sql)
    for ds in c:
        try:
            f = urlopen(ds[DS_META_URL]) #Note: meta files are not in json format, hence manual parse
            content = f.readline().decode('UTF-8')
            date_new = datetime.strptime(content, 'lastModifiedDate:%Y-%m-%dT%H:%M:%S-04:00\r\n')
            date_past = datetime.strptime(ds[DS_LAST_MODIFIED_DATE], '%Y-%m-%d %H:%M:%S')

            log.write("BEGINNING NIST UPDATES\n")
            #determine if we are out of date and apply updates if true
            if (date_new > date_past):
               pre_update_time = datetime.now()   #used for logging purposes only

               nist_json(ds[DS_FILE_PATH], ds[DS_URL], ds[DS_ID], ds[DS_FILE_PATH], log, date_past, incremental)
               log.write("began updates: %s\n" % str(pre_update_time))
               log.write("finished updates: %s\n" % str(datetime.now()) )
               log.write("=============================================================================\n")
               log.write("\n")

               #update datasource's lastModifiedDate after successsfuly updating it
               sql = "UPDATE orm_datasource SET lastModifiedDate = ? WHERE id='%s'" % ds[DS_ID]
               #c.execute(sql, (str(date_new),))
               conn.commit()
            else:

               log.write("No update needed\n")
               log.write("Checked: %s\n" % datetime.now())
               log.write("=============================================================================\n")
               log.write("\n")
               print("NO UPDATE NEEDED")

            # Reset datasource's update_time as today
            sql = "UPDATE orm_datasource SET update_time = ? WHERE id='%s'" % ds[DS_ID]
            c.execute(sql, (datetime.today().strftime('%Y-%m-%d %H:%M:%S'),) )
            conn.commit()

            #######
            ## TESTING PURPOSES ONLY: reset lastModifiedDate so will always need update!
            #######
            # sql = '''UPDATE orm_datasource
            #        SET lastModifiedDate = "0001-01-01 01:01:01"
            #        WHERE description="NIST JSON Modified Data 2017" '''
            # c.execute(sql)
            # conn.commit()

            f.close()
        except URLError as e:
            raise Exception("Failed to open %s: %s" % (href, e.reason))
            continue
    log.close()
    c.close()
    conn.close()

def nist_scan_configuration_or(cve, cpe_or_node, name, and_enum):
    cpe_list = '[or]|'
    for cpe in cpe_or_node['cpe']:
        cpe23Uri = cpe['cpe23Uri']
        if 'cpeMatchString' in cpe:
            cpeMatchString = cpe['cpeMatchString']
        else:
            cpeMatchString = ''
        if 'versionEndIncluding' in cpe:
            versionEndIncluding = cpe['versionEndIncluding']
        else:
            versionEndIncluding = ''
        cpe_list += '%s,%s,%s,%s|' % (cpe['vulnerable'],cpe23Uri,cpeMatchString,versionEndIncluding)
    cpe_list += '[/or]|'
    return cpe_list

#parses JSON, creates CVE object, and updates database as necessary. Commits to database on success
#will EITHER create new record in orm_cve if cve does not exist OR overwrite every field if existing cve out-of-date OR ignore cve
#requires json to be formatted with NIST Json schema (https://csrc.nist.gov/schema/nvd/feed/0.1/nvd_cve_feed_json_0.1_beta.schema)
def nist_json(file_path, json_url, datasource_id, datasource_filename, log, date_past, incremental):
    import traceback
    import gzip
    import requests

    response = urlopen(json_url)
    dct = json.loads(gzip.decompress(response.read()).decode('utf-8'))  #uncompress and decode json.gz

    #save datasource feed to "data"
    datasource_file = open(os.path.join(SRT_BASE_DIR, datasource_filename), 'w+')
    datasource_file.write(json.dumps(dct))

    conn = sqlite3.connect(srtDbName)
    c = conn.cursor()

    CVE_Items = dct['CVE_Items']
    total = len(CVE_Items)
    v = Cve()

    cache_path = os.path.join(SRT_BASE_DIR, "data/cache")
    #begin parsing each cve in the JSON data
    for i, CVE_Item in enumerate(CVE_Items):
        # Development support
        if get_override('SRTDBG_MINIMAL_DB') and (i > 10):
            break

        cve = CVE_Item['cve']
        references = cve['references']['reference_data']
        CVE_data_meta  = cve['CVE_data_meta']['ID']

        #if cve exists in cache, delete it
        cve_path = os.path.join(cache_path, CVE_data_meta + ".json")
        if (os.path.isfile(cve_path)):
            os.remove(cve_path)

        #if incremental update, save json copy of the cve to cache
        if incremental:
            file = open(cve_path, 'w+')
            file.write(json.dumps(CVE_Item))

        #print('.', end='', flush=True)
        print('[%4d]%30s\r' % ((i * 100)/ total,  CVE_data_meta), end='', flush=True)
        try:
            v.name = CVE_data_meta
            v.source = 'NIST'
            v.datasource = datasource_id

            v.cve_data_type = cve['data_type']
            v.cve_data_format = cve['data_format']
            v.cve_data_version = cve['data_version']

            v.description = cve['description']['description_data'][0]['value']
            v.publishedDate = re.sub('T.*','',CVE_Item['publishedDate'])
            v.lastModifiedDate = re.sub('T.*','',CVE_Item['lastModifiedDate'])
            v.public = True #always true since NIST is public source
            v.publish = Cve.PUBLISH_PUBLISHED
            v.publish_date = v.publishedDate

            if ('impact' in CVE_Item) and ('baseMetricV3' in CVE_Item['impact']):
                baseMetricV3 = CVE_Item['impact']['baseMetricV3']
                v.cvssV3_baseScore = baseMetricV3['cvssV3']['baseScore']
                v.cvssV3_baseSeverity = baseMetricV3['cvssV3']['baseSeverity']
                # v.cvssV3_vectorString = baseMetricV3['cvssV3']['vectorString']
                # v.cvssV3_exploitabilityScore = baseMetricV3['exploitabilityScore']
                # v.cvssV3_impactScore = baseMetricV3['impactScore']
                # v.cvssV3_attackVector = baseMetricV3['cvssV3']['attackVector']
                # v.cvssV3_attackComplexity = baseMetricV3['cvssV3']['attackComplexity']
                # v.cvssV3_privilegesRequired = baseMetricV3['cvssV3']['privilegesRequired']
                # v.cvssV3_userInteraction = baseMetricV3['cvssV3']['userInteraction']
                # v.cvssV3_scope = baseMetricV3['cvssV3']['scope']
                # v.cvssV3_confidentialityImpact = baseMetricV3['cvssV3']['confidentialityImpact']
                # v.cvssV3_integrityImpact = baseMetricV3['cvssV3']['integrityImpact']
                # v.cvssV3_availabilityImpact = baseMetricV3['cvssV3']['availabilityImpact']
            if ('impact' in CVE_Item) and ('baseMetricV2' in CVE_Item['impact']):
                baseMetricV2 = CVE_Item['impact']['baseMetricV2']
                v.cvssV2_baseScore = baseMetricV2['cvssV2']['baseScore']
                # v.cvssV2_severity = baseMetricV2['severity']
                # v.cvssV2_vectorString = baseMetricV2['cvssV2']['vectorString']
                # v.cvssV2_exploitabilityScore = baseMetricV2['exploitabilityScore']
                # v.cvssV2_impactScore = baseMetricV2['exploitabilityScore']
                # v.cvssV2_accessVector = baseMetricV2['cvssV2']['accessVector']
                # v.cvssV2_accessComplexity = baseMetricV2['cvssV2']['accessComplexity']
                # v.cvssV2_authentication = baseMetricV2['cvssV2']['authentication']
                # v.cvssV2_confidentialityImpact = baseMetricV2['cvssV2']['confidentialityImpact']
                # v.cvssV2_integrityImpact = baseMetricV2['cvssV2']['integrityImpact']

            # configurations = CVE_Item['configurations']
            # is_first_and = True
            # for i, config in enumerate(configurations['nodes']):
            #    v.cpe_list += '[config]|'
            #    v.cpe_list += '[and]|'
            #    if "AND" == config['operator']:
            #        # create AND record
            #        if not is_first_and:
            #             v.cpe_list += '[/and]|'
            #             v.cpe_list += '[and]|'
            #             #is_first_and = False
            #        if 'children' in config:
            #            for j, cpe_or_node in enumerate(config['children']):
            #                 if "OR" == cpe_or_node['operator']:
            #                     v.cpe_list += nist_scan_configuration_or(v,cpe_or_node, CVE_data_meta, j)
            #                 else:
            #                     print("ERROR CONFIGURE:OR_OP?:%s" % cpe_or_node['operator'])
            #    elif "OR" == config['operator']:
            #        v.cpe_list += nist_scan_configuration_or(v,config, CVE_data_meta, 0)
            #    else:
            #        print("ERROR CONFIGURE:OP?:%s" % config_rec['operator'])
            #    v.cpe_list += '[/and]|'
            #    v.cpe_list += '[/config]|'


            #check if cve object `v` need to be uploaded to database (cases: new cve, modified cve, or no changes)
            #if true, apply changes. Else ignore and continue
            v_id, is_change = sql_cve_query(conn, v, log)


            #if CVE `v` updates, must check and update associated records (CWEs, references, and CVE2CWE)
            #sql_reference_query, sql_cwe_query, and sql_cve2cwe_query require valid CVE record primary key at some point during their execution, therefore must always be after call to sql_cve_query
            if is_change:
                r = Reference()
                for ref in references:
                    r.cve_id = v_id
                    r.hyperlink = ref['url']
                    sql_reference_query(conn, r)

                problem_list = cve['problemtype']['problemtype_data']
                for problem_Item in problem_list:
                    description_list = problem_Item['description']
                    for description_Item in description_list:
                        value = description_Item['value']
                        cwe_id = sql_cwe_query(conn, value)
                        sql_cve2cwe_query(conn, v_id, cwe_id)

        except Exception as e:
            print(traceback.format_exc())
            print("UPDATE FAILED")
            c.close()
            conn.close()
            return
    print()
    log.write("total number of CVEs checked: %s\n" % total)
    conn.commit()
    c.close()
    conn.close()

#################################
# cve class
#
class Cve():
    # WR Status
    NEW = 0
    EXCLUDE = 1
    INVESTIGATE = 2
    VULNERABLE = 3
    NOT_VULNERABLE = 4
    STATUS = (
        (NEW, 'New'),
        (EXCLUDE, 'Exclude'),
        (INVESTIGATE, 'Investigate'),
        (VULNERABLE, 'Vulnerable'),
        (NOT_VULNERABLE, 'Not Vulnerable'),
    )

    # Publish options
    PUBLISH_UNPUBLISHED = 0
    PUBLISH_NOPUBLISH = 1
    PUBLISH_PUBLISHED = 2
    PUBLISH_REQUEST = 3
    PUBLISH_UPDATE = 4
    PUBLISH_SUBMITTED = 5

    # index - primary key
    id = -1

    name = ''
    datasource = 0

    source = ''
    priority = 0
    status = NEW

    comments = ''
    comments_private = ''

    cve_data_type = ''
    cve_data_format = ''
    cve_data_version = ''

    public = False
    publish_state = PUBLISH_UNPUBLISHED
    publish_date = ''

    description = ''
    publishedDate = ''
    lastModifiedDate = ''
    problemtype = ''

    # cpe_list = ''

    cvssV3_baseScore = ''
    cvssV3_baseSeverity = ''
    # cvssV3_vectorString = ''
    # cvssV3_exploitabilityScore = ''
    # cvssV3_impactScore = ''
    # cvssV3_attackVector = ''
    # cvssV3_attackComplexity = ''
    # cvssV3_privilegesRequired = ''
    # cvssV3_userInteraction = ''
    # cvssV3_scope = ''
    # cvssV3_confidentialityImpact = ''
    # cvssV3_integrityImpact = ''
    # cvssV3_availabilityImpact = ''

    cvssV2_baseScore = ''
    cvssV2_severity = ''
    # cvssV2_vectorString = ''
    # cvssV2_exploitabilityScore = ''
    # cvssV2_impactScore = ''
    # cvssV2_accessVector = ''
    # cvssV2_accessComplexity = ''
    # cvssV2_authentication = ''
    # cvssV2_confidentialityImpact = ''
    # cvssV2_integrityImpact = ''

    recommend = 0
    recommend_list = []

    #generates importance score based on key-words in description of CVE, higher indicates more important
    def recommendation(self):
        description = ' '+self.description.lower()+' '
        total = 0
        list = ''

        for keypair in keywords_for:
            #print("keypair='%s'" % keypair)
            key,w = keypair.split(',')
            weight = 1
#            if w:
#                weight = int(w)
            #if ' '+key+' ' in description:
            if re.search(r'\b%s\b' % key, description):
                list += ",+%s" % key
                total += weight

        for keypair in keywords_against:
            #print("keypair='%s'" % keypair)
            key,w = keypair.split(',')
            weight = 1
#            if w:
#                weight = int(w)
            # re.search(r'\bis\b', your_string)
            #if ' '+key+' ' in description:
            #if re.search(re.escape(r'\b%s\b' % key), description):
            if re.search(r'\b%s\b' % key, description):
                list += ",-%s" % key
                total -= weight

        # set filter maximums
        if total < -3:
            total = -3
        if total > 3:
            total = 3
        self.recommend = total
        self.recommend_list = list[1:]

#generates and executes appropriate SQLite query for CVE depending on situation
#new CVE -> INSERT || modified CVE -> UPDATE || no change -> ignore and return
#returns (CVE_ID, BOOL) tuple, True if insert or update executed
### THIS DOES NOT CALL CONNECTION.COMMIT()
def sql_cve_query(conn, cve, log):
    CVE_LASTMODIFIEDDATE = 14
    is_change = False
    cur = conn.cursor()
    sql = '''SELECT * FROM orm_cve WHERE name=?'''
    exists = cur.execute(sql, (cve.name,)).fetchone()
    cve_id = -1
    if exists is None:
        cve.recommendation()
        sql = ''' INSERT into orm_cve (name, source, datasource_id, priority, status, comments, comments_private, cve_data_type, cve_data_format, cve_data_version, public, publish_state, publish_date, description, publishedDate, lastModifiedDate, recommend, recommend_list, cvssV3_baseScore, cvssV3_baseSeverity, cvssV2_baseScore, cvssV2_severity)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'''
        cur.execute(sql, (cve.name, cve.source, cve.datasource, cve.priority, cve.status, cve.comments, cve.comments_private, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, 1, cve.publish_state, cve.publish_date, cve.description, cve.publishedDate, cve.lastModifiedDate, cve.recommend, cve.recommend_list, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity))
        is_change = True
        cve_id = cur.lastrowid
        log.write("\tINSERTED '%s'\n" % cve.name)
    elif exists[CVE_LASTMODIFIEDDATE] < cve.lastModifiedDate:
        cve.recommendation()
        sql = ''' UPDATE orm_cve
                  SET source = ?,
                      recommend = ?,
                      recommend_list = ?,
                      cve_data_type = ?,
                      cve_data_format = ?,
                      cve_data_version = ?,
                      description = ?,
                      lastModifiedDate = ?,
                      cvssV3_baseScore = ?,
                      cvssV3_baseSeverity = ?,
                      cvssV2_baseScore = ?,
                      cvssV2_severity = ?
                  WHERE id = ?'''
        cur.execute(sql, (cve.source, cve.recommend, cve.recommend_list, cve.cve_data_type, cve.cve_data_format, cve.cve_data_version, cve.description, cve.lastModifiedDate, cve.cvssV3_baseScore, cve.cvssV3_baseSeverity, cve.cvssV2_baseScore, cve.cvssV2_severity, exists[0]))
        is_change = True
        log.write("\tUPDATED '%s'\n" % cve.name)
        cve_id = exists[0]
    else:
        is_change = False
        log.write("\tSKIPPED '%s'\n" % cve.name)
    cur.close()
    return (cve_id, is_change)


#################################
# reference
#
class Reference():
    cve_id = -1
    hyperlink = ''
    resource = ''
    type_ = ''
    source = ''
    name = ''

#generates and executes appropriate SQLite query for a new reference
### THIS DOES NOT CALL CONNECTION.COMMIT()
def sql_reference_query(conn, ref):
    cur = conn.cursor()
    sql = '''SELECT 1 FROM orm_cvereference WHERE hyperlink=?'''
    record = cur.execute(sql, (ref.hyperlink,)).fetchone()
    if record is None:
        sql = '''INSERT INTO orm_cvereference (cve_id, hyperlink) VALUES (?, ?)'''
        cur.execute(sql, (ref.cve_id, ref.hyperlink))
    cur.close()

#################################
# cwe and cve2cwe
#

#generates and executes appropriate SQLite query for a new CWE
#returns CWE_ID
### THIS DOES NOT CALL CONNECTION.COMMIT()
def sql_cwe_query(conn, value):
    cur = conn.cursor()
    sql = '''SELECT 1 FROM orm_cwetable WHERE name=?'''
    cwe = cur.execute(sql, (value,)).fetchone()
    if cwe is None:
        sql = '''INSERT INTO orm_cwetable (name, name_sort, href, summary, description, vulnerable_count, found) VALUES (?,'','','','',1,1)'''
        cur.execute(sql, (value,))
        cwe = cur.lastrowid
        cur.close()
        return cwe
    cur.close()
    return cwe[0]

#generates and executes appropriate SQLite query for new CVE to CWE relation
### THIS DOES NOT CALL CONNECTION.COMMIT()
def sql_cve2cwe_query(conn, cve_id, cwe_id):
    cur = conn.cursor()
    sql = '''SELECT * FROM orm_cvetocwe WHERE cve_id=? AND cwe_id=?'''
    cve2cwe = cur.execute(sql, (cve_id, cwe_id))
    if cve2cwe is None:
        sql = '''INSERT INTO orm_cvetocwe (cve_id, cwe_id) VALUES (?, ?)'''
        cur.execute(sql, (cve_id, cwe_id))
    cur.close()

#################################
# main loop
#

def main(argv):
    parser = argparse.ArgumentParser(description='srtool_cve.py: manage the CVEs within SRTool database')

    parser.add_argument('--update_nist', '-n', nargs=1, help='Check nvd.nist.gov/vuln/data-feeds for updates on a specified datasource')
    parser.add_argument('--update_nist_incremental', '-i', action='store_const', const='update_nist_incremental', dest='command', help='Check nvd.nist.gov/vuln/data-feeds for updates')
    args = parser.parse_args()

    master_log = open(os.path.join(SRT_BASE_DIR, "update_logs/master_log.txt"), "a")

    if args.update_nist:
        try:
            print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time")
            update_nist(args.update_nist[0], False)
            master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tUPDATED\n" % (date.today(), args.update_nist[0]))
            print("DATABASE UPDATE FINISHED\n")
        except Exception as e:
            print("DATABASE UPDATED FAILED ... %s" % e)
            master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), args.update_nist[0], e))
    elif 'update_nist_incremental' == args.command:
        try:
            print ("BEGINNING NIST UPDATES PLEASE WAIT ... this can take some time")
            update_nist('NIST JSON Modified Data', True)
            master_log.write("SRTOOL:%s:'NIST JSON Modified Data':\t\t\t...\t\t\tUPDATED\n" % date.today())
            print("DATABASE UPDATE FINISHED\n")
        except Exception as e:
            print("DATABASE UPDATED FAILED ... %s" % e)
            master_log.write("SRTOOL:%s:%s:\t\t\t...\t\t\tFAILED ... %s\n" % (date.today(), 'NIST JSON Modified Data', e))

    else:
        print("Command not found")
    master_log.close()

if __name__ == '__main__':
    global SRT_BASE_DIR
    from os.path import abspath
    # fetch any environment overrides
    set_override('SRTDBG_SKIP_CVE_IMPORT')
    set_override('SRTDBG_MINIMAL_DB')
    if get_override('SRTDBG_SKIP_CVE_IMPORT'):
        exit(0)

    SRT_BASE_DIR = os.path.abspath(os.path.dirname(os.path.dirname(sys.argv[0])))

    main(sys.argv[1:])