diff options
Diffstat (limited to 'lib/orm')
| -rw-r--r-- | lib/orm/fixtures/common.xml | 27 | ||||
| -rw-r--r-- | lib/orm/fixtures/nist.xml | 12 | ||||
| -rw-r--r-- | lib/orm/fixtures/samples.xml | 188 | ||||
| -rw-r--r-- | lib/orm/management/commands/checksettings.py | 40 | ||||
| -rw-r--r-- | lib/orm/management/commands/lsupdates.py | 109 |
5 files changed, 210 insertions, 166 deletions
diff --git a/lib/orm/fixtures/common.xml b/lib/orm/fixtures/common.xml new file mode 100644 index 00000000..5f095372 --- /dev/null +++ b/lib/orm/fixtures/common.xml @@ -0,0 +1,27 @@ +<?xml version="1.0" encoding="utf-8"?> +<django-objects version="1.0"> + <!-- Set the common data sources (starts at 1) --> + <object model="orm.datasource" pk="1"> + <field type="CharField" name="data">triage_keywords</field> + <field type="CharField" name="source">common</field> + <field type="CharField" name="type">csv</field> + <field type="TextField" name="description">Table of keyword filters</field> + <field type="FilePathField" name="file_path">data/keyword_filters.csv</field> + <field type="TextField" name="url"></field> + </object> + + <!-- TEST DATA SOURCES --> + +<!-- + <object model="orm.datasource" pk="10"> + <field type="CharField" name="data">test</field> + <field type="CharField" name="source">common</field> + <field type="CharField" name="type">csv</field> + <field type="TextField" name="description">TEST: CVE composite status charts</field> + <field type="FilePathField" name="file_path">data/test_data.csv</field> + <field type="TextField" name="url"></field> + </object> +--> + +</django-objects> + diff --git a/lib/orm/fixtures/nist.xml b/lib/orm/fixtures/nist.xml index 52417032..3b65f2e5 100644 --- a/lib/orm/fixtures/nist.xml +++ b/lib/orm/fixtures/nist.xml @@ -1,8 +1,8 @@ <?xml version="1.0" encoding="utf-8"?> <django-objects version="1.0"> - <!-- Set the NIST data sources (starts at 1) --> + <!-- Set the NIST data sources (starts at 20) --> - <object model="orm.datasource" pk="1"> + <object model="orm.datasource" pk="20"> <field type="CharField" name="data">cwe</field> <field type="CharField" name="source">nist</field> <field type="CharField" name="type">html</field> @@ -12,7 +12,7 @@ </object> <!-- NIST data feeds: https://nvd.nist.gov/vuln/data-feeds#JSON_FEED --> - <object model="orm.datasource" pk="3"> + <object model="orm.datasource" pk="23"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> <field type="CharField" name="type">json</field> @@ -21,7 +21,7 @@ <field type="TextField" name="url">https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2017.json.gz</field> </object> - <object model="orm.datasource" pk="4"> + <object model="orm.datasource" pk="24"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> <field type="CharField" name="type">json</field> @@ -31,7 +31,7 @@ </object> <!-- - <object model="orm.datasource" pk="5"> + <object model="orm.datasource" pk="25"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> <field type="CharField" name="type">json</field> @@ -41,7 +41,7 @@ </object> --> - <object model="orm.datasource" pk="6"> + <object model="orm.datasource" pk="26"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> <field type="CharField" name="type">json</field> diff --git a/lib/orm/fixtures/samples.xml b/lib/orm/fixtures/samples.xml index 674c3783..09c218ff 100644 --- a/lib/orm/fixtures/samples.xml +++ b/lib/orm/fixtures/samples.xml @@ -4,44 +4,18 @@ <!-- Set up test data for Products --> <object model="orm.product" pk="1"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">LTS-17</field> + <field type="CharField" name="name">Yocto Project</field> + <field type="CharField" name="version">2.5 (Sumo)</field> <field type="CharField" name="profile"></field> </object> <object model="orm.product" pk="2"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">9</field> + <field type="CharField" name="name">Yocto Project</field> + <field type="CharField" name="version">2.4 (Rocko)</field> <field type="CharField" name="profile"></field> </object> - <object model="orm.product" pk="3"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">8</field> - <field type="CharField" name="profile"></field> - </object> - <object model="orm.product" pk="4"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">OVP</field> - <field type="CharField" name="profile"></field> - </object> - <object model="orm.product" pk="5"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">7</field> - <field type="CharField" name="profile"></field> - </object> - <object model="orm.product" pk="6"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">7</field> - <field type="CharField" name="profile">SCP</field> - </object> - <object model="orm.product" pk="7"> - <field type="CharField" name="name">Wind River Linux</field> - <field type="CharField" name="version">7</field> - <field type="CharField" name="profile">CGP</field> - </object> - <object model="orm.product" pk="8"> - <field type="CharField" name="name">VxWorks</field> - <field type="CharField" name="version">5.5</field> + <field type="CharField" name="name">Yocto Project</field> + <field type="CharField" name="version">2.3 (Pyro)</field> <field type="CharField" name="profile"></field> </object> @@ -59,7 +33,7 @@ </object> <object model="orm.cve" pk="2"> - <field type="CharField" name="name">WINDCVE-2018-0000</field> + <field type="CharField" name="name">SRTCVE-2018-0000</field> <field type="BooleanField" name="public">False</field> <field type="TextField" name="description">(TEST) Something is rotten in the state of Denmark, and we are going to figure it out and report it</field> <field type="CharField" name="publishedDate">2018-01-11</field> @@ -113,36 +87,6 @@ <field type="ForeignKey" name="product">3</field> <field type="IntegerField" name="relation">0</field> </object> - <object model="orm.vulnerabilityproduct" pk="4"> - <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="product">4</field> - <field type="IntegerField" name="relation">0</field> - </object> - <object model="orm.vulnerabilityproduct" pk="5"> - <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="product">5</field> - <field type="IntegerField" name="relation">0</field> - </object> - <object model="orm.vulnerabilityproduct" pk="6"> - <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="product">6</field> - <field type="IntegerField" name="relation">0</field> - </object> - <object model="orm.vulnerabilityproduct" pk="7"> - <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="product">7</field> - <field type="IntegerField" name="relation">0</field> - </object> - <object model="orm.vulnerabilityproduct" pk="8"> - <field type="ForeignKey" name="vulnerability">1</field> - <!-- <field type="ForeignKey" name="investigation"></field> --> - <field type="ForeignKey" name="product">8</field> - <field type="IntegerField" name="relation">1</field> - </object> <object model="orm.vulnerabilitycomments" pk="1"> <field type="ForeignKey" name="vulnerability">1</field> @@ -207,28 +151,28 @@ <object model="orm.vulnerabilityuploads" pk="1"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="TextField" name="description">PowerDNS Security Advisories 2018-02</field> - <field type="TextField" name="path">powerdns-2018-02.txt</field> + <field type="TextField" name="description">Internal Security Advisories 2018-02</field> + <field type="TextField" name="path">internal-2018-02.txt</field> <field type="DateField" name="date">2017-12-14</field> <field type="TextField" name="author">Mark Hatle</field> </object> - <!-- Set up test Jira's--> + <!-- Set up test defects --> <object model="orm.jira" pk="1"> - <field type="CharField" name="name">LIN10-9991</field> + <field type="CharField" name="name">YP25-9991</field> <field type="CharField" name="summary">(TEST) This is a defect</field> <field type="IntegerField" name="priority">3</field> <field type="IntegerField" name="status">5</field> <field type="IntegerField" name="resolution">1</field> <field type="IntegerField" name="publishOLS">Reviewed - Publish</field> - <field type="CharField" name="rcpl">10.0.0.2</field> + <field type="CharField" name="rcpl">2.5.1</field> <field type="ForeignKey" name="product">1</field> </object> <object model="orm.jira" pk="2"> - <field type="CharField" name="name">LIN10-9992</field> + <field type="CharField" name="name">YP24-9992</field> <field type="CharField" name="summary">(TEST) This is another defect</field> <field type="IntegerField" name="priority">2</field> <field type="IntegerField" name="status">0</field> @@ -281,62 +225,30 @@ </object> <object model="orm.user" pk="3"> - <field type="TextField" name="name">Neeraja Vemulapalli</field> + <field type="TextField" name="name">Ross Burton</field> <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Vulnerability Manager VxWorks</field> + <field type="TextField" name="role">Security Manager Yocto Project</field> <field type="IntegerField" name="access">1</field> <field type="TextField" name="password"></field> </object> <object model="orm.user" pk="4"> - <field type="TextField" name="name">Huabing Chu</field> + <field type="TextField" name="name">Richard Purtie</field> <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Vulnerability Manager VxWorks (backup)</field> + <field type="TextField" name="role">Security Manager Yocto Project (backup)</field> <field type="IntegerField" name="access">1</field> <field type="TextField" name="password"></field> </object> <object model="orm.user" pk="5"> - <field type="TextField" name="name">Yue Tao</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Vulnerability Manager WRLinux</field> - <field type="IntegerField" name="access">2</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="6"> - <field type="TextField" name="name">Jyothi Bhattaram</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Vulnerability Manager WRLinux (backup)</field> - <field type="IntegerField" name="access">2</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="7"> - <field type="TextField" name="name">Roger Boden</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Security Expert VxWorks</field> - <field type="IntegerField" name="access">2</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="8"> - <field type="TextField" name="name">Markus Carlstedt</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Security Expert VxWorks (backup)</field> - <field type="IntegerField" name="access">2</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="9"> <field type="TextField" name="name">Mark Hatle</field> <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Security Expert WRLinux</field> + <field type="TextField" name="role">Product Security Expert Wind River</field> <field type="IntegerField" name="access">2</field> <field type="TextField" name="password"></field> </object> - <object model="orm.user" pk="10"> + <object model="orm.user" pk="6"> <field type="TextField" name="name">Jason Wessel</field> <field type="TextField" name="email"></field> <field type="TextField" name="role">Product Security Expert WRLinux (backup)</field> @@ -344,58 +256,26 @@ <field type="TextField" name="password"></field> </object> - <object model="orm.user" pk="11"> - <field type="TextField" name="name">Michel Chabroux</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Owner VxWorks</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="12"> - <field type="TextField" name="name">Graham Morphew</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Owner VxWorks (backup)</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="13"> - <field type="TextField" name="name">Kamal Desai</field> + <object model="orm.user" pk="7"> + <field type="TextField" name="name">Jefro Osier-Mixon</field> <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Owner WRLinux</field> + <field type="TextField" name="role">Product Owner Yocto Project</field> <field type="IntegerField" name="access">1</field> <field type="TextField" name="password"></field> </object> - <object model="orm.user" pk="14"> - <field type="TextField" name="name">Teo Bobirnila</field> + <object model="orm.user" pk="8"> + <field type="TextField" name="name">Stephen Jolley</field> <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Owner WRLinux (backup)</field> + <field type="TextField" name="role">Product Owner Yocto Project (backup)</field> <field type="IntegerField" name="access">1</field> <field type="TextField" name="password"></field> </object> - <object model="orm.user" pk="15"> - <field type="TextField" name="name">Tim Skutt</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Owner VxWorks,WRLinux</field> - <field type="IntegerField" name="access">3</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="16"> - <field type="TextField" name="name">Alex deVries</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Owner VxWorks,WRLinux (backup)</field> - <field type="IntegerField" name="access">3</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="17"> + <object model="orm.user" pk="9"> <field type="TextField" name="name">David Reyna</field> <field type="TextField" name="email">david.reyna@windriver.com</field> - <field type="TextField" name="role">Developer WRLinux</field> + <field type="TextField" name="role">Developer Wind River</field> <field type="IntegerField" name="access">1</field> <field type="TextField" name="password"></field> </object> @@ -404,34 +284,34 @@ <object model="orm.investigationaccess" pk="1"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">9</field> + <field type="ForeignKey" name="user">1</field> </object> <object model="orm.investigationaccess" pk="2"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">10</field> + <field type="ForeignKey" name="user">2</field> </object> <object model="orm.investigationaccess" pk="3"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">13</field> + <field type="ForeignKey" name="user">3</field> </object> <object model="orm.vulnerabilityaccess" pk="4"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="user">9</field> + <field type="ForeignKey" name="user">2</field> </object> <object model="orm.vulnerabilityaccess" pk="5"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="user">17</field> + <field type="ForeignKey" name="user">4</field> </object> <object model="orm.investigationnotification" pk="1"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">9</field> + <field type="ForeignKey" name="user">5</field> </object> <object model="orm.vulnerabilitynotification" pk="2"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="user">13</field> + <field type="ForeignKey" name="user">6</field> </object> </django-objects> diff --git a/lib/orm/management/commands/checksettings.py b/lib/orm/management/commands/checksettings.py index c9876a95..65e9ab8a 100644 --- a/lib/orm/management/commands/checksettings.py +++ b/lib/orm/management/commands/checksettings.py @@ -46,13 +46,27 @@ class Command(BaseCommand): print("Loading default settings") call_command("loaddata", "settings") + # Import the Common fixture if it's present + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing Common settings if present") + try: + call_command("loaddata", "common") + except: + print("NOTE: optional fixture 'common' not found") + # Import the Mitre fixture if it's present with warnings.catch_warnings(): warnings.filterwarnings( action="ignore", message="^.*No fixture named.*$") print("Importing Mitre settings if present") - call_command("loaddata", "mitre") + try: + call_command("loaddata", "mitre") + except: + print("NOTE: optional fixture 'mitre' not found") # Import the NIST fixture if it's present with warnings.catch_warnings(): @@ -60,7 +74,21 @@ class Command(BaseCommand): action="ignore", message="^.*No fixture named.*$") print("Importing NIST settings if present") - call_command("loaddata", "nist") + try: + call_command("loaddata", "nist") + except: + print("NOTE: optional fixture 'nist' not found") + + # Import the Sample_Test fixture if it's present + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing Sample Test settings if present") + try: + call_command("loaddata", "samples") + except: + print("NOTE: optional fixture 'samples' not found") # restore data source loaded flags for source in DataSource.objects.all(): @@ -98,10 +126,10 @@ class Command(BaseCommand): SrtSetting.objects.filter(name='DEFAULT_RELEASE').delete() SrtSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='') - # TEST: set up the test flags - # * TEST_SKIP_NIST_IMPORT: we already have NIST data in the DB - # * TEST_SKIP_CPE_IMPORT: we do not need to (re-)scan the CPEs for vulnerable CVEs - # * TEST_MINIMAL_DB: only load the minimal database items for fast GUI tests + # TEST: set up the test flags based on ENVIRONMENT values + # * export TEST_SKIP_NIST_IMPORT=1: we already have NIST data in the DB + # * export TEST_SKIP_CPE_IMPORT=1: we do not need to (re-)scan the CPEs for vulnerable CVEs + # * export TEST_MINIMAL_DB=1: only load the minimal database items for fast GUI tests self._test_settings('TEST_SKIP_NIST_IMPORT') self._test_settings('TEST_SKIP_CPE_IMPORT') self._test_settings('TEST_MINIMAL_DB') diff --git a/lib/orm/management/commands/lsupdates.py b/lib/orm/management/commands/lsupdates.py index ccb98074..7bbcb066 100644 --- a/lib/orm/management/commands/lsupdates.py +++ b/lib/orm/management/commands/lsupdates.py @@ -473,6 +473,105 @@ class Command(BaseCommand): # print("[%d]cpe23Uri=%s,%s" % (i,company,product)) + def cve_keywords_old(self, csvfile_name): + # mode,type,keyword,weight + # y,key,abiword, + + KEY_MODE=0 + KEY_TYPE=1 + KEY_KEY=2 + KEY_WEIGHT=3 + + i_index=0 + is_header = True + with open(csvfile_name, newline='') as csvfile: + CPE_reader = csv.reader(csvfile, delimiter=',', quotechar='"') + for row in CPE_reader: + if is_header or not len(row): + is_header = False + continue + + if (KEY_WEIGHT+1) != len(row): + print("KEY_ROWLEN_ERROR:'%s'" % row) + continue + + i_index += 1 + if 0 == i_index % 100: + print('%04d: %20s\r' % (i_index,row[KEY_KEY]), end='') + +# # DEBUG ### TODO +# if 0 < Command.debug_jira_limit: +# if i_index > Command.debug_jira_limit: +# return + + k, created = Keywords.objects.get_or_create(keyword=row[KEY_KEY]) + if 'y' == row[KEY_MODE]: + k.key_mode = Keywords.FOR + else: + k.key_mode = Keywords.AGAINST + if 'keyword' == row[KEY_KEY]: + k.key_type = Keywords.KEYWORD + else: + k.key_type = Keywords.CPE + if row[KEY_WEIGHT]: + k.weight = int(row[KEY_WEIGHT]) + else: + if Keywords.FOR == k.key_mode: + k.weight = 1 + else: + k.weight = -1 + k.save() + + def cve_keywords(self, csvfile_name): + # mode,type,keyword,weight + # y,key,abiword, + + KEY_MODE=0 + KEY_TYPE=1 + KEY_KEY=2 + KEY_WEIGHT=3 + + keywords_for = '' + keywords_against = '' + + i_index=0 + is_header = True + with open(csvfile_name, newline='') as csvfile: + CPE_reader = csv.reader(csvfile, delimiter=',', quotechar='"') + for row in CPE_reader: + if is_header or not len(row): + is_header = False + continue + + if (KEY_WEIGHT+1) != len(row): + print("KEY_ROWLEN_ERROR:'%s'" % row) + continue + + i_index += 1 + if 0 == i_index % 100: + print('%04d: %20s\r' % (i_index,row[KEY_KEY]), end='') + + key = row[KEY_MODE] + if '#' == key[0]: + key = key[1:] + + if 'y' == key: + keywords_for += "|%s,%s" % (row[KEY_KEY],row[KEY_WEIGHT]) + elif 'n' == key: + keywords_against += "|%s,%s" % (row[KEY_KEY],row[KEY_WEIGHT]) + + setting = SrtSetting.objects.get_or_create(name='keywords_for')[0] + setting.value = keywords_for[1:] + setting.save() + setting = SrtSetting.objects.get_or_create(name='keywords_against')[0] + setting.value = keywords_against[1:] + setting.save() + + S = SrtSetting.objects.get(name='keywords_for') + #print("FOO_FOR:[%s]='%s'" % (S.name,S.value[0:30])) + S = SrtSetting.objects.get(name='keywords_against') + #print("FOO_NOT:[%s]='%s'" % (S.name,S.value[0:30])) + def debug_set_cve(self,key,public,vulnerability,wr_comments,wr_comments_private): try: @@ -557,9 +656,18 @@ class Command(BaseCommand): logger.error("Unknown data source path for '%s' (%s,%s) " % (source.source.description,source.file_path,source.url)) continue + # testing shortcut if ('nist' == source.source) and ('yes' == SrtSetting.objects.get(name='TEST_SKIP_NIST_IMPORT').value): continue + + # Common data sources + if 'common' == source.source: + if 'triage_keywords' == source.data: + self.cve_keywords(csvfile_name) + source.loaded = True + source.save() + continue # Common Vulnerabilities and Exposures if 'cve' == source.data: @@ -587,6 +695,7 @@ class Command(BaseCommand): source.loaded = True source.save() continue + # data source not handled logger.error("Unknown data source type for '%s' (%s,%s,%s) " % (source.file_path,source.data,source.source,source.type)) |