diff options
Diffstat (limited to 'lib/orm')
-rw-r--r-- | lib/orm/fixtures/common.xml | 100 | ||||
-rw-r--r-- | lib/orm/fixtures/nist.xml | 31 | ||||
-rw-r--r-- | lib/orm/fixtures/samples.xml | 117 | ||||
-rw-r--r-- | lib/orm/fixtures/yp.xml | 65 | ||||
-rw-r--r-- | lib/orm/management/commands/checksettings.py | 110 | ||||
-rw-r--r-- | lib/orm/management/commands/lsupdates.py | 327 | ||||
-rw-r--r-- | lib/orm/migrations/0001_initial.py | 68 | ||||
-rw-r--r-- | lib/orm/models.py | 363 |
8 files changed, 588 insertions, 593 deletions
diff --git a/lib/orm/fixtures/common.xml b/lib/orm/fixtures/common.xml index 5f095372..97bbd8e9 100644 --- a/lib/orm/fixtures/common.xml +++ b/lib/orm/fixtures/common.xml @@ -1,27 +1,107 @@ <?xml version="1.0" encoding="utf-8"?> <django-objects version="1.0"> - <!-- Set the common data sources (starts at 1) --> + +<!-- Set the common data settings (starts at 1) --> + + <object model="orm.srtsetting" pk="1"> + <field type="CharField" name="name">SRTOOL_FIXTURE_LIST_FALLBACK</field> + <field type="CharField" name="value">yp,nist</field> + </object> + + <object model="orm.srtsetting" pk="10"> + <field type="CharField" name="name">SRTOOL_DEFECT_UPDATE_FALLBACK</field> + <field type="CharField" name="value">bin/srtool_defect.py --update</field> + </object> + <object model="orm.srtsetting" pk="11"> + <field type="CharField" name="name">SRTOOL_DEFECT_ADD_FALLBACK</field> + <field type="CharField" name="value">bin/srtool_defect.py --add</field> + </object> + <object model="orm.srtsetting" pk="12"> + <field type="CharField" name="name">SRTOOL_DEFECT_DEL_FALLBACK</field> + <field type="CharField" name="value">bin/srtool_defect.py --del</field> + </object> + <object model="orm.srtsetting" pk="13"> + <field type="CharField" name="name">SRTOOL_DEFECT_NEW_FALLBACK</field> + <field type="CharField" name="value">bin/srtool_defect.py --new</field> + </object> + <object model="orm.srtsetting" pk="14"> + <field type="CharField" name="name">SRTOOL_DEFECT_SAMPLENAME_FALLBACK</field> + <field type="CharField" name="value">54321</field> + </object> + +<!-- Set the common data sources (starts at 1) --> + +<!-- Full production size keyword list <object model="orm.datasource" pk="1"> <field type="CharField" name="data">triage_keywords</field> <field type="CharField" name="source">common</field> <field type="CharField" name="type">csv</field> <field type="TextField" name="description">Table of keyword filters</field> - <field type="FilePathField" name="file_path">data/keyword_filters.csv</field> + <field type="FilePathField" name="file_path">data/keyword_filters_full.csv</field> <field type="TextField" name="url"></field> + <field type="CharField" name="lastModifiedDate">2018-03-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">3</field> + <field type="CharField" name="update_time">02:00:00</field> </object> - - <!-- TEST DATA SOURCES --> +--> -<!-- - <object model="orm.datasource" pk="10"> - <field type="CharField" name="data">test</field> +<!-- Debug size keyword list --> + <object model="orm.datasource" pk="1"> + <field type="CharField" name="data">triage_keywords</field> <field type="CharField" name="source">common</field> <field type="CharField" name="type">csv</field> - <field type="TextField" name="description">TEST: CVE composite status charts</field> - <field type="FilePathField" name="file_path">data/test_data.csv</field> + <field type="TextField" name="description">Table of keyword filters</field> + <field type="FilePathField" name="file_path">data/keyword_filters.csv</field> <field type="TextField" name="url"></field> + <field type="CharField" name="lastModifiedDate">2018-03-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">3</field> + <field type="CharField" name="update_time">02:00:00</field> + </object> +<!-- --> + + <object model="orm.datasource" pk="2"> + <field type="CharField" name="data">backup_weekly</field> + <field type="CharField" name="source">common</field> + <field type="CharField" name="type">script</field> + <field type="TextField" name="description">Weekly archive database backup</field> + <field type="FilePathField" name="file_path"></field> + <field type="TextField" name="url"></field> + <field type="CharField" name="lastModifiedDate">0001-01-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">3</field> + <field type="CharField" name="update_time">02:00:00</field> + <field type="TextField" name="command">bin/srtool_utils.py --backup-db-json</field> + </object> + + <object model="orm.datasource" pk="3"> + <field type="CharField" name="data">backup_daily</field> + <field type="CharField" name="source">common</field> + <field type="CharField" name="type">script</field> + <field type="TextField" name="description">Daily database backup (wheel)</field> + <field type="FilePathField" name="file_path"></field> + <field type="TextField" name="url"></field> + <field type="CharField" name="lastModifiedDate">0001-01-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">2</field> + <field type="CharField" name="update_time">02:00:00</field> + <field type="TextField" name="command">bin/srtool_utils.py --backup-db-json-daily</field> + </object> + +<!-- Built-in Users : first user is default log-on user --> + + <object model="orm.user" pk="1"> + <field type="TextField" name="name">Guest</field> + <field type="TextField" name="email"></field> + <field type="TextField" name="role">Guest</field> + <field type="IntegerField" name="access">0</field> + <field type="TextField" name="password"></field> + </object> + + <object model="orm.user" pk="2"> + <field type="TextField" name="name">SRTool</field> + <field type="TextField" name="email"></field> + <field type="TextField" name="role">SRTool automation scripts</field> + <field type="IntegerField" name="access">3</field> + <field type="TextField" name="password"></field> </object> ---> </django-objects> diff --git a/lib/orm/fixtures/nist.xml b/lib/orm/fixtures/nist.xml index 3b65f2e5..b392e57b 100644 --- a/lib/orm/fixtures/nist.xml +++ b/lib/orm/fixtures/nist.xml @@ -9,27 +9,43 @@ <field type="TextField" name="description">NIST Common Weakness Enumeration Data</field> <field type="FilePathField" name="file_path">data/nist-cwe-summary.html</field> <field type="TextField" name="url">https://nvd.nist.gov/vuln/categories</field> + <field type="CharField" name="lastModifiedDate">0001-01-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">3</field> + <field type="CharField" name="update_time">02:00:00</field> + <field type="TextField" name="command"></field> </object> - <!-- NIST data feeds: https://nvd.nist.gov/vuln/data-feeds#JSON_FEED --> + <object model="orm.datasource" pk="23"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> - <field type="CharField" name="type">json</field> + <field type="CharField" name="type">script</field> <field type="TextField" name="description">NIST JSON Data 2017</field> <field type="FilePathField" name="file_path">data/nvdcve-1.0-2017.json</field> <field type="TextField" name="url">https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2017.json.gz</field> + <field type="TextField" name="meta_url">https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2017.meta</field> + <field type="CharField" name="lastModifiedDate">0001-01-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">3</field> + <field type="CharField" name="update_time">02:00:00</field> + <field type="TextField" name="command">bin/srtool_cve.py -n "NIST JSON Data 2017"</field> </object> + <object model="orm.datasource" pk="24"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> - <field type="CharField" name="type">json</field> + <field type="CharField" name="type">script</field> <field type="TextField" name="description">NIST JSON Data 2018</field> <field type="FilePathField" name="file_path">data/nvdcve-1.0-2018.json</field> <field type="TextField" name="url">https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2018.json.gz</field> + <field type="TextField" name="meta_url">https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2018.meta</field> + <field type="CharField" name="lastModifiedDate">0001-01-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">3</field> + <field type="CharField" name="update_time">02:00:00</field> + <field type="TextField" name="command">bin/srtool_cve.py -n "NIST JSON Data 2018"</field> </object> + <!-- <object model="orm.datasource" pk="25"> <field type="CharField" name="data">cve</field> @@ -44,10 +60,15 @@ <object model="orm.datasource" pk="26"> <field type="CharField" name="data">cve</field> <field type="CharField" name="source">nist</field> - <field type="CharField" name="type">json</field> - <field type="TextField" name="description">NIST JSON Modified Data 2017</field> + <field type="CharField" name="type">script</field> + <field type="TextField" name="description">NIST JSON Modified Data</field> <field type="FilePathField" name="file_path">data/nvdcve-1.0-modified.json</field> <field type="TextField" name="url">https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.gz</field> + <field type="TextField" name="meta_url">https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.meta</field> + <field type="CharField" name="lastModifiedDate">0001-01-01 01:01:01</field> + <field type="IntegerField" name="update_frequency">2</field> + <field type="CharField" name="update_time">02:00:00</field> + <field type="TextField" name="command">bin/srtool_cve.py -n "NIST JSON Modified Data"</field> </object> </django-objects> diff --git a/lib/orm/fixtures/samples.xml b/lib/orm/fixtures/samples.xml index 0d22770a..04b865b0 100644 --- a/lib/orm/fixtures/samples.xml +++ b/lib/orm/fixtures/samples.xml @@ -1,25 +1,6 @@ <?xml version="1.0" encoding="utf-8"?> <django-objects version="1.0"> - <!-- Set up test data for Products --> - - <object model="orm.product" pk="1"> - <field type="CharField" name="name">Yocto Project</field> - <field type="CharField" name="version">2.5 (Sumo)</field> - <field type="CharField" name="profile"></field> - </object> - <object model="orm.product" pk="2"> - <field type="CharField" name="name">Yocto Project</field> - <field type="CharField" name="version">2.4 (Rocko)</field> - <field type="CharField" name="profile"></field> - </object> - <object model="orm.product" pk="3"> - <field type="CharField" name="name">Yocto Project</field> - <field type="CharField" name="version">2.3 (Pyro)</field> - <field type="CharField" name="profile"></field> - </object> - - <object model="orm.cve" pk="1"> <field type="CharField" name="name">CVE-2017-0000</field> <field type="BooleanField" name="public">False</field> @@ -94,12 +75,6 @@ <field type="DateField" name="date">2017-12-14</field> <field type="TextField" name="author">Mark Hatle</field> </object> - <object model="orm.vulnerabilitycomments" pk="2"> - <field type="ForeignKey" name="vulnerability">1</field> - <field type="TextField" name="comment">ACTION: User 'Paul Gortmaker' 'Bruce Ashfield' added</field> - <field type="DateField" name="date">2017-12-14</field> - <field type="TextField" name="author">Mark Hatle</field> - </object> <object model="orm.vulnerabilitycomments" pk="3"> <field type="ForeignKey" name="vulnerability">1</field> <field type="TextField" name="comment">ACTION: Attachment 'PowerDNS Security Advisories 2018-02'</field> @@ -166,7 +141,7 @@ <field type="IntegerField" name="priority">3</field> <field type="IntegerField" name="status">5</field> <field type="IntegerField" name="resolution">1</field> - <field type="IntegerField" name="publishOLS">Reviewed - Publish</field> + <field type="IntegerField" name="publish">Reviewed - Publish</field> <field type="CharField" name="release_version">2.5.1</field> <field type="ForeignKey" name="product">1</field> </object> @@ -176,7 +151,7 @@ <field type="CharField" name="summary">(TEST) This is another defect</field> <field type="IntegerField" name="priority">2</field> <field type="IntegerField" name="status">0</field> - <field type="IntegerField" name="publishOLS">Not Reviewed</field> + <field type="IntegerField" name="publish">Not Reviewed</field> <field type="CharField" name="release_version"></field> <field type="ForeignKey" name="product">1</field> </object> @@ -206,112 +181,38 @@ <field type="ForeignKey" name="defect">2</field> </object> - <!-- Set up default users --> - - <object model="orm.user" pk="1"> - <field type="TextField" name="name">Guest</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Guest</field> - <field type="IntegerField" name="access">0</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="2"> - <field type="TextField" name="name">All</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">internal all access placeholder</field> - <field type="IntegerField" name="access">0</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="3"> - <field type="TextField" name="name">Ross Burton</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Manager Yocto Project</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="4"> - <field type="TextField" name="name">Richard Purtie</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Security Manager Yocto Project (backup)</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="5"> - <field type="TextField" name="name">Mark Hatle</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Security Expert Wind River</field> - <field type="IntegerField" name="access">2</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="6"> - <field type="TextField" name="name">Jason Wessel</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Security Expert WRLinux (backup)</field> - <field type="IntegerField" name="access">2</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="7"> - <field type="TextField" name="name">Jefro Osier-Mixon</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Owner Yocto Project</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="8"> - <field type="TextField" name="name">Stephen Jolley</field> - <field type="TextField" name="email"></field> - <field type="TextField" name="role">Product Owner Yocto Project (backup)</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - - <object model="orm.user" pk="9"> - <field type="TextField" name="name">David Reyna</field> - <field type="TextField" name="email">david.reyna@windriver.com</field> - <field type="TextField" name="role">Developer Wind River</field> - <field type="IntegerField" name="access">1</field> - <field type="TextField" name="password"></field> - </object> - <!-- Set up default users to investigations --> <object model="orm.investigationaccess" pk="1"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">1</field> + <field type="ForeignKey" name="user">10</field> </object> <object model="orm.investigationaccess" pk="2"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">2</field> + <field type="ForeignKey" name="user">11</field> </object> <object model="orm.investigationaccess" pk="3"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">3</field> + <field type="ForeignKey" name="user">12</field> </object> <object model="orm.vulnerabilityaccess" pk="4"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="user">2</field> + <field type="ForeignKey" name="user">10</field> </object> <object model="orm.vulnerabilityaccess" pk="5"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="user">4</field> + <field type="ForeignKey" name="user">11</field> </object> <object model="orm.investigationnotification" pk="1"> <field type="ForeignKey" name="investigation">1</field> - <field type="ForeignKey" name="user">5</field> + <field type="ForeignKey" name="user">12</field> </object> <object model="orm.vulnerabilitynotification" pk="2"> <field type="ForeignKey" name="vulnerability">1</field> - <field type="ForeignKey" name="user">6</field> + <field type="ForeignKey" name="user">10</field> </object> </django-objects> diff --git a/lib/orm/fixtures/yp.xml b/lib/orm/fixtures/yp.xml new file mode 100644 index 00000000..38e0f875 --- /dev/null +++ b/lib/orm/fixtures/yp.xml @@ -0,0 +1,65 @@ +<?xml version="1.0" encoding="utf-8"?> +<django-objects version="1.0"> + + <!-- Set up test data for Products --> + + <object model="orm.product" pk="1"> + <field type="CharField" name="name">Yocto Project</field> + <field type="CharField" name="version">2.5 (Sumo)</field> + <field type="CharField" name="profile"></field> + </object> + <object model="orm.product" pk="2"> + <field type="CharField" name="name">Yocto Project</field> + <field type="CharField" name="version">2.4 (Rocko)</field> + <field type="CharField" name="profile"></field> + </object> + <object model="orm.product" pk="3"> + <field type="CharField" name="name">Yocto Project</field> + <field type="CharField" name="version">2.3 (Pyro)</field> + <field type="CharField" name="profile"></field> + </object> + + <!-- Set up default users --> + + <object model="orm.user" pk="10"> + <field type="TextField" name="name">Ross Burton</field> + <field type="TextField" name="email"></field> + <field type="TextField" name="role">Security Manager Yocto Project</field> + <field type="IntegerField" name="access">3</field> + <field type="TextField" name="password"></field> + </object> + + <object model="orm.user" pk="11"> + <field type="TextField" name="name">Richard Purtie</field> + <field type="TextField" name="email"></field> + <field type="TextField" name="role">Security Manager Yocto Project (backup)</field> + <field type="IntegerField" name="access">3</field> + <field type="TextField" name="password"></field> + </object> + + <object model="orm.user" pk="12"> + <field type="TextField" name="name">Mark Hatle</field> + <field type="TextField" name="email"></field> + <field type="TextField" name="role">Product Security Expert Wind River</field> + <field type="IntegerField" name="access">2</field> + <field type="TextField" name="password"></field> + </object> + + <object model="orm.user" pk="13"> + <field type="TextField" name="name">Stephen Jolley</field> + <field type="TextField" name="email"></field> + <field type="TextField" name="role">Product Owner Yocto Project</field> + <field type="IntegerField" name="access">1</field> + <field type="TextField" name="password"></field> + </object> + + <object model="orm.user" pk="14"> + <field type="TextField" name="name">David Reyna</field> + <field type="TextField" name="email">david.reyna@windriver.com</field> + <field type="TextField" name="role">Developer Wind River</field> + <field type="IntegerField" name="access">1</field> + <field type="TextField" name="password"></field> + </object> + +</django-objects> + diff --git a/lib/orm/management/commands/checksettings.py b/lib/orm/management/commands/checksettings.py index 65e9ab8a..e65d16ac 100644 --- a/lib/orm/management/commands/checksettings.py +++ b/lib/orm/management/commands/checksettings.py @@ -29,7 +29,7 @@ class Command(BaseCommand): def _verify_srt_source(self): ds_loaded = {} - + needs_import = False if 0 == DataSource.objects.all().count(): needs_import = True @@ -46,7 +46,7 @@ class Command(BaseCommand): print("Loading default settings") call_command("loaddata", "settings") - # Import the Common fixture if it's present + # Import the common fixture with warnings.catch_warnings(): warnings.filterwarnings( action="ignore", @@ -54,41 +54,57 @@ class Command(BaseCommand): print("Importing Common settings if present") try: call_command("loaddata", "common") - except: - print("NOTE: optional fixture 'common' not found") - - # Import the Mitre fixture if it's present - with warnings.catch_warnings(): - warnings.filterwarnings( - action="ignore", - message="^.*No fixture named.*$") - print("Importing Mitre settings if present") - try: - call_command("loaddata", "mitre") - except: - print("NOTE: optional fixture 'mitre' not found") + except Exception as e: + print("NOTE: optional fixture 'common' not found (%s)" % e) - # Import the NIST fixture if it's present + # Import the 'custom' fixture to allow local custom overrides with warnings.catch_warnings(): warnings.filterwarnings( action="ignore", message="^.*No fixture named.*$") - print("Importing NIST settings if present") - try: - call_command("loaddata", "nist") - except: - print("NOTE: optional fixture 'nist' not found") - - # Import the Sample_Test fixture if it's present - with warnings.catch_warnings(): - warnings.filterwarnings( - action="ignore", - message="^.*No fixture named.*$") - print("Importing Sample Test settings if present") + print("Importing Common settings if present") try: - call_command("loaddata", "samples") - except: - print("NOTE: optional fixture 'samples' not found") + call_command("loaddata", "custom") + except Exception as e: + print("NOTE: optional fixture 'custom' not found (%s)" % e) + + # Promote fallback values to missing configure defines + for setting in SrtSetting.objects.all(): + if '_fallback' in setting.name: + name = setting.name.replace('_fallback','') + s,create = SrtSetting.objects.get_or_create(name=key) + if create: + s.value = setting.value + s.save + + # Import the requested source fixture list + try: + fixture_list = SrtSetting.objects.get(name='SRTOOL_FIXTURE_LIST').value + except: + fixture_list = 'yp,nist' + for fixture in fixture_list.split(','): + fixture = fixture.strip() + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing %s fixture if present" % fixture) + try: + call_command("loaddata", fixture) + except Exception as e: + print("NOTE: optional fixture '%s' not found (%s)" % (fixture,e)) + + # Import the Sample_Test fixture if it's requested and present + if self._test_settings_get('SRTDBG_SAMPLES'): + with warnings.catch_warnings(): + warnings.filterwarnings( + action="ignore", + message="^.*No fixture named.*$") + print("Importing Sample Test settings if present") + try: + call_command("loaddata", "samples") + except Exception as e: + print("NOTE: optional fixture 'samples' not found (%s)" % e) # restore data source loaded flags for source in DataSource.objects.all(): @@ -109,8 +125,20 @@ class Command(BaseCommand): traceback.print_exc() return 0 - - def _test_settings(self,key): + + def _test_settings_get(self,key): + try: + key_record = SrtSetting.objects.get(name=key) + if 'yes' == key_record.value: + return True + elif 'no' == key_record.value: + return False + else: + return key_record.value + except: + return False + + def _test_settings_set(self,key): key_record,create = SrtSetting.objects.get_or_create(name=key) if key in os.environ.keys(): key_record.value = 'yes' if os.environ[key].startswith('1') else 'no' @@ -127,12 +155,16 @@ class Command(BaseCommand): SrtSetting.objects.get_or_create(name='DEFAULT_RELEASE', value='') # TEST: set up the test flags based on ENVIRONMENT values - # * export TEST_SKIP_NIST_IMPORT=1: we already have NIST data in the DB - # * export TEST_SKIP_CPE_IMPORT=1: we do not need to (re-)scan the CPEs for vulnerable CVEs - # * export TEST_MINIMAL_DB=1: only load the minimal database items for fast GUI tests - self._test_settings('TEST_SKIP_NIST_IMPORT') - self._test_settings('TEST_SKIP_CPE_IMPORT') - self._test_settings('TEST_MINIMAL_DB') + # * export SRTDBG_SKIP_CVE_IMPORT=1: we do not need to (re-)scan the CVE data + # * export SRTDBG_SKIP_CPE_IMPORT=1: we do not need to (re-)scan the CPE data + # * export SRTDBG_SKIP_DEFECT_IMPORT=1: we do not need to (re-)scan the CPE data + # * export SRTDBG_MINIMAL_DB=1: only load the minimal database items for fast GUI tests + # * export SRTDBG_SAMPLES=1: load sample fixture (if any) database items for fast GUI tests + self._test_settings_set('SRTDBG_SKIP_CVE_IMPORT') + self._test_settings_set('SRTDBG_SKIP_CPE_IMPORT') + self._test_settings_set('SRTDBG_SKIP_DEFECT_IMPORT') + self._test_settings_set('SRTDBG_MINIMAL_DB') + self._test_settings_set('SRTDBG_SAMPLES') # TEMP: set up default user info current_user = SrtSetting.objects.get_or_create(name='current_user')[0] diff --git a/lib/orm/management/commands/lsupdates.py b/lib/orm/management/commands/lsupdates.py index 32548ddb..d72d732a 100644 --- a/lib/orm/management/commands/lsupdates.py +++ b/lib/orm/management/commands/lsupdates.py @@ -33,22 +33,21 @@ from orm.models import Keywords import os import sys import re +import subprocess +import time +from datetime import datetime, date import json import xml.etree.ElementTree as ET import csv import logging import threading -import time -logger = logging.getLogger("srt") import urllib +logger = logging.getLogger("srt") -def _log(msg): - f1=open('/tmp/srt.log', 'a') - f1.write("|" + msg + "|\n" ) - f1.close() - +# quick development/debugging support +from srtgui.api import _log # === Debugging limited database loading support === debug_cve_list = [] # empty list for any @@ -112,6 +111,27 @@ class Command(BaseCommand): sys.stdout.flush() + # Execute a shell script to import data, relative the SRT base + def execute_script(self,command): + SRT_BASE_DIR = os.environ.get('SRT_BASE_DIR') + CWD = os.getcwd() + os.chdir(SRT_BASE_DIR) + script = os.path.join(SRT_BASE_DIR,command) + print("====vvv Executing script '%s' vvv====" % script) + os.system(script) + os.chdir(CWD) + print("====^^^ Script Done ^^^====") + + # Mark database as loaded + def source_loaded(self,id,update_modified=True): + # Re-fetch record in case external script updates + updated_source=DataSource.objects.get(id=id) + updated_source.loaded = True + if update_modified: + updated_source.update_time = datetime.today().strftime('%Y-%m-%d %H:%M:%S') + updated_source.lastModifiedDate = updated_source.update_time + updated_source.save() + def nist_scan_configuration_or(self, cve, cpe_or_node, name, and_enum): cpe_list = '<or>|' for cpe in cpe_or_node['cpe']: @@ -128,207 +148,6 @@ class Command(BaseCommand): cpe_list += '</or>|' return cpe_list - def nist_jason(self, dct): - CVE_Items = dct['CVE_Items'] - total = len(CVE_Items) - for i, CVE_Item in enumerate(CVE_Items): - cve = CVE_Item['cve'] - references = cve['references']['reference_data'] - CVE_data_meta = cve['CVE_data_meta']['ID'] - - # DEBUGGING SUPPORT !!! TODO - scan = True - status = Cve.NOT_VULNERABLE - if (Command.debug_cve_count or len(debug_cve_list)) and not CVE_data_meta.startswith(Command.debug_include_id_prefix): - scan = False - if Command.debug_cve_count: - if i < Command.debug_cve_count: - scan = True - if len(debug_cve_list): - for debug_cve in debug_cve_list: - if cve['CVE_data_meta']['ID'].startswith(debug_cve): - scan = True - status = Cve.INVESTIGATE - if not scan: - continue - - if False: - print(" publishedDate: %s" % CVE_Item['publishedDate']) - print(" lastModifiedDate: %s" % CVE_Item['lastModifiedDate']) - - print(" publishedDate: %s" % re.sub('T.*','',CVE_Item['publishedDate'])) - print(" lastModifiedDate: %s" % re.sub('T.*','',CVE_Item['lastModifiedDate'])) - - print(" data_type: %s" % cve['data_type']) - print(" data_format: %s" % cve['data_format']) - print(" CVE_data_meta: %s" % cve['CVE_data_meta']['ID']) - print(" problemtype: %s" % cve['problemtype']['problemtype_data'][0]['description'][0]['value']) - print(" description: '%s'" % cve['description']['description_data'][0]['value']) - references = cve['references']['reference_data'] - print(" References = %d" % len(references)) - for ref in references: - print(" reference: %s" % ref['url']) - if CVE_Item['impact'] and CVE_Item['impact']['baseMetricV3']: - baseMetricV3 = CVE_Item['impact']['baseMetricV3'] - print(" cvssV3 : %s,%s" % (baseMetricV3['exploitabilityScore'],baseMetricV3['impactScore'])) - print(" vectorString = %s" % baseMetricV3['cvssV3']['vectorString']) - print(" attackVector = %s" % baseMetricV3['cvssV3']['attackVector']) - print(" attackComplexity = %s" % baseMetricV3['cvssV3']['attackComplexity']) - print(" privilegesRequired = %s" % baseMetricV3['cvssV3']['privilegesRequired']) - print(" userInteraction = %s" % baseMetricV3['cvssV3']['userInteraction']) - print(" scope = %s" % baseMetricV3['cvssV3']['scope']) - print(" confidentialityImpact = %s" % baseMetricV3['cvssV3']['confidentialityImpact']) - print(" integrityImpact = %s" % baseMetricV3['cvssV3']['integrityImpact']) - print(" availabilityImpact = %s" % baseMetricV3['cvssV3']['availabilityImpact']) - print(" baseScore = %s" % baseMetricV3['cvssV3']['baseScore']) - print(" baseSeverity = %s" % baseMetricV3['cvssV3']['baseSeverity']) - if CVE_Item['impact'] and CVE_Item['impact']['baseMetricV2']: - baseMetricV2 = CVE_Item['impact']['baseMetricV2'] - print(" cvssV2 : %s,%s" % (baseMetricV2['exploitabilityScore'],baseMetricV2['exploitabilityScore'])) - print(" vectorString = %s" % baseMetricV2['cvssV2']['vectorString']) - print(" accessVector = %s" % baseMetricV2['cvssV2']['accessVector']) - print(" accessComplexity = %s" % baseMetricV2['cvssV2']['accessComplexity']) - print(" authentication = %s" % baseMetricV2['cvssV2']['authentication']) - print(" confidentialityImpact = %s" % baseMetricV2['cvssV2']['confidentialityImpact']) - print(" integrityImpact = %s" % baseMetricV2['cvssV2']['integrityImpact']) - print(" availabilityImpact = %s" % baseMetricV2['cvssV2']['availabilityImpact']) - print(" baseScore = %s" % baseMetricV2['cvssV2']['baseScore']) - print(" severity = %s" % baseMetricV2['severity']) - - try: - CVE_data_meta = cve['CVE_data_meta']['ID'] - v, created = Cve.objects.get_or_create(name=CVE_data_meta) - - v.name = CVE_data_meta - v.source = 'NIST' - status = Cve.NOT_VULNERABLE - - # Debugging support -# if v.name.startswith("CVE-2018"): -# status = Cve.NEW -# v.status = status - v.tags = '' - v.tags_private = '' - - v.cve_data_type = cve['data_type'] - v.cve_data_format = cve['data_format'] - v.cve_data_version = cve['data_version'] - - v.description = cve['description']['description_data'][0]['value'] - v.publishedDate = re.sub('T.*','',CVE_Item['publishedDate']) - v.lastModifiedDate = re.sub('T.*','',CVE_Item['lastModifiedDate']) - - v.public = True - v.publish = Cve.PUBLISH_PUBLISHED - v.publish_date = v.publishedDate - - #v.problemtype = cve['problemtype']['problemtype_data'][0]['description'][0]['value'] - problem_list = cve['problemtype']['problemtype_data'] - CveToCwe.objects.filter(cve=v).delete() - for problem_Item in problem_list: - description_list = problem_Item['description'] - for description_Item in description_list: - value = description_Item['value'] - cwe, created = CweTable.objects.get_or_create(name=value) - if created: - print("WARNING Missing CWE = '%s'"% value) - cwe.save() - cve2cwe, created = CveToCwe.objects.get_or_create(cve=v,cwe=cwe) - if created: - cve2cwe.save() - -# if CVE_Item['impact'] and CVE_Item['impact']['baseMetricV3']: - if ('impact' in CVE_Item) and ('baseMetricV3' in CVE_Item['impact']): - baseMetricV3 = CVE_Item['impact']['baseMetricV3'] - v.cvssV3_baseScore = baseMetricV3['cvssV3']['baseScore'] - v.cvssV3_baseSeverity = baseMetricV3['cvssV3']['baseSeverity'] - v.cvssV3_vectorString = baseMetricV3['cvssV3']['vectorString'] - v.cvssV3_exploitabilityScore = baseMetricV3['exploitabilityScore'] - v.cvssV3_impactScore = baseMetricV3['impactScore'] - v.cvssV3_attackVector = baseMetricV3['cvssV3']['attackVector'] - v.cvssV3_attackComplexity = baseMetricV3['cvssV3']['attackComplexity'] - v.cvssV3_privilegesRequired = baseMetricV3['cvssV3']['privilegesRequired'] - v.cvssV3_userInteraction = baseMetricV3['cvssV3']['userInteraction'] - v.cvssV3_scope = baseMetricV3['cvssV3']['scope'] - v.cvssV3_confidentialityImpact = baseMetricV3['cvssV3']['confidentialityImpact'] - v.cvssV3_integrityImpact = baseMetricV3['cvssV3']['integrityImpact'] - v.cvssV3_availabilityImpact = baseMetricV3['cvssV3']['availabilityImpact'] - if ('impact' in CVE_Item) and ('baseMetricV2' in CVE_Item['impact']): - baseMetricV2 = CVE_Item['impact']['baseMetricV2'] - v.cvssV2_baseScore = baseMetricV2['cvssV2']['baseScore'] - v.cvssV2_severity = baseMetricV2['severity'] - v.cvssV2_vectorString = baseMetricV2['cvssV2']['vectorString'] - v.cvssV2_exploitabilityScore = baseMetricV2['exploitabilityScore'] - v.cvssV2_impactScore = baseMetricV2['exploitabilityScore'] - v.cvssV2_accessVector = baseMetricV2['cvssV2']['accessVector'] - v.cvssV2_accessComplexity = baseMetricV2['cvssV2']['accessComplexity'] - v.cvssV2_authentication = baseMetricV2['cvssV2']['authentication'] - v.cvssV2_confidentialityImpact = baseMetricV2['cvssV2']['confidentialityImpact'] - v.cvssV2_integrityImpact = baseMetricV2['cvssV2']['integrityImpact'] - -## v.save() - - CveReference.objects.filter(cve=v).delete() - for ref in references: - r, created = CveReference.objects.get_or_create(cve=v, - hyperlink=ref['url']) - r.resource = '' - r.type = '' - r.source = '' - r.name = '' - r.save() -# print(" reference: %s,%s" % (ref['url'],created)) - - - configurations = CVE_Item['configurations'] - v.cpe_list = '' - is_first_and = True - for i, config in enumerate(configurations['nodes']): - v.cpe_list += '<config>|' - v.cpe_list += '<and>|' - if "AND" == config['operator']: - # create AND record - if not is_first_and: - v.cpe_list += '</and>|' - v.cpe_list += '<and>|' - for j, cpe_or_node in enumerate(config['children']): - if "OR" == cpe_or_node['operator']: - v.cpe_list += self.nist_scan_configuration_or(v,cpe_or_node, CVE_data_meta, j) - else: - print("ERROR CONFIGURE:OR_OP?:%s" % cpe_or_node['operator']) - elif "OR" == config['operator']: - v.cpe_list += self.nist_scan_configuration_or(v,config, CVE_data_meta, 0) - else: - print("ERROR CONFIGURE:OP?:%s" % config_rec['operator']) - v.cpe_list += '</and>|' - v.cpe_list += '</config>|' - -# # create a parent CveSet if needed -# cve_set, created = CveSet.objects.get_or_create(name=v.name) -# if created: -# cve_set.name = v.name -# cve_set.description = v.description -# cve_set.cvssV3_baseScore = v.cvssV3_baseScore -# cve_set.cvssV3_baseSeverity = v.cvssV3_baseSeverity -# cve_set.publishedDate = v.publishedDate -# cve_set.lastModifiedDate = v.lastModifiedDate -# cve_set.save() -# -# # connect the Cve with the CveSet -# cve2cveset, created = CveToCveSet.objects.get_or_create(cve=v, cveset=cve_set) -# cve2cveset.save() - - # save the final CVE content - v.recommend = v.recommendation() - v.save() - - except Exception as e: - logger.warning("Failed saving CVE %s (%s)", (cve['CVE_data_meta']['ID'],e)) - return - - self.mini_progress("CVE's", i, total) - - def nist_cwe(self, content): # <td nowrap><span id="cweIdEntry-CWE-123">CWE-123</span></td> # <td nowrap><a href="http://cwe.mitre.org/data/definitions/123.html" target="_blank"> Write-what-where Condition</a></td> @@ -473,7 +292,8 @@ class Command(BaseCommand): # print("[%d]cpe23Uri=%s,%s" % (i,company,product)) - def cve_keywords_old(self, csvfile_name): + # Obsolete (slow) table-based keyword management + def cve_keywords_table(self, csvfile_name): # mode,type,keyword,weight # y,key,abiword, @@ -566,26 +386,6 @@ class Command(BaseCommand): setting = SrtSetting.objects.get_or_create(name='keywords_against')[0] setting.value = keywords_against[1:] setting.save() - - S = SrtSetting.objects.get(name='keywords_for') - #print("FOO_FOR:[%s]='%s'" % (S.name,S.value[0:30])) - S = SrtSetting.objects.get(name='keywords_against') - #print("FOO_NOT:[%s]='%s'" % (S.name,S.value[0:30])) - - - def debug_set_cve(self,key,public,vulnerability,comments,comments_private): - try: - c = Cve.objects.get(name=key) - c.public = public - c.comments = comments - c.comments_private = comments_private - c.save() - if vulnerability: - v = Vulnerability.objects.get(name=vulnerability) - cv, created = CveToVulnerablility.objects.get_or_create(vulnerability=v,cve=c) - cv.save() - except ObjectDoesNotExist: - print("Cve %s not found" % key) def update(self): """ @@ -607,11 +407,11 @@ class Command(BaseCommand): if source.loaded: logger.info("Skipping source data from %s",source.file_path) - print("Skipping CVE data for %s (already loaded)" % (source.description)) + print("Skipping CVE data from %s (already loaded)" % (source.description)) continue else: - logger.info("Fetching source data from %s",source.file_path) - print("Fetching source data for '%s'" % (source.description)) + logger.info("Fetching source data from %s:%s" % (source.source,source.file_path)) + print("Fetching source data from '%s:%s:%s'" % (source.source,source.description,source.file_path)) file_path = source.file_path url_path = source.url @@ -619,14 +419,17 @@ class Command(BaseCommand): root = None content = None source_doc = None - # prefer the file path before the url - if file_path: + # precedence: (1) script, (2) local file path, (3) url + if 'script' == source.type: + if not source.command: + # no script to run + logger.error("Data source is script but no script provided '%s'" % (source.description)) + continue + elif file_path: # load the file source_doc = source.file_path if not source.file_path.startswith('/'): file_path = os.path.join(os.getenv('SRT_BASE_DIR'), source.file_path) - source.loaded = True - source.save() if not os.path.isfile(file_path): logger.error("Data file not found '%s'" % (file_path)) continue @@ -643,7 +446,7 @@ class Command(BaseCommand): content = text_data.read() elif url_path: # load the HTML page - source_doc = source.url + href = source.url try: f = urlopen(href) content = f.read().decode('UTF-8') @@ -653,66 +456,64 @@ class Command(BaseCommand): continue else: # no data source path to load - logger.error("Unknown data source path for '%s' (%s,%s) " % (source.source.description,source.file_path,source.url)) + logger.error("Unknown data source path for '%s' (%s,%s,%s) " % (source.description,source.type,source.file_path,source.url)) continue # testing shortcut - if ('nist' == source.source) and ('yes' == SrtSetting.objects.get(name='TEST_SKIP_NIST_IMPORT').value): + if ('cve' == source.data) and ('yes' == SrtSetting.objects.get(name='SRTDBG_SKIP_CVE_IMPORT').value): + continue + if ('cpe' == source.data) and ('yes' == SrtSetting.objects.get(name='SRTDBG_SKIP_CPE_IMPORT').value): + continue + if ('defect' == source.data) and ('yes' == SrtSetting.objects.get(name='SRTDBG_SKIP_DEFECT_IMPORT').value): continue - + + # Script-based update? + if 'script' == source.type and source.command: + # do not perform backups for brand new databases + if source.data in ['backup_weekly','backup_daily']: + pass + else: + self.execute_script(source.command) + self.source_loaded(source.id,False) + continue + # Common data sources if 'common' == source.source: if 'triage_keywords' == source.data: self.cve_keywords(csvfile_name) - source.loaded = True - source.save() + self.source_loaded(source.id) continue # Common Vulnerabilities and Exposures - if 'cve' == source.data: - if 'nist' == source.source and 'json' == source.type: - self.nist_jason(dct) - source.loaded = True - source.save() - continue + # Handled by "srtool_cve.py" + # Common Weakness Enumeration if 'cwe' == source.data: if 'nist' == source.source and 'html' == source.type: self.nist_cwe(content) - source.loaded = True - source.save() + self.source_loaded(source.id) continue # Common Product Enumeration if 'cpe' == source.data: if 'nist' == source.source and 'xml' == source.type: self.nist_cpe(root) - source.loaded = True - source.save() + self.source_loaded(source.id) continue if 'nist' == source.source and 'csv' == source.type: self.nist_cpe_csv(csvfile_name) - source.loaded = True - source.save() + self.source_loaded(source.id) continue - # data source not handled logger.error("Unknown data source type for '%s' (%s,%s,%s) " % (source.file_path,source.data,source.source,source.type)) - # TEST DEBUG -# self.debug_set_cve('CVE-2017-0002',False,'','','') -# self.debug_set_cve('CVE-2017-0010',False,'','','') - self.debug_set_cve('CVE-2017-5753',True,'V0000','Spectre Variant 1: Bounds check bypass','community calling this "spectre"') - self.debug_set_cve('CVE-2017-5715',True,'V0000','Spectre Variant 2: Branch target injection','see if this will required compiler changes') - self.debug_set_cve('CVE-2017-5754',True,'V0000','Meltdown: Rogue data cache load, memory access permission check performed after kernel memory read','mostly for Intel parts') - os.system('setterm -cursor on') def handle(self, **options): # testing shortcuts - if 'yes' == SrtSetting.objects.get(name='TEST_MINIMAL_DB').value: + if 'yes' == SrtSetting.objects.get(name='SRTDBG_MINIMAL_DB').value: print("TEST: MINIMAL DATABASE LOADING") Command.debug_cve_count = 10 # 0 for any Command.debug_include_id_prefix = 'XXX' # always include CVEs with this prefix diff --git a/lib/orm/migrations/0001_initial.py b/lib/orm/migrations/0001_initial.py index 6d8b9472..764b27df 100644 --- a/lib/orm/migrations/0001_initial.py +++ b/lib/orm/migrations/0001_initial.py @@ -42,6 +42,11 @@ class Migration(migrations.Migration): ('file_path', models.FilePathField()), ('url', models.TextField(blank=True)), ('loaded', models.BooleanField(default=False)), + ('meta_url', models.TextField(blank=True)), + ('lastModifiedDate', models.CharField(max_length=50, blank=True)), + ('update_frequency', models.IntegerField(default=2)), + ('update_time', models.CharField(max_length=50, blank=True)), + ('command', models.TextField(blank=True)), ], ), @@ -58,14 +63,17 @@ class Migration(migrations.Migration): ('found', models.BooleanField(default=False)), ], ), + migrations.CreateModel( name='Cve', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('name', models.CharField(max_length=50)), - ('source', models.CharField(max_length=50)), + ('datasource', models.ForeignKey(default=None, to='orm.datasource',null=True)), + ('source', models.CharField(max_length=50)), + ('priority', models.IntegerField(default=0)), ('status', models.IntegerField(default=0)), ('comments', models.TextField(blank=True)), ('comments_private', models.TextField(blank=True)), @@ -75,44 +83,21 @@ class Migration(migrations.Migration): ('cve_data_version', models.CharField(max_length=50, blank=True)), ('public', models.BooleanField(default=False)), - ('publish', models.IntegerField(default=0)), + ('publish_state', models.IntegerField(default=0)), ('publish_date', models.CharField(max_length=50, blank=True)), ('description', models.TextField(blank=True)), ('publishedDate', models.CharField(max_length=50, blank=True)), ('lastModifiedDate', models.CharField(max_length=50, blank=True)), -# ('problemtype', models.CharField(max_length=40, blank=True)), -# ('problemtype', models.ForeignKey(CweTable, related_name='cwe')), - ('recommend', models.IntegerField(default=0)), - - ('cpe_list', models.TextField(blank=True)), + ('recommend_list', models.TextField(blank=True)), ('cvssV3_baseScore', models.CharField(max_length=50, blank=True)), ('cvssV3_baseSeverity', models.CharField(max_length=50, blank=True)), - ('cvssV3_vectorString', models.TextField(blank=True)), - ('cvssV3_exploitabilityScore', models.CharField(max_length=50, blank=True)), - ('cvssV3_impactScore', models.CharField(max_length=50, blank=True)), - ('cvssV3_attackVector', models.CharField(max_length=5, blank=True)), - ('cvssV3_attackComplexity', models.CharField(max_length=50, blank=True)), - ('cvssV3_privilegesRequired', models.CharField(max_length=50, blank=True)), - ('cvssV3_userInteraction', models.CharField(max_length=50, blank=True)), - ('cvssV3_scope', models.CharField(max_length=50, blank=True)), - ('cvssV3_confidentialityImpact', models.CharField(max_length=50, blank=True)), - ('cvssV3_integrityImpact', models.CharField(max_length=50, blank=True)), - ('cvssV3_availabilityImpact', models.CharField(max_length=50, blank=True)), ('cvssV2_baseScore',models.CharField(max_length=50, blank=True)), ('cvssV2_severity', models.CharField(max_length=50, blank=True)), - ('cvssV2_vectorString', models.TextField(blank=True)), - ('cvssV2_exploitabilityScore', models.CharField(max_length=50, blank=True)), - ('cvssV2_impactScore', models.CharField(max_length=50, blank=True)), - ('cvssV2_accessVector', models.CharField(max_length=50, blank=True)), - ('cvssV2_accessComplexity', models.CharField(max_length=50, blank=True)), - ('cvssV2_authentication', models.CharField(max_length=50, blank=True)), - ('cvssV2_confidentialityImpact', models.CharField(max_length=50, blank=True)), - ('cvssV2_integrityImpact', models.CharField(max_length=50, blank=True)), ], ), @@ -190,11 +175,11 @@ class Migration(migrations.Migration): fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('name', models.CharField(max_length=50)), - ('description', models.TextField(blank=True)), - ('cve_primary_name', models.CharField(max_length=50)), + ('description', models.TextField(blank=True, default='')), + ('cve_primary_name', models.CharField(max_length=50, default='')), ('public', models.BooleanField(default=False)), - ('comments', models.TextField(blank=True)), - ('comments_private', models.TextField(blank=True)), + ('comments', models.TextField(blank=True, default='')), + ('comments_private', models.TextField(blank=True, default='')), ('status', models.IntegerField(default=0)), ('outcome', models.IntegerField(default=0)), ('severity', models.IntegerField(default=0)), @@ -251,10 +236,11 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('name', models.CharField(max_length=50)), ('summary', models.TextField(blank=True)), + ('url', models.TextField(blank=True)), ('priority', models.IntegerField(default=0)), ('status', models.IntegerField(default=0)), ('resolution', models.IntegerField(default=0)), - ('publishOLS', models.TextField(blank=True)), + ('publish', models.TextField(blank=True)), ('release_version', models.CharField(max_length=50)), ('product', models.ForeignKey(default=None, to='orm.product', null=True)), ('date_created', models.CharField(max_length=50)), @@ -421,13 +407,15 @@ class Migration(migrations.Migration): ], ), -# migrations.AddField( -# model_name='project', -# name='release', -# field=models.ForeignKey(to='orm.Release', null=True), -# ), -# migrations.AlterUniqueTogether( -# name='layersource', -# unique_together=set([('sourcetype', 'apiurl')]), -# ), + migrations.CreateModel( + name='PublishPending', + fields=[ + ('cve', models.ForeignKey(default=None, to='orm.cve',blank=True,null=True)), + ('vulnerability', models.ForeignKey(default=None, to='orm.vulnerability',blank=True,null=True)), + ('investigation', models.ForeignKey(default=None, to='orm.investigation',blank=True,null=True)), + ('date', models.DateField(null=True, blank=True)), + ('note', models.TextField()), + ], + ), + ] diff --git a/lib/orm/models.py b/lib/orm/models.py index 3a4a39a7..0d287bb4 100644 --- a/lib/orm/models.py +++ b/lib/orm/models.py @@ -42,10 +42,8 @@ from signal import SIGUSR1 import logging logger = logging.getLogger("srt") -def _log(msg): - f1=open('/tmp/srt.log', 'a') - f1.write("|" + msg + "|\n" ) - f1.close() +# quick development/debugging support +from srtgui.api import _log # Sqlite support @@ -127,7 +125,6 @@ class SrtSetting(models.Model): def __unicode__(self): return "Setting %s = %s" % (self.name, self.value) - class HelpText(models.Model): VARIABLE = 0 HELPTEXT_AREA = ((VARIABLE, 'variable'), ) @@ -136,7 +133,25 @@ class HelpText(models.Model): key = models.CharField(max_length=100) text = models.TextField() + +#UPDATE_FREQUENCY: 0 = every minute, 1 = every hour, 2 = every day, 3 = every week, 4 = every month, 5 = every year class DataSource(models.Model): + #UPDATE FREQUENCT + MINUTELY = 0 + HOURLY = 1 + DAILY = 2 + WEEKLY = 3 + MONTHLY = 4 + ONDEMAND = 5 + FREQUENCY = ( + (MINUTELY, 'New'), + (HOURLY, 'Hourly'), + (DAILY, 'Daily'), + (WEEKLY, 'Weekly'), + (MONTHLY, 'Monthly'), + (ONDEMAND, 'OnDemand'), + ) + data = models.CharField(max_length=20) source = models.CharField(max_length=20) type = models.CharField(max_length=20) @@ -144,6 +159,15 @@ class DataSource(models.Model): file_path = models.FilePathField() url = models.TextField(blank=True) loaded = models.BooleanField(default=False) + meta_url = models.TextField(blank=True) + lastModifiedDate = models.CharField(max_length=50, blank=True) + update_frequency = models.IntegerField(choices=FREQUENCY, default=DAILY) + update_time = models.CharField(max_length=50, blank=True) + command = models.TextField(blank=True) + + + def get_frequency_text(self): + return DataSource.FREQUENCY[int(self.update_frequency)][1] class CweTable(models.Model): search_allowed_fields = ['name', 'href', 'description', 'summary'] @@ -155,9 +179,25 @@ class CweTable(models.Model): vulnerable_count = models.IntegerField(default=0) found = models.BooleanField(default=False) + class Cve(models.Model): search_allowed_fields = ['name', 'description', 'publishedDate', 'lastModifiedDate', 'comments', 'comments_private'] + + # SRTool Priority + UNDEFINED = 0 + MINOR = 1 + LOW = 2 + MEDIUM = 3 + HIGH = 4 + PRIORITY = ( + (UNDEFINED, ''), + (MINOR, 'Minor'), + (LOW, 'Low'), + (MEDIUM, 'Medium'), + (HIGH, 'High'), + ) + # WR Status NEW = 0 INVESTIGATE = 1 @@ -170,18 +210,20 @@ class Cve(models.Model): (NOT_VULNERABLE, 'Not Vulnerable'), ) - # Publish options - PUBLISH_UNDEF = 0 - PUBLISH_AUTO = 1 - PUBLISH_REQUEST = 2 - PUBLISH_PUBLISHED = 3 - PUBLISH_NOPUBLISH = 4 - PUBLISH = ( - (PUBLISH_UNDEF, 'Undetermined'), - (PUBLISH_AUTO, 'Automatic Publish Date'), - (PUBLISH_REQUEST, 'Request Publish Date'), + # Publish state + PUBLISH_UNPUBLISHED = 0 + PUBLISH_NOPUBLISH = 1 + PUBLISH_PUBLISHED = 2 + PUBLISH_REQUEST = 3 + PUBLISH_UPDATE = 4 + PUBLISH_SUBMITTED = 5 + PUBLISH_STATE = ( + (PUBLISH_UNPUBLISHED, 'Unpublished'), + (PUBLISH_NOPUBLISH, 'Not to be Published'), (PUBLISH_PUBLISHED, 'Published'), - (PUBLISH_NOPUBLISH, 'Do Not Published'), + (PUBLISH_REQUEST, 'Publish Request (New)'), + (PUBLISH_UPDATE, 'Publish Request (Update)'), + (PUBLISH_SUBMITTED, 'Publish Submitted'), ) # CPE item list @@ -193,69 +235,44 @@ class Cve(models.Model): name = models.CharField(max_length=50) source = models.CharField(max_length=50) + datasource = models.ForeignKey(DataSource,related_name="cve_datasource") + priority = models.IntegerField(default=0) status = models.IntegerField(choices=STATUS, default=NEW) comments = models.TextField(blank=True) comments_private = models.TextField(blank=True) - + cve_data_type = models.CharField(max_length=100, blank=True) cve_data_format = models.CharField(max_length=50, blank=True) cve_data_version = models.CharField(max_length=50, blank=True) public = models.BooleanField(default=True) - publish = models.IntegerField(choices=PUBLISH, default=PUBLISH_UNDEF) + publish_state = models.IntegerField(choices=PUBLISH_STATE, default=PUBLISH_UNPUBLISHED) publish_date = models.CharField(max_length=50, blank=True) description = models.TextField(blank=True) publishedDate = models.CharField(max_length=50, blank=True) lastModifiedDate = models.CharField(max_length=50, blank=True) -# problemtype = models.CharField(max_length=40, blank=True) -# problemtype = models.ForeignKey(CweTable, related_name='cwe') recommend = models.IntegerField(default=0) - - cpe_list= models.TextField(blank=True) + recommend_list = models.TextField(blank=True) cvssV3_baseScore = models.CharField(max_length=50, blank=True) cvssV3_baseSeverity = models.CharField(max_length=50, blank=True) - cvssV3_vectorString = models.TextField(blank=True) - cvssV3_exploitabilityScore = models.CharField(max_length=50, blank=True) - cvssV3_impactScore = models.CharField(max_length=50, blank=True) - cvssV3_attackVector = models.CharField(max_length=50, blank=True) - cvssV3_attackComplexity = models.CharField(max_length=50, blank=True) - cvssV3_privilegesRequired = models.CharField(max_length=50, blank=True) - cvssV3_userInteraction = models.CharField(max_length=50, blank=True) - cvssV3_scope = models.CharField(max_length=50, blank=True) - cvssV3_confidentialityImpact = models.CharField(max_length=50, blank=True) - cvssV3_integrityImpact = models.CharField(max_length=50, blank=True) - cvssV3_availabilityImpact = models.CharField(max_length=50, blank=True) cvssV2_baseScore = models.CharField(max_length=50, blank=True) cvssV2_severity = models.CharField(max_length=50, blank=True) - cvssV2_vectorString = models.TextField(blank=True) - cvssV2_exploitabilityScore = models.CharField(max_length=50, blank=True) - cvssV2_impactScore = models.CharField(max_length=50, blank=True) - cvssV2_accessVector = models.CharField(max_length=50, blank=True) - cvssV2_accessComplexity = models.CharField(max_length=50, blank=True) - cvssV2_authentication = models.CharField(max_length=50, blank=True) - cvssV2_confidentialityImpact = models.CharField(max_length=50, blank=True) - cvssV2_integrityImpact = models.CharField(max_length=50, blank=True) class Meta: unique_together = ('name', 'source' ) @property - def problemtype_summary(self): - """ Return the summary of the CWE """ - summary = '?' - try: - r = CweTable.objects.get(name=self.problemtype) - summary = r.summary - except Exception as e: - logger.warning("ERROR: could not find CWE %s" % self.problemtype) - return summary + def get_priority_text(self): + return Cve.PRIORITY[int(self.priority)][1] + @property def get_publish_text(self): - return Cve.PUBLISH[int(self.publish)][1] + return Cve.PUBLISH_STATE[int(self.publish_state)][1] + @property def get_status_text(self): return Cve.STATUS[int(self.status)][1] def get_cpe_list(self): @@ -264,31 +281,54 @@ class Cve(models.Model): cpe_array.append(cpe.split(',')) return cpe_array - FOR_LIST = ['linux','openjpeg','libtiff','libav','tcp','binutil ','ssl','ssh','glibc'] - AGAINST_LIST = ['cisco','microsoft','windows','ibm','oracle','sun','java','peoplesoft','hancom','zyxel','wordpress','sugarcrm','cobham', - 'juniper'] - - def recommendation(self): - recommendation = 0 - for s in Cve.FOR_LIST: - if 0 <= self.description.lower().find(s): - recommendation += 1 - for s in Cve.AGAINST_LIST: - if 0 <= self.description.lower().find(s): - recommendation -= 1 - return recommendation - def reasons_for(self): - reason = '' - for s in Cve.FOR_LIST: - if 0 <= self.description.lower().find(s): - reason += '%s ' % s - return reason - def reasons_against(self): - reason = '' - for s in Cve.AGAINST_LIST: - if 0 <= self.description.lower().find(s): - reason += '%s ' % s - return reason + +class CveDetail(): + # CPE item list + CPE_LIST_KEY = 0 # entry is <[/]component|and|or> tag or '|' delimited list + CPE_LIST_VULNERABLE = 0 + CPE_LIST_CPE23 = 1 + CPE_LIST_CPE22 = 2 + CPE_LIST_VERSIONEND = 3 + + name = '' +# problemtype = '' + + recommend = '' + recommend_list = '' + + cpe_list= '' + + cvssV3_baseScore = '' + cvssV3_baseSeverity = '' + cvssV3_vectorString = '' + cvssV3_exploitabilityScore = '' + cvssV3_impactScore = '' + cvssV3_attackVector = '' + cvssV3_attackComplexity = '' + cvssV3_privilegesRequired = '' + cvssV3_userInteraction = '' + cvssV3_scope = '' + cvssV3_confidentialityImpact = '' + cvssV3_integrityImpact = '' + cvssV3_availabilityImpact = '' + + cvssV2_baseScore = '' + cvssV2_severity = '' + cvssV2_vectorString = '' + cvssV2_exploitabilityScore = '' + cvssV2_impactScore = '' + cvssV2_accessVector = '' + cvssV2_accessComplexity = '' + cvssV2_authentication = '' + cvssV2_confidentialityImpact = '' + cvssV2_integrityImpact = '' + + def get_cpe_list(self): + cpe_array = [] + for cpe in self.cpe_list.split('|'): + cpe_array.append(cpe.split(',')) + return cpe_array + # Same CVE from multiple sources and/or revisions class CveSet(models.Model): @@ -319,7 +359,7 @@ class CpeTable(models.Model): cpeMatchString = models.TextField(blank=True) cpe23Uri = models.TextField(blank=True) versionEndIncluding = models.TextField(blank=True) - + class CpeToCve(models.Model): cpe = models.ForeignKey(CpeTable,related_name="cpe2cve") cve = models.ForeignKey(Cve,related_name="cve2cpe") @@ -353,7 +393,7 @@ class CpeFilter(models.Model): @property def get_status_text(self): return CpeFilter.STATUS[int(self.status)][1] - + # CVE/CWE Mapping @@ -373,6 +413,8 @@ class CveReference(models.Model): # PRODUCT class Product(models.Model): + search_allowed_fields = ['name', 'version', 'profile'] + name = models.CharField(max_length=40) version = models.CharField(max_length=40) profile = models.CharField(max_length=40) @@ -382,13 +424,14 @@ class Product(models.Model): class Meta: unique_together = ('name', 'version', 'profile', ) + @property def long_name(self): return '%s %s %s' % (self.name,self.version,self.profile) # VULNERABILITY -# Company-level Vulnerablility Record +# Company-level Vulnerablility Record class Vulnerability(models.Model): search_allowed_fields = ['name', 'comments', 'comments_private'] @@ -410,22 +453,27 @@ class Vulnerability(models.Model): (FIXED, 'Closed (Fixed)'), (NOT_FIX, "Closed (Won't Fix)"), ) - LOW = 0 - MEDIUM = 1 - HIGH = 2 + # SRTool Severity, matched with Cve/Defect Priority with placeholder for 'minor' + UNDEFINED = 0 + MINOR = 1 + LOW = 2 + MEDIUM = 3 + HIGH = 4 SEVERITY = ( + (UNDEFINED, ''), + (MINOR, 'Minor'), (LOW, 'Low'), (MEDIUM, 'Medium'), (HIGH, 'High'), ) name = models.CharField(max_length=50) - cve_primary_name = models.CharField(max_length=50) - description = models.TextField(blank=True) + cve_primary_name = models.CharField(max_length=50, default='') + description = models.TextField(blank=True, default='') - public = models.BooleanField(default=False) - comments = models.TextField(blank=True) - comments_private = models.TextField(blank=True) + public = models.BooleanField(default=True) + comments = models.TextField(blank=True, default='') + comments_private = models.TextField(blank=True, default='') status = models.IntegerField(choices=STATUS, default=INVESTIGATE) outcome = models.IntegerField(choices=OUTCOME, default=OPEN) @@ -485,7 +533,7 @@ class Vulnerability(models.Model): except IntegrityError: print("Error in new_vulnerability_name") raise - return "V%04d" % index + return "V%05d" % index class VulnerabilityComments(models.Model): vulnerability = models.ForeignKey(Vulnerability,related_name="vulnerability_comments") @@ -513,7 +561,7 @@ class CveToVulnerablility(models.Model): # Defects -# Defect Record +# Defect Record class Defect(models.Model): search_allowed_fields = ['name', 'summary', 'release_version'] #, 'product'] @@ -521,11 +569,13 @@ class Defect(models.Model): #Issue Type,Key,Summary,Priority,Status,Resolution,Publish To OLS,Fix Version #Bug,LIN10-2031,Security Advisory - libvorbis - CVE-2017-14633,P3,Closed,Fixed,Reviewed - Publish,10.17.41.3 - MINOR = 0 - LOW = 1 - MEDIUM = 2 - HIGH = 3 + NONE = 0 + MINOR = 1 + LOW = 2 + MEDIUM = 3 + HIGH = 4 Priority = ( + (NONE, 'None'), (MINOR, 'P4'), (LOW, 'P3'), (MEDIUM, 'P2'), @@ -546,35 +596,49 @@ class Defect(models.Model): (CLOSED, 'Closed'), ) UNRESOLVED = 0 - FIXED = 1 - WILL_NOT_FIX = 2 - WITHDRAWN = 3 - REJECTED = 4 - DUPLICATE = 5 + RESOLVED = 1 + FIXED = 2 + WILL_NOT_FIX = 3 + WITHDRAWN = 4 + REJECTED = 5 + DUPLICATE = 6 + NOT_APPLICABLE = 7 + REPLACED_BY_REQUIREMENT = 8 + CANNOT_REPRODUCE = 9 + DONE = 10 Resolution = ( (UNRESOLVED, 'Unresolved'), + (RESOLVED, 'Resolved'), (FIXED, 'Fixed'), (WILL_NOT_FIX, 'Won\'t Fix'), (WITHDRAWN, 'Withdrawn'), (REJECTED, 'Rejected'), (DUPLICATE, 'Duplicate'), + (NOT_APPLICABLE, 'Not Applicable'), + (REPLACED_BY_REQUIREMENT, 'Replaced By Requirement'), + (CANNOT_REPRODUCE, 'Cannot Reproduce'), + (DONE, 'Done'), ) name = models.CharField(max_length=50) summary = models.TextField(blank=True) + url = models.TextField(blank=True) priority = models.IntegerField(choices=Priority, default=MINOR) status = models.IntegerField(choices=Status, default=OPEN) resolution = models.IntegerField(choices=Resolution, default=UNRESOLVED) - publishOLS = models.TextField(blank=True) + publish = models.TextField(blank=True) release_version = models.CharField(max_length=50) product = models.ForeignKey(Product,related_name="product_defect") date_created = models.CharField(max_length=50) date_updated = models.CharField(max_length=50) # Methods + @property def get_priority_text(self): return Defect.Priority[int(self.priority)][1] + @property def get_status_text(self): return Defect.Status[int(self.status)][1] + @property def get_resolution_text(self): return Defect.Resolution[int(self.resolution)][1] def get_long_name(self): @@ -585,7 +649,7 @@ class Defect(models.Model): # INVESTIGATION -# Product-level Vulnerablility Investigation Record +# Product-level Vulnerablility Investigation Record class Investigation(models.Model): search_allowed_fields = ['name', 'comments', 'comments_private'] @@ -607,10 +671,15 @@ class Investigation(models.Model): (FIXED, 'Fixed'), (NOT_FIX, "Won't Fix"), ) - LOW = 0 - MEDIUM = 1 - HIGH = 2 + # SRTool Severity, matched with Cve/Defect Priority with placeholder for 'minor' + UNDEFINED = 0 + MINOR = 1 + LOW = 2 + MEDIUM = 3 + HIGH = 4 SEVERITY = ( + (UNDEFINED, ''), + (MINOR, 'Minor'), (LOW, 'Low'), (MEDIUM, 'Medium'), (HIGH, 'High'), @@ -650,7 +719,7 @@ class Investigation(models.Model): index = int(current_investigation_index.value) + 1 current_investigation_index.value = str(index) current_investigation_index.save() - return "I%04d" % index + return "I%05d" % index class InvestigationToDefect(models.Model): investigation = models.ForeignKey(Investigation,related_name="investigation_to_defect") @@ -700,7 +769,7 @@ class User(models.Model): # the default guest user account is ID=1 USER_GUEST = 1 - # Access model + # Access model READER = 0 CONTRIBUTOR = 1 CREATOR = 2 @@ -720,6 +789,32 @@ class User(models.Model): def get_access_text(self): return User.ACCESS[int(self.access)][1] + @property + def builtin(self): + return( ('Guest' == self.name) or ('SRTool' == self.name)) + +# Minimal and safe User object to pass to web pages (no passwords) +class UserSafe(): + def __init__(self, pk, name, email): + self.pk = pk + self.name = name + self.email = email + + def __str__(self): + return "UserSafeStr=%d,%s,%s" % (self.pk, self.name, self.email) + + @staticmethod + def get_safe_userlist(allow_assignment=True): + user_list = [] + for user in User.objects.all(): + if 'SRTool' == user.name: + continue + if allow_assignment and ('Guest' == user.name): + continue + u = UserSafe(user.id,user.name,user.email) + user_list.append(u) + return user_list + class VulnerabilityAccess(models.Model): vulnerability = models.ForeignKey(Vulnerability,related_name="vulnerability_users") user = models.ForeignKey(User,related_name="vulnerability_user") @@ -759,47 +854,59 @@ class Keywords(models.Model): def get_keytype_text(self): return Keywords.KeyType[int(self.key_type)][1] - - -# ==== +# Items waiting for SRTool external publishing +class PublishPending(models.Model): + cve = models.ForeignKey(Cve,related_name="publish_pending_cves",blank=True,null=True) + vulnerability = models.ForeignKey(Vulnerability,related_name="publish_pending_vulnerabilities",blank=True,null=True) + investigation = models.ForeignKey(Investigation,related_name="publish_pending_investigations",blank=True,null=True) + date = models.DateField(null=True, blank=True) + note = models.TextField(blank=True) + +# ==== Support clases, meta classes ==== + +def _log_args(msg, *args, **kwargs): + s = '%s:(' % msg + if args: + for a in args: + s += '%s,' % a + s += '),(' + if kwargs: + for key, value in kwargs.iteritems(): + s += '(%s=%s),' % (key,value) + s += ')' + _log(s) class Access(): # default user is "Guest" - - def read_values(self): - v, created = SrtSetting.objects.get_or_create(name='current_user') - if created: - v.value = User.USER_GUEST - v.save() - self.current_user = int(v.value) + + def __init__(self, *args, **kwargs): + _log_args("ACCESS:", *args, **kwargs) + srt_user_id = args[0] + + # default to "Guest" + if 0 == srt_user_id: + srt_user_id = 1 + + self.current_user = srt_user_id try: - self.current_user_name = User.objects.get(pk=self.current_user).name + user = User.objects.get(pk=self.current_user) + self.current_user_name = user.name + self.current_user_access = user.access except: self.current_user_name = '<not_found>' - - v, created = SrtSetting.objects.get_or_create(name='current_user_access') - if created: - v.value = User.READER - v.save() - self.current_user_access = int(v.value) + self.current_user_access = User.READER def is_guest(self): - self.read_values() return self.current_user == User.USER_GUEST def is_reader(self): - self.read_values() return self.current_user_access >= User.READER def is_contributor(self): - self.read_values() return self.current_user_access >= User.CONTRIBUTOR def is_creator(self): - self.read_values() return self.current_user_access >= User.CREATOR def is_admin(self): - self.read_values() return self.current_user_access >= User.ADMIN def user_name(self): - self.read_values() return self.current_user_name # Database Cache Support |