diff options
Diffstat (limited to 'bitbake/lib/toaster/toastermain')
9 files changed, 267 insertions, 110 deletions
diff --git a/bitbake/lib/toaster/toastermain/logs.py b/bitbake/lib/toaster/toastermain/logs.py new file mode 100644 index 0000000000..62d871963a --- /dev/null +++ b/bitbake/lib/toaster/toastermain/logs.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import logging +import json +from pathlib import Path +from django.http import HttpRequest + +BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp')) + +def log_api_request(request, response, view, logger_name='api'): + """Helper function for LogAPIMixin""" + + repjson = { + 'view': view, + 'path': request.path, + 'method': request.method, + 'status': response.status_code + } + + logger = logging.getLogger(logger_name) + logger.info( + json.dumps(repjson, indent=4, separators=(", ", " : ")) + ) + + +def log_view_mixin(view): + def log_view_request(*args, **kwargs): + # get request from args else kwargs + request = None + if len(args) > 0: + for req in args: + if isinstance(req, HttpRequest): + request = req + break + elif request is None: + request = kwargs.get('request') + + response = view(*args, **kwargs) + view_name = 'unknown' + if hasattr(request, 'resolver_match'): + if hasattr(request.resolver_match, 'view_name'): + view_name = request.resolver_match.view_name + + log_api_request( + request, response, view_name, 'toaster') + return response + return log_view_request + + + +class LogAPIMixin: + """Logs API requests + + tested with: + - APIView + - ModelViewSet + - ReadOnlyModelViewSet + - GenericAPIView + + Note: you can set `view_name` attribute in View to override get_view_name() + """ + + def get_view_name(self): + if hasattr(self, 'view_name'): + return self.view_name + return super().get_view_name() + + def finalize_response(self, request, response, *args, **kwargs): + log_api_request(request, response, self.get_view_name()) + return super().finalize_response(request, response, *args, **kwargs) + + +LOGGING_SETTINGS = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'datetime': { + 'format': '%(asctime)s %(levelname)s %(message)s' + }, + 'verbose': { + 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}', + 'datefmt': "%d/%b/%Y %H:%M:%S", + 'style': '{', + }, + 'api': { + 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', + 'style': '{' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'datetime', + }, + 'file_django': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BUILDDIR / 'toaster_logs/django.log', + 'when': 'D', # interval type + 'interval': 1, # defaults to 1 + 'backupCount': 10, # how many files to keep + 'formatter': 'verbose', + }, + 'file_api': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BUILDDIR / 'toaster_logs/api.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + 'file_toaster': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BUILDDIR / 'toaster_logs/web.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['file_django', 'console'], + 'level': 'WARN', + 'propagate': True, + }, + 'django': { + 'handlers': ['file_django', 'console'], + 'level': 'WARNING', + 'propogate': True, + }, + 'toaster': { + 'handlers': ['file_toaster'], + 'level': 'INFO', + 'propagate': False, + }, + 'api': { + 'handlers': ['file_api'], + 'level': 'INFO', + 'propagate': False, + } + } +} diff --git a/bitbake/lib/toaster/toastermain/management/commands/builddelete.py b/bitbake/lib/toaster/toastermain/management/commands/builddelete.py index c2d773a577..93919dec2d 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/builddelete.py +++ b/bitbake/lib/toaster/toastermain/management/commands/builddelete.py @@ -2,12 +2,10 @@ # SPDX-License-Identifier: GPL-2.0-only # -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from django.core.exceptions import ObjectDoesNotExist from orm.models import Build from django.db import OperationalError -import os - class Command(BaseCommand): diff --git a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py index 408ad44e6e..f7139aa041 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py +++ b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py @@ -31,12 +31,10 @@ # ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import -from django.core.management.base import BaseCommand, CommandError -from django.core.exceptions import ObjectDoesNotExist -from orm.models import ProjectManager, Project, Release, ProjectVariable +from django.core.management.base import BaseCommand +from orm.models import Project, Release, ProjectVariable from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer from toastergui.api import scan_layer_content -from django.db import OperationalError import os import re @@ -116,6 +114,15 @@ class Command(BaseCommand): help='command (configure,reconfigure,import)', ) + def get_var(self, varname): + value = self.vars.get(varname, '') + if value: + varrefs = re.findall('\${([^}]*)}', value) + for ref in varrefs: + if ref in self.vars: + value = value.replace('${%s}' % ref, self.vars[ref]) + return value + # Extract the bb variables from a conf file def scan_conf(self,fn): vars = self.vars @@ -243,7 +250,7 @@ class Command(BaseCommand): # Apply table of all layer versions def extract_bblayers(self): # set up the constants - bblayer_str = self.vars['BBLAYERS'] + bblayer_str = self.get_var('BBLAYERS') TOASTER_DIR = os.environ.get('TOASTER_DIR') INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/" TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/" @@ -423,6 +430,7 @@ class Command(BaseCommand): # Scan the project's conf files (if any) def scan_conf_variables(self,project_path): + self.vars['TOPDIR'] = project_path # scan the project's settings, add any new layers or variables if os.path.isfile("%s/conf/local.conf" % project_path): self.scan_conf("%s/conf/local.conf" % project_path) @@ -443,7 +451,7 @@ class Command(BaseCommand): # Catch vars relevant to Toaster (in case no Toaster section) self.update_project_vars(project,'DISTRO') self.update_project_vars(project,'MACHINE') - self.update_project_vars(project,'IMAGE_INSTALL_append') + self.update_project_vars(project,'IMAGE_INSTALL:append') self.update_project_vars(project,'IMAGE_FSTYPES') self.update_project_vars(project,'PACKAGE_CLASSES') # These vars are typically only assigned by Toaster @@ -468,7 +476,6 @@ class Command(BaseCommand): release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit)) - def handle(self, *args, **options): project_name = options['name'] project_path = options['path'] @@ -507,7 +514,7 @@ class Command(BaseCommand): default_release = Release.objects.get(id=1) # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import' - if ("reconfigure" == options['command']) and (None == project): + if ("reconfigure" == options['command']) and project is None: options['command'] = 'import' # 'Configure': @@ -538,7 +545,7 @@ class Command(BaseCommand): # Find the directory's release, and promote to default_release if local paths release = self.find_import_release(layers_list,lv_dict,default_release) # create project, SANITY: reuse any project of same name - project = Project.objects.create_project(project_name,release,project) + project = Project.objects.create_project(project_name,release,project, imported=True) # Apply any new layers or variables self.apply_conf_variables(project,layers_list,lv_dict,release) # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific' @@ -553,6 +560,7 @@ class Command(BaseCommand): # preset the mode and default image recipe project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW) project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal") + # Assert any extended/custom actions or variables for new non-Toaster projects if not len(self.toaster_vars): pass diff --git a/bitbake/lib/toaster/toastermain/management/commands/buildslist.py b/bitbake/lib/toaster/toastermain/management/commands/buildslist.py index 1ed20224d3..3ad5289c5e 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/buildslist.py +++ b/bitbake/lib/toaster/toastermain/management/commands/buildslist.py @@ -2,9 +2,8 @@ # SPDX-License-Identifier: GPL-2.0-only # -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand from orm.models import Build -import os diff --git a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py index 811fd5d516..b2c002da7a 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py +++ b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py @@ -13,7 +13,7 @@ import errno import socket from django.core.management.base import BaseCommand, CommandError -from django.utils.encoding import force_text +from django.utils.encoding import force_str DEFAULT_ADDRPORT = "0.0.0.0:8000" @@ -51,7 +51,7 @@ class Command(BaseCommand): if hasattr(err, 'errno') and err.errno in errors: errtext = errors[err.errno] else: - errtext = force_text(err) + errtext = force_str(err) raise CommandError(errtext) self.stdout.write("OK") diff --git a/bitbake/lib/toaster/toastermain/management/commands/perf.py b/bitbake/lib/toaster/toastermain/management/commands/perf.py index 7d629fb37f..5c41c5b2f2 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/perf.py +++ b/bitbake/lib/toaster/toastermain/management/commands/perf.py @@ -17,46 +17,46 @@ class Command(BaseCommand): help = "Test the response time for all toaster urls" def handle(self, *args, **options): - root_urlconf = __import__(settings.ROOT_URLCONF) - patterns = root_urlconf.urls.urlpatterns - global full_url - for pat in patterns: - if pat.__class__.__name__ == 'RegexURLResolver': - url_root_res = str(pat).split('^')[1].replace('>', '') - if 'gui' in url_root_res: - for url_patt in pat.url_patterns: - full_url = self.get_full_url(url_patt, url_root_res) - info = self.url_info(full_url) - status_code = info[0] - load_time = info[1] - print('Trying \'' + full_url + '\', ' + str(status_code) + ', ' + str(load_time)) + root_urlconf = __import__(settings.ROOT_URLCONF) + patterns = root_urlconf.urls.urlpatterns + global full_url + for pat in patterns: + if pat.__class__.__name__ == 'RegexURLResolver': + url_root_res = str(pat).split('^')[1].replace('>', '') + if 'gui' in url_root_res: + for url_patt in pat.url_patterns: + full_url = self.get_full_url(url_patt, url_root_res) + info = self.url_info(full_url) + status_code = info[0] + load_time = info[1] + print('Trying \'' + full_url + '\', ' + str(status_code) + ', ' + str(load_time)) def get_full_url(self, url_patt, url_root_res): - full_url = str(url_patt).split('^')[1].replace('$>', '').replace('(?P<file_path>(?:/[', '/bin/busybox').replace('.*', '') - full_url = str(url_root_res + full_url) - full_url = re.sub('\(\?P<.*?>\\\d\+\)', '1', full_url) - full_url = 'http://localhost:8000/' + full_url - return full_url + full_url = str(url_patt).split('^')[1].replace('$>', '').replace('(?P<file_path>(?:/[', '/bin/busybox').replace('.*', '') + full_url = str(url_root_res + full_url) + full_url = re.sub('\(\?P<.*?>\\\d\+\)', '1', full_url) + full_url = 'http://localhost:8000/' + full_url + return full_url def url_info(self, full_url): - client = Client() - info = [] - try: - resp = client.get(full_url, follow = True) - except Exception as e_status_code: + client = Client() + info = [] + try: + resp = client.get(full_url, follow = True) + except Exception as e_status_code: self.error('Url: %s, error: %s' % (full_url, e_status_code)) resp = type('object', (), {'status_code':0, 'content': str(e_status_code)}) - status_code = resp.status_code - info.append(status_code) - try: - req = requests.get(full_url) - except Exception as e_load_time: + status_code = resp.status_code + info.append(status_code) + try: + req = requests.get(full_url) + except Exception as e_load_time: self.error('Url: %s, error: %s' % (full_url, e_load_time)) - load_time = req.elapsed - info.append(load_time) - return info + load_time = req.elapsed + info.append(load_time) + return info def error(self, *args): - for arg in args: - print(arg, end=' ', file=sys.stderr) - print(file=sys.stderr) + for arg in args: + print(arg, end=' ', file=sys.stderr) + print(file=sys.stderr) diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py index 74501fa26b..e06adc5a93 100644 --- a/bitbake/lib/toaster/toastermain/settings.py +++ b/bitbake/lib/toaster/toastermain/settings.py @@ -9,6 +9,8 @@ # Django settings for Toaster project. import os +from pathlib import Path +from toastermain.logs import LOGGING_SETTINGS DEBUG = True @@ -39,6 +41,9 @@ DATABASES = { } } +# New in Django 3.2 +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + # Needed when Using sqlite especially to add a longer timeout for waiting # for the database lock to be released # https://docs.djangoproject.com/en/1.6/ref/databases/#database-is-locked-errors @@ -84,14 +89,17 @@ else: from pytz.exceptions import UnknownTimeZoneError try: if pytz.timezone(zonename) is not None: - zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename + with open(filepath, 'rb') as f: + zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename except UnknownTimeZoneError as ValueError: # we expect timezone failures here, just move over pass except ImportError: - zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename + with open(filepath, 'rb') as f: + zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename - TIME_ZONE = zonefilelist[hashlib.md5(open('/etc/localtime', 'rb').read()).hexdigest()] + with open('/etc/localtime', 'rb') as f: + TIME_ZONE = zonefilelist[hashlib.md5(f.read()).hexdigest()] # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html @@ -103,10 +111,6 @@ SITE_ID = 1 # to load the internationalization machinery. USE_I18N = True -# If you set this to False, Django will not format dates, numbers and -# calendars according to the current locale. -USE_L10N = True - # If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = True @@ -147,6 +151,8 @@ STATICFILES_FINDERS = ( # Make this unique, and don't share it with anybody. SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT' +TMPDIR = os.environ.get('TOASTER_DJANGO_TMPDIR', '/tmp') + class InvalidString(str): def __mod__(self, other): from django.template.base import TemplateSyntaxError @@ -183,21 +189,28 @@ TEMPLATES = [ 'django.template.loaders.app_directories.Loader', #'django.template.loaders.eggs.Loader', ], - 'string_if_invalid': InvalidString("%s"), + # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled + # Generally, string_if_invalid should only be enabled in order to debug + # a specific template problem, then cleared once debugging is complete. + # If you assign a value other than '' to string_if_invalid, + # you will experience rendering problems with these templates and sites. + # 'string_if_invalid': InvalidString("%s"), + 'string_if_invalid': "", 'debug': DEBUG, }, }, ] -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', - # Uncomment the next line for simple clickjacking protection: - # 'django.middleware.clickjacking.XFrameOptionsMiddleware', -) + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', +] CACHES = { # 'default': { @@ -206,7 +219,7 @@ CACHES = { # }, 'default': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/tmp/toaster_cache_%d' % os.getuid(), + 'LOCATION': '%s/toaster_cache_%d' % (TMPDIR, os.getuid()), 'TIMEOUT': 1, } } @@ -238,6 +251,9 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'bldcollector', 'toastermain', + + # 3rd-lib + "log_viewer", ) @@ -248,7 +264,7 @@ FRESH_ENABLED = False if os.environ.get('TOASTER_DEVEL', None) is not None: try: import fresh - MIDDLEWARE_CLASSES = ("fresh.middleware.FreshMiddleware",) + MIDDLEWARE_CLASSES + MIDDLEWARE = ["fresh.middleware.FreshMiddleware",] + MIDDLEWARE INSTALLED_APPS = INSTALLED_APPS + ('fresh',) FRESH_ENABLED = True except: @@ -258,8 +274,8 @@ DEBUG_PANEL_ENABLED = False if os.environ.get('TOASTER_DEVEL', None) is not None: try: import debug_toolbar, debug_panel - MIDDLEWARE_CLASSES = ('debug_panel.middleware.DebugPanelMiddleware',) + MIDDLEWARE_CLASSES - #MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ('debug_toolbar.middleware.DebugToolbarMiddleware',) + MIDDLEWARE = ['debug_panel.middleware.DebugPanelMiddleware',] + MIDDLEWARE + #MIDDLEWARE = MIDDLEWARE + ['debug_toolbar.middleware.DebugToolbarMiddleware',] INSTALLED_APPS = INSTALLED_APPS + ('debug_toolbar','debug_panel',) DEBUG_PANEL_ENABLED = True @@ -298,43 +314,21 @@ for t in os.walk(os.path.dirname(currentdir)): # the site admins on every HTTP 500 error when DEBUG=False. # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - } - }, - 'formatters': { - 'datetime': { - 'format': '%(asctime)s %(levelname)s %(message)s' - } - }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false'], - 'class': 'django.utils.log.AdminEmailHandler' - }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'datetime', - } - }, - 'loggers': { - 'toaster' : { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'WARN', - 'propagate': True, - }, - } -} +LOGGING = LOGGING_SETTINGS + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BUILDDIR = os.environ.get("BUILDDIR", TMPDIR) + +# LOG VIEWER +# https://pypi.org/project/django-log-viewer/ +LOG_VIEWER_FILES_PATTERN = '*.log*' +LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/") +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] + +# Optionally you can set the next variables in order to customize the admin: +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" if DEBUG and SQL_DEBUG: LOGGING['loggers']['django.db.backends'] = { @@ -352,5 +346,3 @@ def activate_synchronous_off(sender, connection, **kwargs): connection_created.connect(activate_synchronous_off) # - - diff --git a/bitbake/lib/toaster/toastermain/settings_test.py b/bitbake/lib/toaster/toastermain/settings_test.py index 6538d9e453..74def2d240 100644 --- a/bitbake/lib/toaster/toastermain/settings_test.py +++ b/bitbake/lib/toaster/toastermain/settings_test.py @@ -19,10 +19,10 @@ TEMPLATE_DEBUG = DEBUG DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '/tmp/toaster-test-db.sqlite', + 'NAME': '%s/toaster-test-db.sqlite' % TMPDIR, 'TEST': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '/tmp/toaster-test-db.sqlite', + 'NAME': '%s/toaster-test-db.sqlite' % TMPDIR, } } } diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py index ac77bc3632..3be46fcf0c 100644 --- a/bitbake/lib/toaster/toastermain/urls.py +++ b/bitbake/lib/toaster/toastermain/urls.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -from django.conf.urls import include, url +from django.urls import re_path as url, include from django.views.generic import RedirectView, TemplateView from django.views.decorators.cache import never_cache import bldcollector.views @@ -28,6 +28,8 @@ urlpatterns = [ # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + url(r'^logs/', include('log_viewer.urls')), + # This is here to maintain backward compatibility and will be deprecated # in the future. url(r'^orm/eventfile$', bldcollector.views.eventfile), @@ -51,7 +53,7 @@ if toastermain.settings.DEBUG_PANEL_ENABLED: urlpatterns = [ # Uncomment the next line to enable the admin: - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), ] + urlpatterns # Automatically discover urls.py in various apps, beside our own @@ -69,7 +71,7 @@ for t in os.walk(os.path.dirname(currentdir)): # make sure we don't have this module name in conflict = False for p in urlpatterns: - if p.regex.pattern == '^' + modulename + '/': + if p.pattern.regex.pattern == '^' + modulename + '/': conflict = True if not conflict: urlpatterns.insert(0, url(r'^' + modulename + '/', include ( modulename + '.urls'))) |