diff options
Diffstat (limited to 'bitbake/lib/toaster/toastermain')
6 files changed, 206 insertions, 53 deletions
diff --git a/bitbake/lib/toaster/toastermain/logs.py b/bitbake/lib/toaster/toastermain/logs.py new file mode 100644 index 0000000000..62d871963a --- /dev/null +++ b/bitbake/lib/toaster/toastermain/logs.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import os +import logging +import json +from pathlib import Path +from django.http import HttpRequest + +BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp')) + +def log_api_request(request, response, view, logger_name='api'): + """Helper function for LogAPIMixin""" + + repjson = { + 'view': view, + 'path': request.path, + 'method': request.method, + 'status': response.status_code + } + + logger = logging.getLogger(logger_name) + logger.info( + json.dumps(repjson, indent=4, separators=(", ", " : ")) + ) + + +def log_view_mixin(view): + def log_view_request(*args, **kwargs): + # get request from args else kwargs + request = None + if len(args) > 0: + for req in args: + if isinstance(req, HttpRequest): + request = req + break + elif request is None: + request = kwargs.get('request') + + response = view(*args, **kwargs) + view_name = 'unknown' + if hasattr(request, 'resolver_match'): + if hasattr(request.resolver_match, 'view_name'): + view_name = request.resolver_match.view_name + + log_api_request( + request, response, view_name, 'toaster') + return response + return log_view_request + + + +class LogAPIMixin: + """Logs API requests + + tested with: + - APIView + - ModelViewSet + - ReadOnlyModelViewSet + - GenericAPIView + + Note: you can set `view_name` attribute in View to override get_view_name() + """ + + def get_view_name(self): + if hasattr(self, 'view_name'): + return self.view_name + return super().get_view_name() + + def finalize_response(self, request, response, *args, **kwargs): + log_api_request(request, response, self.get_view_name()) + return super().finalize_response(request, response, *args, **kwargs) + + +LOGGING_SETTINGS = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'formatters': { + 'datetime': { + 'format': '%(asctime)s %(levelname)s %(message)s' + }, + 'verbose': { + 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}', + 'datefmt': "%d/%b/%Y %H:%M:%S", + 'style': '{', + }, + 'api': { + 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', + 'style': '{' + } + }, + 'handlers': { + 'mail_admins': { + 'level': 'ERROR', + 'filters': ['require_debug_false'], + 'class': 'django.utils.log.AdminEmailHandler' + }, + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'datetime', + }, + 'file_django': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BUILDDIR / 'toaster_logs/django.log', + 'when': 'D', # interval type + 'interval': 1, # defaults to 1 + 'backupCount': 10, # how many files to keep + 'formatter': 'verbose', + }, + 'file_api': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BUILDDIR / 'toaster_logs/api.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + 'file_toaster': { + 'level': 'INFO', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': BUILDDIR / 'toaster_logs/web.log', + 'when': 'D', + 'interval': 1, + 'backupCount': 10, + 'formatter': 'verbose', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['file_django', 'console'], + 'level': 'WARN', + 'propagate': True, + }, + 'django': { + 'handlers': ['file_django', 'console'], + 'level': 'WARNING', + 'propogate': True, + }, + 'toaster': { + 'handlers': ['file_toaster'], + 'level': 'INFO', + 'propagate': False, + }, + 'api': { + 'handlers': ['file_api'], + 'level': 'INFO', + 'propagate': False, + } + } +} diff --git a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py index 59da6ff7ac..f7139aa041 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py +++ b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py @@ -451,7 +451,7 @@ class Command(BaseCommand): # Catch vars relevant to Toaster (in case no Toaster section) self.update_project_vars(project,'DISTRO') self.update_project_vars(project,'MACHINE') - self.update_project_vars(project,'IMAGE_INSTALL_append') + self.update_project_vars(project,'IMAGE_INSTALL:append') self.update_project_vars(project,'IMAGE_FSTYPES') self.update_project_vars(project,'PACKAGE_CLASSES') # These vars are typically only assigned by Toaster @@ -545,7 +545,7 @@ class Command(BaseCommand): # Find the directory's release, and promote to default_release if local paths release = self.find_import_release(layers_list,lv_dict,default_release) # create project, SANITY: reuse any project of same name - project = Project.objects.create_project(project_name,release,project) + project = Project.objects.create_project(project_name,release,project, imported=True) # Apply any new layers or variables self.apply_conf_variables(project,layers_list,lv_dict,release) # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific' diff --git a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py index 811fd5d516..b2c002da7a 100644 --- a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py +++ b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py @@ -13,7 +13,7 @@ import errno import socket from django.core.management.base import BaseCommand, CommandError -from django.utils.encoding import force_text +from django.utils.encoding import force_str DEFAULT_ADDRPORT = "0.0.0.0:8000" @@ -51,7 +51,7 @@ class Command(BaseCommand): if hasattr(err, 'errno') and err.errno in errors: errtext = errors[err.errno] else: - errtext = force_text(err) + errtext = force_str(err) raise CommandError(errtext) self.stdout.write("OK") diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py index a4b370c8d4..e06adc5a93 100644 --- a/bitbake/lib/toaster/toastermain/settings.py +++ b/bitbake/lib/toaster/toastermain/settings.py @@ -9,6 +9,8 @@ # Django settings for Toaster project. import os +from pathlib import Path +from toastermain.logs import LOGGING_SETTINGS DEBUG = True @@ -39,6 +41,9 @@ DATABASES = { } } +# New in Django 3.2 +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + # Needed when Using sqlite especially to add a longer timeout for waiting # for the database lock to be released # https://docs.djangoproject.com/en/1.6/ref/databases/#database-is-locked-errors @@ -84,14 +89,17 @@ else: from pytz.exceptions import UnknownTimeZoneError try: if pytz.timezone(zonename) is not None: - zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename + with open(filepath, 'rb') as f: + zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename except UnknownTimeZoneError as ValueError: # we expect timezone failures here, just move over pass except ImportError: - zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename + with open(filepath, 'rb') as f: + zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename - TIME_ZONE = zonefilelist[hashlib.md5(open('/etc/localtime', 'rb').read()).hexdigest()] + with open('/etc/localtime', 'rb') as f: + TIME_ZONE = zonefilelist[hashlib.md5(f.read()).hexdigest()] # Language code for this installation. All choices can be found here: # http://www.i18nguy.com/unicode/language-identifiers.html @@ -103,10 +111,6 @@ SITE_ID = 1 # to load the internationalization machinery. USE_I18N = True -# If you set this to False, Django will not format dates, numbers and -# calendars according to the current locale. -USE_L10N = True - # If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = True @@ -147,6 +151,8 @@ STATICFILES_FINDERS = ( # Make this unique, and don't share it with anybody. SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT' +TMPDIR = os.environ.get('TOASTER_DJANGO_TMPDIR', '/tmp') + class InvalidString(str): def __mod__(self, other): from django.template.base import TemplateSyntaxError @@ -183,7 +189,13 @@ TEMPLATES = [ 'django.template.loaders.app_directories.Loader', #'django.template.loaders.eggs.Loader', ], - 'string_if_invalid': InvalidString("%s"), + # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled + # Generally, string_if_invalid should only be enabled in order to debug + # a specific template problem, then cleared once debugging is complete. + # If you assign a value other than '' to string_if_invalid, + # you will experience rendering problems with these templates and sites. + # 'string_if_invalid': InvalidString("%s"), + 'string_if_invalid': "", 'debug': DEBUG, }, }, @@ -207,7 +219,7 @@ CACHES = { # }, 'default': { 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/tmp/toaster_cache_%d' % os.getuid(), + 'LOCATION': '%s/toaster_cache_%d' % (TMPDIR, os.getuid()), 'TIMEOUT': 1, } } @@ -239,6 +251,9 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'bldcollector', 'toastermain', + + # 3rd-lib + "log_viewer", ) @@ -299,43 +314,21 @@ for t in os.walk(os.path.dirname(currentdir)): # the site admins on every HTTP 500 error when DEBUG=False. # See http://docs.djangoproject.com/en/dev/topics/logging for # more details on how to customize your logging configuration. -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse' - } - }, - 'formatters': { - 'datetime': { - 'format': '%(asctime)s %(levelname)s %(message)s' - } - }, - 'handlers': { - 'mail_admins': { - 'level': 'ERROR', - 'filters': ['require_debug_false'], - 'class': 'django.utils.log.AdminEmailHandler' - }, - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'datetime', - } - }, - 'loggers': { - 'toaster' : { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'WARN', - 'propagate': True, - }, - } -} +LOGGING = LOGGING_SETTINGS + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BUILDDIR = os.environ.get("BUILDDIR", TMPDIR) + +# LOG VIEWER +# https://pypi.org/project/django-log-viewer/ +LOG_VIEWER_FILES_PATTERN = '*.log*' +LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/") +LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page +LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read +LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] + +# Optionally you can set the next variables in order to customize the admin: +LOG_VIEWER_FILE_LIST_TITLE = "Logs list" if DEBUG and SQL_DEBUG: LOGGING['loggers']['django.db.backends'] = { diff --git a/bitbake/lib/toaster/toastermain/settings_test.py b/bitbake/lib/toaster/toastermain/settings_test.py index 6538d9e453..74def2d240 100644 --- a/bitbake/lib/toaster/toastermain/settings_test.py +++ b/bitbake/lib/toaster/toastermain/settings_test.py @@ -19,10 +19,10 @@ TEMPLATE_DEBUG = DEBUG DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '/tmp/toaster-test-db.sqlite', + 'NAME': '%s/toaster-test-db.sqlite' % TMPDIR, 'TEST': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': '/tmp/toaster-test-db.sqlite', + 'NAME': '%s/toaster-test-db.sqlite' % TMPDIR, } } } diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py index 5fb520b384..3be46fcf0c 100644 --- a/bitbake/lib/toaster/toastermain/urls.py +++ b/bitbake/lib/toaster/toastermain/urls.py @@ -6,7 +6,7 @@ # SPDX-License-Identifier: GPL-2.0-only # -from django.conf.urls import include, url +from django.urls import re_path as url, include from django.views.generic import RedirectView, TemplateView from django.views.decorators.cache import never_cache import bldcollector.views @@ -28,6 +28,8 @@ urlpatterns = [ # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), + url(r'^logs/', include('log_viewer.urls')), + # This is here to maintain backward compatibility and will be deprecated # in the future. url(r'^orm/eventfile$', bldcollector.views.eventfile), |