diff options
Diffstat (limited to 'bitbake/lib/toaster/bldcontrol')
8 files changed, 162 insertions, 44 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/admin.py b/bitbake/lib/toaster/bldcontrol/admin.py index e85c30ed11..1754bc11cc 100644 --- a/bitbake/lib/toaster/bldcontrol/admin.py +++ b/bitbake/lib/toaster/bldcontrol/admin.py @@ -3,7 +3,6 @@ # from django.contrib import admin -from django.contrib.admin.filters import RelatedFieldListFilter from .models import BuildEnvironment class BuildEnvironmentAdmin(admin.ModelAdmin): diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index 301df1880a..71c288df34 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -8,10 +8,8 @@ import os import sys -import re -from django.db import transaction from django.db.models import Q -from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake +from bldcontrol.models import BuildEnvironment, BRLayer, BRBitbake # load Bitbake components path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 39ea736b58..577e765f11 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -7,27 +7,24 @@ # import os -import sys import re import shutil import time -from django.db import transaction -from django.db.models import Q -from bldcontrol.models import BuildEnvironment, BuildRequest, BRLayer, BRVariable, BRTarget, BRBitbake, Build -from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ProjectLayer, ToasterSetting +from bldcontrol.models import BuildEnvironment, BuildRequest, Build +from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ToasterSetting from orm.models import signal_runbuilds import subprocess from toastermain import settings -from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, BitbakeController +from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException import logging logger = logging.getLogger("toaster") install_dir = os.environ.get('TOASTER_DIR') -from pprint import pprint, pformat +from pprint import pformat class LocalhostBEController(BuildEnvironmentController): """ Implementation of the BuildEnvironmentController for the localhost; @@ -203,7 +200,7 @@ class LocalhostBEController(BuildEnvironmentController): localdirpath = os.path.join(localdirname, dirpath) logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath) if not os.path.exists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) + raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Exiting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/"))) @@ -470,7 +467,7 @@ class LocalhostBEController(BuildEnvironmentController): logger.debug("localhostbecontroller: waiting for bblock content to appear") time.sleep(1) else: - raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock) + raise BuildSetupException("Cannot find bitbake server lock file '%s'. Exiting." % bblock) with open(bblock) as fplock: for line in fplock: diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index fe2c4dc2bb..20f9dce569 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -2,11 +2,9 @@ # SPDX-License-Identifier: GPL-2.0-only # -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction +from django.core.management.base import BaseCommand from django.core.management import call_command -from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException from bldcontrol.models import BuildRequest, BuildEnvironment, BRError from orm.models import ToasterSetting, Build, Layer @@ -80,7 +78,7 @@ class Command(BaseCommand): template_conf = os.environ.get("TEMPLATECONF", "") custom_xml_only = os.environ.get("CUSTOM_XML_ONLY") - if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or (not custom_xml_only == None): + if ToasterSetting.objects.filter(name='CUSTOM_XML_ONLY').count() > 0 or custom_xml_only is not None: # only use the custom settings pass elif "poky" in template_conf: diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 50ec409860..834e32b36f 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -168,17 +168,88 @@ class Command(BaseCommand): try: self.cleanup() except Exception as e: - logger.warn("runbuilds: cleanup exception %s" % str(e)) + logger.warning("runbuilds: cleanup exception %s" % str(e)) try: self.archive() except Exception as e: - logger.warn("runbuilds: archive exception %s" % str(e)) + logger.warning("runbuilds: archive exception %s" % str(e)) try: self.schedule() except Exception as e: - logger.warn("runbuilds: schedule exception %s" % str(e)) + logger.warning("runbuilds: schedule exception %s" % str(e)) + + # Test to see if a build pre-maturely died due to a bitbake crash + def check_dead_builds(self): + do_cleanup = False + try: + for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS): + # Get the build directory + if br.project.builddir: + builddir = br.project.builddir + else: + builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id) + # Check log to see if there is a recent traceback + toaster_ui_log = os.path.join(builddir, 'toaster_ui.log') + test_file = os.path.join(builddir, '._toaster_check.txt') + os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file)) + traceback_text = '' + is_traceback = False + with open(test_file,'r') as test_file_fd: + test_file_tail = test_file_fd.readlines() + for line in test_file_tail: + if line.startswith('Traceback (most recent call last):'): + traceback_text = line + is_traceback = True + elif line.startswith('NOTE: ToasterUI waiting for events'): + # Ignore any traceback before new build start + traceback_text = '' + is_traceback = False + elif line.startswith('Note: Toaster traceback auto-stop'): + # Ignore any traceback before this previous traceback catch + traceback_text = '' + is_traceback = False + elif is_traceback: + traceback_text += line + # Test the results + is_stop = False + if is_traceback: + # Found a traceback + errtype = 'Bitbake crash' + errmsg = 'Bitbake crash\n' + traceback_text + state = BuildRequest.REQ_FAILED + # Clean up bitbake files + bitbake_lock = os.path.join(builddir, 'bitbake.lock') + if os.path.isfile(bitbake_lock): + os.remove(bitbake_lock) + bitbake_sock = os.path.join(builddir, 'bitbake.sock') + if os.path.isfile(bitbake_sock): + os.remove(bitbake_sock) + if os.path.isfile(test_file): + os.remove(test_file) + # Add note to ignore this traceback on next check + os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log) + is_stop = True + # Add more tests here + #elif ... + # Stop the build request? + if is_stop: + brerror = BRError( + req = br, + errtype = errtype, + errmsg = errmsg, + traceback = traceback_text, + ) + brerror.save() + br.state = state + br.save() + do_cleanup = True + # Do cleanup + if do_cleanup: + self.cleanup() + except Exception as e: + logger.error("runbuilds: Error in check_dead_builds %s" % e) def handle(self, **options): pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."), @@ -187,10 +258,18 @@ class Command(BaseCommand): with open(pidfile_path, 'w') as pidfile: pidfile.write("%s" % os.getpid()) + # Clean up any stale/failed builds from previous Toaster run self.runbuild() signal.signal(signal.SIGUSR1, lambda sig, frame: None) while True: - signal.pause() - self.runbuild() + sigset = signal.sigtimedwait([signal.SIGUSR1], 5) + if sigset: + for sig in sigset: + # Consume each captured pending event + self.runbuild() + else: + # Check for build exceptions + self.check_dead_builds() + diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py b/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py index 67db378561..7ee19f4745 100644 --- a/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py +++ b/bitbake/lib/toaster/bldcontrol/migrations/0001_initial.py @@ -37,7 +37,7 @@ class Migration(migrations.Migration): ('giturl', models.CharField(max_length=254)), ('commit', models.CharField(max_length=254)), ('dirpath', models.CharField(max_length=254)), - ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True)), + ('layer_version', models.ForeignKey(to='orm.Layer_Version', null=True, on_delete=models.CASCADE)), ], ), migrations.CreateModel( @@ -80,34 +80,34 @@ class Migration(migrations.Migration): ('state', models.IntegerField(default=0, choices=[(0, b'created'), (1, b'queued'), (2, b'in progress'), (3, b'completed'), (4, b'failed'), (5, b'deleted'), (6, b'archive')])), ('created', models.DateTimeField(auto_now_add=True)), ('updated', models.DateTimeField(auto_now=True)), - ('build', models.OneToOneField(null=True, to='orm.Build')), - ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True)), - ('project', models.ForeignKey(to='orm.Project')), + ('build', models.OneToOneField(null=True, to='orm.Build', on_delete=models.CASCADE)), + ('environment', models.ForeignKey(to='bldcontrol.BuildEnvironment', null=True, on_delete=models.CASCADE)), + ('project', models.ForeignKey(to='orm.Project', on_delete=models.CASCADE)), ], ), migrations.AddField( model_name='brvariable', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brtarget', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brlayer', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brerror', name='req', - field=models.ForeignKey(to='bldcontrol.BuildRequest'), + field=models.ForeignKey(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), migrations.AddField( model_name='brbitbake', name='req', - field=models.OneToOneField(to='bldcontrol.BuildRequest'), + field=models.OneToOneField(to='bldcontrol.BuildRequest', on_delete=models.CASCADE), ), ] diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py new file mode 100644 index 0000000000..45b477d02c --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py @@ -0,0 +1,48 @@ +# Generated by Django 3.2.12 on 2022-03-06 03:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bldcontrol', '0007_brlayers_optional_gitinfo'), + ] + + operations = [ + migrations.AlterField( + model_name='brbitbake', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brerror', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brlayer', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brtarget', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='brvariable', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='buildenvironment', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='buildrequest', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py index bcffcf5e20..42750e7180 100644 --- a/bitbake/lib/toaster/bldcontrol/models.py +++ b/bitbake/lib/toaster/bldcontrol/models.py @@ -4,9 +4,8 @@ from __future__ import unicode_literals from django.db import models -from django.core.validators import MaxValueValidator, MinValueValidator -from django.utils.encoding import force_text -from orm.models import Project, ProjectLayer, ProjectVariable, ProjectTarget, Build, Layer_Version +from django.utils.encoding import force_str +from orm.models import Project, Build, Layer_Version import logging logger = logging.getLogger("toaster") @@ -86,9 +85,9 @@ class BuildRequest(models.Model): search_allowed_fields = ("brtarget__target", "build__project__name") - project = models.ForeignKey(Project) - build = models.OneToOneField(Build, null = True) # TODO: toasterui should set this when Build is created - environment = models.ForeignKey(BuildEnvironment, null = True) + project = models.ForeignKey(Project, on_delete=models.CASCADE) + build = models.OneToOneField(Build, on_delete=models.CASCADE, null = True) # TODO: toasterui should set this when Build is created + environment = models.ForeignKey(BuildEnvironment, on_delete=models.CASCADE, null = True) state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED) created = models.DateTimeField(auto_now_add = True) updated = models.DateTimeField(auto_now = True) @@ -125,39 +124,39 @@ class BuildRequest(models.Model): return self.brvariable_set.get(name="MACHINE").value def __str__(self): - return force_text('%s %s' % (self.project, self.get_state_display())) + return force_str('%s %s' % (self.project, self.get_state_display())) # These tables specify the settings for running an actual build. # They MUST be kept in sync with the tables in orm.models.Project* class BRLayer(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) name = models.CharField(max_length=100) giturl = models.CharField(max_length=254, null=True) local_source_dir = models.CharField(max_length=254, null=True) commit = models.CharField(max_length=254, null=True) dirpath = models.CharField(max_length=254, null=True) - layer_version = models.ForeignKey(Layer_Version, null=True) + layer_version = models.ForeignKey(Layer_Version, on_delete=models.CASCADE, null=True) class BRBitbake(models.Model): - req = models.OneToOneField(BuildRequest) # only one bitbake for a request + req = models.OneToOneField(BuildRequest, on_delete=models.CASCADE) # only one bitbake for a request giturl = models.CharField(max_length =254) commit = models.CharField(max_length = 254) dirpath = models.CharField(max_length = 254) class BRVariable(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) name = models.CharField(max_length=100) value = models.TextField(blank = True) class BRTarget(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) target = models.CharField(max_length=100) task = models.CharField(max_length=100, null=True) class BRError(models.Model): - req = models.ForeignKey(BuildRequest) + req = models.ForeignKey(BuildRequest, on_delete=models.CASCADE) errtype = models.CharField(max_length=100) errmsg = models.TextField() traceback = models.TextField() |