from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction from orm.models import Build, ToasterSetting, LogMessage, Target from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable import os import logging logger = logging.getLogger("ToasterScheduler") class Command(NoArgsCommand): args = "" help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)" @transaction.commit_on_success def _selectBuildEnvironment(self): bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE) bec.be.lock = BuildEnvironment.LOCK_LOCK bec.be.save() return bec @transaction.commit_on_success def _selectBuildRequest(self): br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0] br.state = BuildRequest.REQ_INPROGRESS br.save() return br def schedule(self): import traceback try: br = None try: # select the build environment and the request to build br = self._selectBuildRequest() except IndexError as e: #logger.debug("runbuilds: No build request") return try: bec = self._selectBuildEnvironment() except IndexError as e: # we could not find a BEC; postpone the BR br.state = BuildRequest.REQ_QUEUED br.save() logger.debug("runbuilds: No build env") return logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) # write the build identification variable BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk)) # let the build request know where it is being executed br.environment = bec.be br.save() # this triggers an async build bec.triggerBuild(br.brbitbake_set.all(), br.brlayer_set.all(), br.brvariable_set.all(), br.brtarget_set.all()) except Exception as e: logger.error("runbuilds: Error launching build %s" % e) traceback.print_exc(e) if "[Errno 111] Connection refused" in str(e): # Connection refused, read toaster_server.out errmsg = bec.readServerLogFile() else: errmsg = str(e) BRError.objects.create(req = br, errtype = str(type(e)), errmsg = errmsg, traceback = traceback.format_exc(e)) br.state = BuildRequest.REQ_FAILED br.save() bec.be.lock = BuildEnvironment.LOCK_FREE bec.be.save() def archive(self): ''' archives data from the builds ''' artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): # save cooker log if br.build == None: br.state = BuildRequest.REQ_FAILED br.save() continue build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) try: os.makedirs(build_artifact_storage_dir) except OSError as ose: if "File exists" in str(ose): pass else: raise ose file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") try: with open(file_name, "w") as f: f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) except IOError: os.unlink(file_name) br.state = BuildRequest.REQ_COMPLETED br.save() def cleanup(self): from django.utils import timezone from datetime import timedelta # environments locked for more than 30 seconds - they should be unlocked BuildEnvironment.objects.filter(buildrequest__state__in=[BuildRequest.REQ_FAILED, BuildRequest.REQ_COMPLETED]).filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE) # update all Builds that failed to start for br in BuildRequest.objects.filter(state = BuildRequest.REQ_FAILED, build__outcome = Build.IN_PROGRESS): # transpose the launch errors in ToasterExceptions br.build.outcome = Build.FAILED for brerror in br.brerror_set.all(): logger.debug("Saving error %s" % brerror) LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) br.build.save() # we don't have a true build object here; hence, toasterui didn't have a change to release the BE lock br.environment.lock = BuildEnvironment.LOCK_FREE br.environment.save() # update all BuildRequests without a build created for br in BuildRequest.objects.filter(build = None): br.build = Build.objects.create(project = br.project, completed_on = br.updated, started_on = br.created) br.build.outcome = Build.REQ_FAILED try: br.build.machine = br.brvariable_set.get(name='MACHINE').value except BRVariable.DoesNotExist: pass br.save() # transpose target information for brtarget in br.brtarget_set.all(): Target.objects.create(build = br.build, target= brtarget.target) # transpose the launch errors in ToasterExceptions for brerror in br.brerror_set.all(): LogMessage.objects.create(build = br.build, level = LogMessage.EXCEPTION, message = brerror.errmsg) br.build.save() pass def handle_noargs(self, **options): self.cleanup() self.archive() self.schedule()