diff options
Diffstat (limited to 'bitbake/lib/toaster/bldcontrol')
10 files changed, 965 insertions, 290 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index 6812ae3e6e..42675d3fc6 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -81,14 +81,6 @@ def getBuildEnvironmentController(**kwargs): raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) -def _getgitcheckoutdirectoryname(url): - """ Utility that returns the last component of a git path as directory - """ - import re - components = re.split(r'[:\.\/]', url) - return components[-2] if components[-1] == "git" else components[-1] - - class BuildEnvironmentController(object): """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST or SHOULD be supported by a Build Environment. It is used to establish the framework, and must @@ -117,19 +109,43 @@ class BuildEnvironmentController(object): self.be = be self.connection = None + @staticmethod + def _updateBBLayers(bblayerconf, layerlist): + conflines = open(bblayerconf, "r").readlines() + + bblayerconffile = open(bblayerconf, "w") + skip = 0 + for i in xrange(len(conflines)): + if skip > 0: + skip =- 1 + continue + if conflines[i].startswith("# line added by toaster"): + skip = 1 + else: + bblayerconffile.write(conflines[i]) + + bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") + bblayerconffile.close() + + + def writeConfFile(self, variable_list = None, raw = None): + """ Writes a configuration file in the build directory. Override with buildenv-specific implementation. """ + raise Exception("FIXME: Must override to actually write a configuration file") + def startBBServer(self): """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. After this method executes, self.be bbaddress/bbport MUST point to a running and free server, and the bbstate MUST be updated to "started". """ - raise Exception("Must override in order to actually start the BB server") + raise Exception("FIXME: Must override in order to actually start the BB server") def stopBBServer(self): """ Stops the currently running BB server. The bbstate MUST be updated to "stopped". self.connection must be none. """ + raise Exception("FIXME: Must override stoBBServer") def setLayers(self, bbs, ls): """ Checks-out bitbake executor and layers from git repositories. @@ -139,7 +155,7 @@ class BuildEnvironmentController(object): a word of attention: by convention, the first layer for any build will be poky! """ - raise Exception("Must override setLayers") + raise Exception("FIXME: Must override setLayers") def getBBController(self): diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index fe7fd81fb9..1bd51437a6 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -30,7 +30,12 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException + +import logging +logger = logging.getLogger("toaster") + +from pprint import pprint, pformat class LocalhostBEController(BuildEnvironmentController): """ Implementation of the BuildEnvironmentController for the localhost; @@ -49,15 +54,19 @@ class LocalhostBEController(BuildEnvironmentController): if cwd is None: cwd = self.be.sourcedir + #logger.debug("lbc_shellcmmd: (%s) %s" % (cwd, command)) p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out,err) = p.communicate() + p.wait() if p.returncode: if len(err) == 0: err = "command: %s \n%s" % (command, out) else: err = "command: %s \n%s" % (command, err) + #logger.warn("localhostbecontroller: shellcmd error %s" % err) raise ShellCmdException(err) else: + #logger.debug("localhostbecontroller: shellcmd success") return out def _createdirpath(self, path): @@ -73,29 +82,118 @@ class LocalhostBEController(BuildEnvironmentController): assert self.pokydirname and os.path.exists(self.pokydirname) self._createdirpath(self.be.builddir) self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) + # delete the templateconf.cfg; it may come from an unsupported layer configuration + os.remove(os.path.join(self.be.builddir, "conf/templateconf.cfg")) + + + def writeConfFile(self, file_name, variable_list = None, raw = None): + filepath = os.path.join(self.be.builddir, file_name) + with open(filepath, "w") as conffile: + if variable_list is not None: + for i in variable_list: + conffile.write("%s=\"%s\"\n" % (i.name, i.value)) + if raw is not None: + conffile.write(raw) + def startBBServer(self): assert self.pokydirname and os.path.exists(self.pokydirname) assert self.islayerset - print("DEBUG: executing ", "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) - # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected - # but since they start async without any return, we just wait a bit - print "Started server" + + # find our own toasterui listener/bitbake + from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path + + own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake")) + + assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake) + + logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake) + + toaster_ui_log_filepath = os.path.join(self.be.builddir, "toaster_ui.log") + # get the file length; we need to detect the _last_ start of the toaster UI, not the first + toaster_ui_log_filelength = 0 + if os.path.exists(toaster_ui_log_filepath): + with open(toaster_ui_log_filepath, "r") as f: + f.seek(0, 2) # jump to the end + toaster_ui_log_filelength = f.tell() + + cmd = "bash -c \"source %s/oe-init-build-env %s 2>&1 >toaster_server.log && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 2>&1 >toaster_server.log && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir, + self.dburl, self.be.builddir, own_bitbake) + port = "-1" + logger.debug("localhostbecontroller: starting builder \n%s\n" % cmd) + cmdoutput = self._shellcmd(cmd) + for i in cmdoutput.split("\n"): + if i.startswith("Bitbake server address"): + port = i.split(" ")[-1] + logger.debug("localhostbecontroller: Found bitbake server port %s" % port) + + def _toaster_ui_started(filepath, filepos = 0): + if not os.path.exists(filepath): + return False + with open(filepath, "r") as f: + f.seek(filepos) + for line in f: + if line.startswith("NOTE: ToasterUI waiting for events"): + return True + return False + + retries = 0 + started = False + while not started and retries < 10: + started = _toaster_ui_started(toaster_ui_log_filepath, toaster_ui_log_filelength) + import time + logger.debug("localhostbecontroller: Waiting bitbake server to start") + time.sleep(0.5) + retries += 1 + + if not started: + toaster_server_log = open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() + raise BuildSetupException("localhostbecontroller: Bitbake server did not start in 5 seconds, aborting (Error: '%s' '%s')" % (cmdoutput, toaster_server_log)) + + logger.debug("localhostbecontroller: Started bitbake server") + + while port == "-1": + # the port specification is "autodetect"; read the bitbake.lock file + with open("%s/bitbake.lock" % self.be.builddir, "r") as f: + for line in f.readlines(): + if ":" in line: + port = line.split(":")[1].strip() + logger.debug("localhostbecontroller: Autodetected bitbake port %s", port) + break + assert self.be.sourcedir and os.path.exists(self.be.builddir) self.be.bbaddress = "localhost" - self.be.bbport = "8200" + self.be.bbport = port self.be.bbstate = BuildEnvironment.SERVER_STARTED self.be.save() def stopBBServer(self): assert self.pokydirname and os.path.exists(self.pokydirname) assert self.islayerset - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % + self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) self.be.bbstate = BuildEnvironment.SERVER_STOPPED self.be.save() - print "Stopped server" + logger.debug("localhostbecontroller: Stopped bitbake server") + + def getGitCloneDirectory(self, url, branch): + """ Utility that returns the last component of a git path as directory + """ + import re + components = re.split(r'[:\.\/]', url) + base = components[-2] if components[-1] == "git" else components[-1] + + if branch != "HEAD": + return "_%s_%s.toaster_cloned" % (base, branch) + + + # word of attention; this is a localhost-specific issue; only on the localhost we expect to have "HEAD" releases + # which _ALWAYS_ means the current poky checkout + from os.path import dirname as DN + local_checkout_path = DN(DN(DN(DN(DN(os.path.abspath(__file__)))))) + #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) + return local_checkout_path + def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ @@ -104,86 +202,109 @@ class LocalhostBEController(BuildEnvironmentController): assert len(bitbakes) == 1 # set layers in the layersource - # 1. get a list of repos, and map dirpaths for each layer + # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - + + gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] + gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) + for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") + if not (layer.giturl, layer.commit) in gitrepos: + gitrepos[(layer.giturl, layer.commit)] = [] + gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + + + logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) + + + # 2. find checked-out git repos in the sourcedir directory that may help faster cloning + cached_layers = {} + for ldir in os.listdir(self.be.sourcedir): + fldir = os.path.join(self.be.sourcedir, ldir) + if os.path.isdir(fldir): + try: + for line in self._shellcmd("git remote -v", fldir).split("\n"): + try: + remote = line.split("\t")[1].split(" ")[0] + if remote not in cached_layers: + cached_layers[remote] = fldir + except IndexError: + pass + except ShellCmdException: + # ignore any errors in collecting git remotes + pass layerlist = [] - # 2. checkout the repositories - for giturl in gitrepos.keys(): - localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) - print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname + + # 3. checkout the repositories + for giturl, commit in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) + logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): - if not giturl in self._shellcmd("git remote -v", localdirname): - raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) + localremotes = self._shellcmd("git remote -v", localdirname) + if not giturl in localremotes: + raise BuildSetupException("Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something." % (localdirname, ", ".join(localremotes.split("\n")), giturl)) else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] + if giturl in cached_layers: + logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) + self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) + self._shellcmd("git remote remove origin", localdirname) + self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) + else: + logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) + self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": - print "DEBUG: checking out commit ", commit, "to", localdirname - self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) + logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname)) + self._shellcmd("git fetch --all && git checkout \"%s\" && git rebase \"origin/%s\"" % (commit, commit) , localdirname) # take the localdirname as poky dir if we can find the oe-init-build-env if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): - print "DEBUG: selected poky dir name", localdirname + logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname) self.pokydirname = localdirname + # make sure we have a working bitbake + if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): + logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname) + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) + # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: + for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) + logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": - layerlist.append(localdirpath) + layerlist.append(localdirpath.rstrip("/")) - print "DEBUG: current layer list ", layerlist + logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) - # 3. configure the build environment, so we have a conf/bblayers.conf + # 4. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() - # 4. update the bblayers.conf + # 5. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - conflines = open(bblayerconf, "r").readlines() - - bblayerconffile = open(bblayerconf, "w") - for i in xrange(len(conflines)): - if conflines[i].startswith("# line added by toaster"): - i += 2 - else: - bblayerconffile.write(conflines[i]) - - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() + BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True + def readServerLogFile(self): + return open(os.path.join(self.be.builddir, "toaster_server.log"), "r").read() + def release(self): assert self.be.sourcedir and os.path.exists(self.be.builddir) import shutil diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py index 4f6a66e711..1ff5c92833 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py @@ -1,9 +1,8 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction -from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version -from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException -from bldcontrol.models import BuildRequest, BuildEnvironment +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from orm.models import ToasterSetting import os def DN(path): @@ -17,16 +16,6 @@ class Command(NoArgsCommand): args = "" help = "Verifies that the configured settings are valid and usable, or prompts the user to fix the settings." - def _reduce_canon_path(self, path): - components = [] - for c in path.split("/"): - if c == "..": - del components[-1] - elif c == ".": - pass - else: - components.append(c) - return "/".join(components) def _find_first_path_for_file(self, startdirectory, filename, level = 0): if level < 0: @@ -45,6 +34,22 @@ class Command(NoArgsCommand): return ret return None + def _recursive_list_directories(self, startdirectory, level = 0): + if level < 0: + return [] + dirs = [] + try: + for i in os.listdir(startdirectory): + j = os.path.join(startdirectory, i) + if os.path.isdir(j): + dirs.append(j) + except OSError: + pass + for j in dirs: + dirs = dirs + self._recursive_list_directories(j, level - 1) + return dirs + + def _get_suggested_sourcedir(self, be): if be.betype != BuildEnvironment.TYPE_LOCAL: return "" @@ -53,151 +58,140 @@ class Command(NoArgsCommand): def _get_suggested_builddir(self, be): if be.betype != BuildEnvironment.TYPE_LOCAL: return "" - return DN(self._find_first_path_for_file(self.guesspath, "bblayers.conf", 3)) - - def _import_layer_config(self, baselayerdir): - filepath = os.path.join(baselayerdir, "meta/conf/toasterconf.json") - if not os.path.exists(filepath) or not os.path.isfile(filepath): - raise Exception("Failed to find toaster config file %s ." % filepath) - - import json, pprint - data = json.loads(open(filepath, "r").read()) - - # verify config file validity before updating settings - for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: - assert i in data - - # import bitbake data - for bvi in data['bitbake']: - bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) - bvo.giturl = bvi['giturl'] - bvo.branch = bvi['branch'] - bvo.dirpath = bvi['dirpath'] - bvo.save() - - # set the layer sources - for lsi in data['layersources']: - assert 'sourcetype' in lsi - assert 'apiurl' in lsi - assert 'name' in lsi - assert 'branches' in lsi - - if lsi['sourcetype'] == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): - apiurl = lsi['apiurl'] - else: - apiurl = self._reduce_canon_path(os.path.join(DN(filepath), lsi['apiurl'])) - - try: - ls = LayerSource.objects.get(sourcetype = lsi['sourcetype'], apiurl = apiurl) - except LayerSource.DoesNotExist: - ls = LayerSource.objects.create( - name = lsi['name'], - sourcetype = lsi['sourcetype'], - apiurl = apiurl - ) - - layerbranches = [] - for branchname in lsi['branches']: - bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) - layerbranches.append(bo) - - if 'layers' in lsi: - for layerinfo in lsi['layers']: - lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) - if layerinfo['local_path'].startswith("/"): - lo.local_path = layerinfo['local_path'] - else: - lo.local_path = self._reduce_canon_path(os.path.join(DN(filepath), layerinfo['local_path'])) - lo.layer_index_url = layerinfo['layer_index_url'] - if 'vcs_url' in layerinfo: - lo.vcs_url = layerinfo['vcs_url'] - lo.save() - - for branch in layerbranches: - lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, - up_branch = branch, - commit = branch.name, - layer = lo) - lvo.dirpath = layerinfo['dirpath'] - lvo.save() - # set releases - for ri in data['releases']: - bvo = BitbakeVersion.objects.get(name = ri['bitbake']) - assert bvo is not None - - ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo) - ro.description = ri['description'] - ro.branch = ri['branch'] - ro.save() - - for dli in ri['defaultlayers']: - lsi, layername = dli.split(":") - layer, created = Layer.objects.get_or_create( - layer_source = LayerSource.objects.get(name = lsi), - name = layername - ) - ReleaseDefaultLayer.objects.get_or_create( release = ro, layer = layer) - - # set default release - if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: - ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) - else: - ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) - - # set default config variables - for configname in data['config']: - if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: - ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) - else: - ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) + return DN(self._find_first_path_for_file(DN(self.guesspath), "bblayers.conf", 4)) + def handle(self, **options): + # verify that we have a settings for downloading artifacts + while ToasterSetting.objects.filter(name="ARTIFACTS_STORAGE_DIR").count() == 0: + guessedpath = os.getcwd() + "/toaster_build_artifacts/" + print("Toaster needs to know in which directory it can download build log files and other artifacts.\n Toaster suggests \"%s\"." % guessedpath) + artifacts_storage_dir = raw_input(" Press Enter to select \"%s\" or type the full path to a different directory: " % guessedpath) + if len(artifacts_storage_dir) == 0: + artifacts_storage_dir = guessedpath + if len(artifacts_storage_dir) > 0 and artifacts_storage_dir.startswith("/"): + try: + os.makedirs(artifacts_storage_dir) + except OSError as ose: + if "File exists" in str(ose): + pass + else: + raise ose + ToasterSetting.objects.create(name="ARTIFACTS_STORAGE_DIR", value=artifacts_storage_dir) + self.guesspath = DN(DN(DN(DN(DN(DN(DN(__file__))))))) + # refuse to start if we have no build environments + while BuildEnvironment.objects.count() == 0: + print(" !! No build environments found. Toaster needs at least one build environment in order to be able to run builds.\n" + + "You can manually define build environments in the database table bldcontrol_buildenvironment.\n" + + "Or Toaster can define a simple localhost-based build environment for you.") + + i = raw_input(" -- Do you want to create a basic localhost build environment ? (Y/n) "); + if not len(i) or i.startswith("y") or i.startswith("Y"): + BuildEnvironment.objects.create(pk = 1, betype = 0) + else: + raise Exception("Toaster cannot start without build environments. Aborting.") + # we make sure we have builddir and sourcedir for all defined build envionments for be in BuildEnvironment.objects.all(): + be.needs_import = False def _verify_be(): is_changed = False - print("Verifying the Build Environment type %s id %d." % (be.get_betype_display(), be.pk)) - if len(be.sourcedir) == 0: + print("Verifying the Build Environment. If the local Build Environment is not properly configured, you will be asked to configure it.") + + def _update_sourcedir(): suggesteddir = self._get_suggested_sourcedir(be) - be.sourcedir = raw_input(" -- Layer sources checkout directory may not be empty [guessed \"%s\"]:" % suggesteddir) + if len(suggesteddir) > 0: + be.sourcedir = raw_input("Toaster needs to know in which directory it should check out the layers that will be needed for your builds.\n Toaster suggests \"%s\". If you select this directory, a layer like \"meta-intel\" will end up in \"%s/meta-intel\".\n Press Enter to select \"%s\" or type the full path to a different directory (must be a parent of current checkout directory): " % (suggesteddir, suggesteddir, suggesteddir)) + else: + be.sourcedir = raw_input("Toaster needs to know in which directory it should check out the layers that will be needed for your builds. Type the full path to the directory (for example: \"%s\": " % os.environ.get('HOME', '/tmp/')) if len(be.sourcedir) == 0 and len(suggesteddir) > 0: be.sourcedir = suggesteddir - is_changed = True + return True + + if len(be.sourcedir) == 0: + print "\n -- Validation: The checkout directory must be set." + is_changed = _update_sourcedir() if not be.sourcedir.startswith("/"): - be.sourcedir = raw_input(" -- Layer sources checkout directory must be an absolute path:") - is_changed = True + print "\n -- Validation: The checkout directory must be set to an absolute path." + is_changed = _update_sourcedir() - if len(be.builddir) == 0: + if not be.sourcedir in DN(__file__): + print "\n -- Validation: The checkout directory must be a parent of the current checkout." + is_changed = _update_sourcedir() + + if is_changed: + if be.betype == BuildEnvironment.TYPE_LOCAL: + be.needs_import = True + return True + + def _update_builddir(): suggesteddir = self._get_suggested_builddir(be) - be.builddir = raw_input(" -- Build directory may not be empty [guessed \"%s\"]:" % suggesteddir) + if len(suggesteddir) > 0: + be.builddir = raw_input("Toaster needs to know where it your build directory is located.\n The build directory is where all the artifacts created by your builds will be stored. Toaster suggests \"%s\".\n Press Enter to select \"%s\" or type the full path to a different directory: " % (suggesteddir, suggesteddir)) + else: + be.builddir = raw_input("Toaster needs to know where is your build directory.\n The build directory is where all the artifacts created by your builds will be stored. Type the full path to the directory (for example: \" %s/build\")" % os.environ.get('HOME','/tmp/')) if len(be.builddir) == 0 and len(suggesteddir) > 0: be.builddir = suggesteddir - is_changed = True + return True - if not be.builddir.startswith("/"): - be.builddir = raw_input(" -- Build directory must be an absolute path:") - is_changed = True + if len(be.builddir) == 0: + print "\n -- Validation: The build directory must be set." + is_changed = _update_builddir() + if not be.builddir.startswith("/"): + print "\n -- Validation: The build directory must to be set to an absolute path." + is_changed = _update_builddir() if is_changed: print "Build configuration saved" be.save() + return True + + + if be.needs_import: + print "\nToaster can use a SINGLE predefined configuration file to set up default project settings and layer information sources.\n" + + # find configuration files + config_files = [] + for dirname in self._recursive_list_directories(be.sourcedir,2): + if os.path.exists(os.path.join(dirname, ".templateconf")): + import subprocess + conffilepath, error = subprocess.Popen('bash -c ". '+os.path.join(dirname, ".templateconf")+'; echo \"\$TEMPLATECONF\""', shell=True, stdout=subprocess.PIPE).communicate() + conffilepath = os.path.join(conffilepath.strip(), "toasterconf.json") + candidatefilepath = os.path.join(dirname, conffilepath) + if os.path.exists(candidatefilepath): + config_files.append(candidatefilepath) + + if len(config_files) > 0: + print " Toaster will list now the configuration files that it found. Select the number to use the desired configuration file." + for cf in config_files: + print " [%d] - %s" % (config_files.index(cf) + 1, cf) + print "\n [0] - Exit without importing any file" + try: + i = raw_input("\n Enter your option: ") + if len(i) and (int(i) - 1 >= 0 and int(i) - 1 < len(config_files)): + print "Importing file: %s" % config_files[int(i)-1] + from loadconf import Command as LoadConfigCommand + + LoadConfigCommand()._import_layer_config(config_files[int(i)-1]) + # we run lsupdates after config update + print "Layer configuration imported. Updating information from the layer sources, please wait.\n You can re-update any time later by running bitbake/lib/toaster/manage.py lsupdates" + from django.core.management import call_command + call_command("lsupdates") + + # we don't look for any other config files + return is_changed + except Exception as e: + print "Failure while trying to import the toaster config file: %s" % e + else: + print "\n Toaster could not find a configuration file. You need to configure Toaster manually using the web interface, or create a configuration file and use\n bitbake/lib/toaster/managepy.py loadconf [filename]\n command to load it. You can use https://wiki.yoctoproject.org/wiki/File:Toasterconf.json.txt.patch as a starting point." + + - if is_changed and be.betype == BuildEnvironment.TYPE_LOCAL: - baselayerdir = DN(DN(self._find_first_path_for_file(be.sourcedir, "toasterconf.json", 3))) - if baselayerdir: - i = raw_input(" -- Do you want to import basic layer configuration from \"%s\" ? (y/N):" % baselayerdir) - if len(i) and i.upper()[0] == 'Y': - self._import_layer_config(baselayerdir) - # we run lsupdates after config update - print "Updating information from the layer source, please wait." - from django.core.management import call_command - call_command("lsupdates") - pass return is_changed @@ -209,4 +203,12 @@ class Command(NoArgsCommand): ToasterSetting.objects.filter(name = 'DEFAULT_RELEASE').delete() ToasterSetting.objects.get_or_create(name = 'DEFAULT_RELEASE', value = '') + # we are just starting up. we must not have any builds in progress, or build environments taken + for b in BuildRequest.objects.filter(state = BuildRequest.REQ_INPROGRESS): + BRError.objects.create(req = b, errtype = "toaster", errmsg = "Toaster found this build IN PROGRESS while Toaster started up. This is an inconsistent state, and the build was marked as failed") + + BuildRequest.objects.filter(state = BuildRequest.REQ_INPROGRESS).update(state = BuildRequest.REQ_FAILED) + + BuildEnvironment.objects.update(lock = BuildEnvironment.LOCK_FREE) + return 0 diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py new file mode 100644 index 0000000000..9163e9bf11 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py @@ -0,0 +1,174 @@ +from django.core.management.base import BaseCommand, CommandError +from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version +from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority +import os + +from checksettings import DN + +def _reduce_canon_path(path): + components = [] + for c in path.split("/"): + if c == "..": + del components[-1] + elif c == ".": + pass + else: + components.append(c) + if len(components) < 2: + components.append('') + return "/".join(components) + +def _get_id_for_sourcetype(s): + for i in LayerSource.SOURCE_TYPE: + if s == i[1]: + return i[0] + raise Exception("Could not find definition for sourcetype '%s'. Valid source types are %s" % (str(s), ', '.join(map(lambda x: "'%s'" % x[1], LayerSource.SOURCE_TYPE )))) + +class Command(BaseCommand): + help = "Loads a toasterconf.json file in the database" + args = "filepath" + + + + def _import_layer_config(self, filepath): + if not os.path.exists(filepath) or not os.path.isfile(filepath): + raise Exception("Failed to find toaster config file %s ." % filepath) + + import json, pprint + data = json.loads(open(filepath, "r").read()) + + # verify config file validity before updating settings + for i in ['bitbake', 'releases', 'defaultrelease', 'config', 'layersources']: + assert i in data + + def _read_git_url_from_local_repository(address): + url = None + # we detect the remote name at runtime + import subprocess + (remote, remote_name) = address.split(":", 1) + cmd = subprocess.Popen("git remote -v", shell=True, cwd = os.path.dirname(filepath), stdout=subprocess.PIPE, stderr = subprocess.PIPE) + (out,err) = cmd.communicate() + if cmd.returncode != 0: + raise Exception("Error while importing layer vcs_url: git error: %s" % err) + for line in out.split("\n"): + try: + (name, path) = line.split("\t", 1) + if name == remote_name: + url = path.split(" ")[0] + break + except ValueError: + pass + if url == None: + raise Exception("Error while looking for remote \"%s\" in \"%s\"" % (remote_name, out)) + return url + + + # import bitbake data + for bvi in data['bitbake']: + bvo, created = BitbakeVersion.objects.get_or_create(name=bvi['name']) + bvo.giturl = bvi['giturl'] + if bvi['giturl'].startswith("remote:"): + bvo.giturl = _read_git_url_from_local_repository(bvi['giturl']) + bvo.branch = bvi['branch'] + bvo.dirpath = bvi['dirpath'] + bvo.save() + + # set the layer sources + for lsi in data['layersources']: + assert 'sourcetype' in lsi + assert 'apiurl' in lsi + assert 'name' in lsi + assert 'branches' in lsi + + + if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"): + apiurl = lsi['apiurl'] + else: + apiurl = _reduce_canon_path(os.path.join(DN(os.path.abspath(filepath)), lsi['apiurl'])) + + assert ((_get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX) or apiurl.startswith("/")), (lsi['sourcetype'],apiurl) + + try: + ls = LayerSource.objects.get(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl) + except LayerSource.DoesNotExist: + ls = LayerSource.objects.create( + name = lsi['name'], + sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), + apiurl = apiurl + ) + + layerbranches = [] + for branchname in lsi['branches']: + bo, created = Branch.objects.get_or_create(layer_source = ls, name = branchname) + layerbranches.append(bo) + + if 'layers' in lsi: + for layerinfo in lsi['layers']: + lo, created = Layer.objects.get_or_create(layer_source = ls, name = layerinfo['name']) + if layerinfo['local_path'].startswith("/"): + lo.local_path = layerinfo['local_path'] + else: + lo.local_path = _reduce_canon_path(os.path.join(ls.apiurl, layerinfo['local_path'])) + + if not os.path.exists(lo.local_path): + raise Exception("Local layer path %s must exists." % lo.local_path) + + lo.vcs_url = layerinfo['vcs_url'] + if layerinfo['vcs_url'].startswith("remote:"): + lo.vcs_url = _read_git_url_from_local_repository(layerinfo['vcs_url']) + else: + lo.vcs_url = layerinfo['vcs_url'] + + if 'layer_index_url' in layerinfo: + lo.layer_index_url = layerinfo['layer_index_url'] + lo.save() + + for branch in layerbranches: + lvo, created = Layer_Version.objects.get_or_create(layer_source = ls, + up_branch = branch, + commit = branch.name, + layer = lo) + lvo.dirpath = layerinfo['dirpath'] + lvo.save() + # set releases + for ri in data['releases']: + bvo = BitbakeVersion.objects.get(name = ri['bitbake']) + assert bvo is not None + + ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch']) + ro.description = ri['description'] + ro.helptext = ri['helptext'] + ro.save() + + # save layer source priority for release + for ls_name in ri['layersourcepriority'].keys(): + rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name)) + rlspo.priority = ri['layersourcepriority'][ls_name] + rlspo.save() + + for dli in ri['defaultlayers']: + # find layers with the same name + ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli) + + # set default release + if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: + ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").update(value = data['defaultrelease']) + else: + ToasterSetting.objects.create(name = "DEFAULT_RELEASE", value = data['defaultrelease']) + + # set default config variables + for configname in data['config']: + if ToasterSetting.objects.filter(name = "DEFCONF_" + configname).count() > 0: + ToasterSetting.objects.filter(name = "DEFCONF_" + configname).update(value = data['config'][configname]) + else: + ToasterSetting.objects.create(name = "DEFCONF_" + configname, value = data['config'][configname]) + + + def handle(self, *args, **options): + if len(args) == 0: + raise CommandError("Need a path to the toasterconf.json file") + filepath = args[0] + self._import_layer_config(filepath) + + + diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 8efe8e62bc..808318f14f 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -1,9 +1,12 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction -from orm.models import Build +from orm.models import Build, ToasterSetting from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException -from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable import os +import logging + +logger = logging.getLogger("toaster") class Command(NoArgsCommand): args = "" @@ -32,6 +35,7 @@ class Command(NoArgsCommand): # select the build environment and the request to build br = self._selectBuildRequest() except IndexError as e: + # logger.debug("runbuilds: No build request") return try: bec = self._selectBuildEnvironment() @@ -39,26 +43,32 @@ class Command(NoArgsCommand): # we could not find a BEC; postpone the BR br.state = BuildRequest.REQ_QUEUED br.save() + logger.debug("runbuilds: No build env") return + logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) + + # write the build identification variable + BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk)) + # let the build request know where it is being executed + br.environment = bec.be + br.save() + # set up the buid environment with the needed layers - print "Build %s, Environment %s" % (br, bec.be) bec.setLayers(br.brbitbake_set.all(), br.brlayer_set.all()) + bec.writeConfFile("conf/toaster-pre.conf", br.brvariable_set.all()) + bec.writeConfFile("conf/toaster.conf", raw = "INHERIT+=\"toaster buildhistory\"") - # get the bb server running + # get the bb server running with the build req id and build env id bbctrl = bec.getBBController() - # let toasterui that this is a managed build - bbctrl.setVariable("TOASTER_BRBE", "%d:%d" % (br.pk, bec.be.pk)) - - # set the build configuration - for variable in br.brvariable_set.all(): - bbctrl.setVariable(variable.name, variable.value) - # trigger the build command - bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all()))) + task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all())) + if len(task) == 0: + task = None + bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task) - print "Build launched, exiting" + logger.debug("runbuilds: Build launched, exiting. Follow build logs at %s/toaster_ui.log" % bec.be.builddir) # disconnect from the server bbctrl.disconnect() @@ -66,17 +76,50 @@ class Command(NoArgsCommand): except Exception as e: - print " EE Error executing shell command\n", e + logger.error("runbuilds: Error executing shell command %s" % e) traceback.print_exc(e) + if "[Errno 111] Connection refused" in str(e): + # Connection refused, read toaster_server.out + errmsg = bec.readServerLogFile() + else: + errmsg = str(e) + BRError.objects.create(req = br, - errtype = str(type(e)), - errmsg = str(e), - traceback = traceback.format_exc(e)) + errtype = str(type(e)), + errmsg = errmsg, + traceback = traceback.format_exc(e)) br.state = BuildRequest.REQ_FAILED br.save() bec.be.lock = BuildEnvironment.LOCK_FREE bec.be.save() + def archive(self): + ''' archives data from the builds ''' + artifact_storage_dir = ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value + for br in BuildRequest.objects.filter(state = BuildRequest.REQ_ARCHIVE): + # save cooker log + if br.build == None: + br.state = BuildRequest.REQ_FAILED + br.save() + continue + build_artifact_storage_dir = os.path.join(artifact_storage_dir, "%d" % br.build.pk) + try: + os.makedirs(build_artifact_storage_dir) + except OSError as ose: + if "File exists" in str(ose): + pass + else: + raise ose + + file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt") + try: + with open(file_name, "w") as f: + f.write(br.environment.get_artifact(br.build.cooker_log_path).read()) + except IOError: + os.unlink(file_name) + + br.state = BuildRequest.REQ_COMPLETED + br.save() def cleanup(self): from django.utils import timezone @@ -87,4 +130,5 @@ class Command(NoArgsCommand): def handle_noargs(self, **options): self.cleanup() + self.archive() self.schedule() diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0007_auto__add_field_buildrequest_environment__chg_field_buildrequest_build.py b/bitbake/lib/toaster/bldcontrol/migrations/0007_auto__add_field_buildrequest_environment__chg_field_buildrequest_build.py new file mode 100644 index 0000000000..70677a294e --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/migrations/0007_auto__add_field_buildrequest_environment__chg_field_buildrequest_build.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'BuildRequest.environment' + db.add_column(u'bldcontrol_buildrequest', 'environment', + self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildEnvironment'], null=True), + keep_default=False) + + + # Changing field 'BuildRequest.build' + db.alter_column(u'bldcontrol_buildrequest', 'build_id', self.gf('django.db.models.fields.related.OneToOneField')(to=orm['orm.Build'], unique=True, null=True)) + # Adding unique constraint on 'BuildRequest', fields ['build'] + db.create_unique(u'bldcontrol_buildrequest', ['build_id']) + + + def backwards(self, orm): + # Removing unique constraint on 'BuildRequest', fields ['build'] + db.delete_unique(u'bldcontrol_buildrequest', ['build_id']) + + # Deleting field 'BuildRequest.environment' + db.delete_column(u'bldcontrol_buildrequest', 'environment_id') + + + # Changing field 'BuildRequest.build' + db.alter_column(u'bldcontrol_buildrequest', 'build_id', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['orm.Build'], null=True)) + + models = { + u'bldcontrol.brbitbake': { + 'Meta': {'object_name': 'BRBitbake'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']", 'unique': 'True'}) + }, + u'bldcontrol.brerror': { + 'Meta': {'object_name': 'BRError'}, + 'errmsg': ('django.db.models.fields.TextField', [], {}), + 'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'traceback': ('django.db.models.fields.TextField', [], {}) + }, + u'bldcontrol.brlayer': { + 'Meta': {'object_name': 'BRLayer'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}) + }, + u'bldcontrol.brtarget': { + 'Meta': {'object_name': 'BRTarget'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}) + }, + u'bldcontrol.brvariable': { + 'Meta': {'object_name': 'BRVariable'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'value': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + u'bldcontrol.buildenvironment': { + 'Meta': {'object_name': 'BuildEnvironment'}, + 'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}), + 'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}), + 'betype': ('django.db.models.fields.IntegerField', [], {}), + 'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'bldcontrol.buildrequest': { + 'Meta': {'object_name': 'BuildRequest'}, + 'build': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['orm.Build']", 'unique': 'True', 'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'environment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildEnvironment']", 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}), + 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'orm.bitbakeversion': { + 'Meta': {'object_name': 'BitbakeVersion'}, + 'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'giturl': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + }, + u'orm.build': { + 'Meta': {'object_name': 'Build'}, + 'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'completed_on': ('django.db.models.fields.DateTimeField', [], {}), + 'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}), + 'started_on': ('django.db.models.fields.DateTimeField', [], {}), + 'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'orm.project': { + 'Meta': {'object_name': 'Project'}, + 'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Release']"}), + 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + }, + u'orm.release': { + 'Meta': {'object_name': 'Release'}, + 'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}), + 'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + } + } + + complete_apps = ['bldcontrol']
\ No newline at end of file diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0008_brarchive.py b/bitbake/lib/toaster/bldcontrol/migrations/0008_brarchive.py new file mode 100644 index 0000000000..f5469607f3 --- /dev/null +++ b/bitbake/lib/toaster/bldcontrol/migrations/0008_brarchive.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import DataMigration +from django.db import models + +class Migration(DataMigration): + # ids that cannot be imported from BuildRequest + + def forwards(self, orm): + REQ_COMPLETED = 3 + REQ_ARCHIVE = 6 + "Write your forwards methods here." + # Note: Don't use "from appname.models import ModelName". + # Use orm.ModelName to refer to models in this application, + # and orm['appname.ModelName'] for models in other applications. + orm.BuildRequest.objects.filter(state=REQ_COMPLETED).update(state=REQ_ARCHIVE) + + def backwards(self, orm): + REQ_COMPLETED = 3 + REQ_ARCHIVE = 6 + "Write your backwards methods here." + orm.BuildRequest.objects.filter(state=REQ_ARCHIVE).update(state=REQ_COMPLETED) + + models = { + u'bldcontrol.brbitbake': { + 'Meta': {'object_name': 'BRBitbake'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']", 'unique': 'True'}) + }, + u'bldcontrol.brerror': { + 'Meta': {'object_name': 'BRError'}, + 'errmsg': ('django.db.models.fields.TextField', [], {}), + 'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'traceback': ('django.db.models.fields.TextField', [], {}) + }, + u'bldcontrol.brlayer': { + 'Meta': {'object_name': 'BRLayer'}, + 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}) + }, + u'bldcontrol.brtarget': { + 'Meta': {'object_name': 'BRTarget'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'}) + }, + u'bldcontrol.brvariable': { + 'Meta': {'object_name': 'BRVariable'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}), + 'value': ('django.db.models.fields.TextField', [], {'blank': 'True'}) + }, + u'bldcontrol.buildenvironment': { + 'Meta': {'object_name': 'BuildEnvironment'}, + 'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}), + 'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}), + 'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}), + 'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}), + 'betype': ('django.db.models.fields.IntegerField', [], {}), + 'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'bldcontrol.buildrequest': { + 'Meta': {'object_name': 'BuildRequest'}, + 'build': ('django.db.models.fields.related.OneToOneField', [], {'to': u"orm['orm.Build']", 'unique': 'True', 'null': 'True'}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'environment': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildEnvironment']", 'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}), + 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}) + }, + u'orm.bitbakeversion': { + 'Meta': {'object_name': 'BitbakeVersion'}, + 'branch': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'giturl': ('django.db.models.fields.URLField', [], {'max_length': '200'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + }, + u'orm.build': { + 'Meta': {'object_name': 'Build'}, + 'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'completed_on': ('django.db.models.fields.DateTimeField', [], {}), + 'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}), + 'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}), + 'started_on': ('django.db.models.fields.DateTimeField', [], {}), + 'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}) + }, + u'orm.project': { + 'Meta': {'object_name': 'Project'}, + 'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}), + 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'release': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Release']"}), + 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}), + 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), + 'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'}) + }, + u'orm.release': { + 'Meta': {'object_name': 'Release'}, + 'bitbake_version': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.BitbakeVersion']"}), + 'branch_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50'}), + 'description': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'helptext': ('django.db.models.fields.TextField', [], {'null': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '32'}) + } + } + + complete_apps = ['bldcontrol'] + symmetrical = True diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py index 4c54a59b1a..02cfaf7086 100644 --- a/bitbake/lib/toaster/bldcontrol/models.py +++ b/bitbake/lib/toaster/bldcontrol/models.py @@ -40,6 +40,50 @@ class BuildEnvironment(models.Model): updated = models.DateTimeField(auto_now = True) + def get_artifact_type(self, path): + if self.betype == BuildEnvironment.TYPE_LOCAL: + try: + import magic + + # fair warning: this is a mess; there are multiple competeing and incompatible + # magic modules floating around, so we try some of the most common combinations + + try: # we try ubuntu's python-magic 5.4 + m = magic.open(magic.MAGIC_MIME_TYPE) + m.load() + return m.file(path) + except AttributeError: + pass + + try: # we try python-magic 0.4.6 + m = magic.Magic(magic.MAGIC_MIME) + return m.from_file(path) + except AttributeError: + pass + + try: # we try pip filemagic 1.6 + m = magic.Magic(flags=magic.MAGIC_MIME_TYPE) + return m.id_filename(path) + except AttributeError: + pass + + return "binary/octet-stream" + except ImportError: + return "binary/octet-stream" + raise Exception("FIXME: artifact type not implemented for build environment type %s" % be.get_betype_display()) + + + def get_artifact(self, path): + if self.betype == BuildEnvironment.TYPE_LOCAL: + return open(path, "r") + raise Exception("FIXME: artifact download not implemented for build environment type %s" % be.get_betype_display()) + + def has_artifact(self, path): + import os + if self.betype == BuildRequest.TYPE_LOCAL: + return os.path.exists(path) + raise Exception("FIXME: has artifact not implemented for build environment type %s" % be.get_betype_display()) + # a BuildRequest is a request that the scheduler will build using a BuildEnvironment # the build request queue is the table itself, ordered by state @@ -49,6 +93,8 @@ class BuildRequest(models.Model): REQ_INPROGRESS = 2 REQ_COMPLETED = 3 REQ_FAILED = 4 + REQ_DELETED = 5 + REQ_ARCHIVE = 6 REQUEST_STATE = ( (REQ_CREATED, "created"), @@ -56,14 +102,28 @@ class BuildRequest(models.Model): (REQ_INPROGRESS, "in progress"), (REQ_COMPLETED, "completed"), (REQ_FAILED, "failed"), + (REQ_DELETED, "deleted"), + (REQ_ARCHIVE, "archive"), ) + search_allowed_fields = ("brtarget__target", "build__project__name") + project = models.ForeignKey(Project) - build = models.ForeignKey(Build, null = True) # TODO: toasterui should set this when Build is created + build = models.OneToOneField(Build, null = True) # TODO: toasterui should set this when Build is created + environment = models.ForeignKey(BuildEnvironment, null = True) state = models.IntegerField(choices = REQUEST_STATE, default = REQ_CREATED) created = models.DateTimeField(auto_now_add = True) updated = models.DateTimeField(auto_now = True) + def get_duration(self): + return (self.updated - self.created).total_seconds() + + def get_sorted_target_list(self): + tgts = self.brtarget_set.order_by( 'target' ); + return( tgts ); + + def get_machine(self): + return self.brvariable_set.get(name="MACHINE").value # These tables specify the settings for running an actual build. # They MUST be kept in sync with the tables in orm.models.Project* diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py index 64674953dc..11ad08d440 100644 --- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py @@ -29,7 +29,7 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException def DN(path): return "/".join(path.split("/")[0:-1]) @@ -77,16 +77,21 @@ class SSHBEController(BuildEnvironmentController): self._pathcreate(self.be.builddir) self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) - def startBBServer(self): + def startBBServer(self, brbe): assert self.pokydirname and self._pathexists(self.pokydirname) assert self.islayerset - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) - # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected - # but since they start async without any return, we just wait a bit - print "Started server" + cmd = self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) + + port = "-1" + for i in cmd.split("\n"): + if i.startswith("Bitbake server address"): + port = i.split(" ")[-1] + print "Found bitbake server port ", port + + assert self.be.sourcedir and self._pathexists(self.be.builddir) self.be.bbaddress = self.be.address.split("@")[-1] - self.be.bbport = "8200" + self.be.bbport = port self.be.bbstate = BuildEnvironment.SERVER_STARTED self.be.save() @@ -99,6 +104,19 @@ class SSHBEController(BuildEnvironmentController): self.be.save() print "Stopped server" + + def _copyFile(self, filepath1, filepath2): + p = subprocess.Popen("scp '%s' '%s'" % (filepath1, filepath2), stdout=subprocess.PIPE, stderr = subprocess.PIPE, shell=True) + (out, err) = p.communicate() + if p.returncode: + raise ShellCmdException(err) + + def pullFile(self, local_filename, remote_filename): + _copyFile(local_filename, "%s:%s" % (self.be.address, remote_filename)) + + def pushFile(self, local_filename, remote_filename): + _copyFile("%s:%s" % (self.be.address, remote_filename), local_filename) + def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ @@ -106,62 +124,8 @@ class SSHBEController(BuildEnvironmentController): assert len(bitbakes) == 1 # set layers in the layersource - # 1. get a list of repos, and map dirpaths for each layer - gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - - for layer in layers: - # we don't process local URLs - if layer.giturl.startswith("file://"): - continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") - - layerlist = [] - - # 2. checkout the repositories - for giturl in gitrepos.keys(): - import os - localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) - print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname - - # make sure our directory is a git repository - if self._pathexists(localdirname): - if not giturl in self._shellcmd("git remote -v", localdirname): - raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) - else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] - - # branch magic name "HEAD" will inhibit checkout - if commit != "HEAD": - print "DEBUG: checking out commit ", commit, "to", localdirname - self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) - - # take the localdirname as poky dir if we can find the oe-init-build-env - if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")): - print "DEBUG: selected poky dir name", localdirname - self.pokydirname = localdirname - - # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: - localdirpath = os.path.join(localdirname, dirpath) - if not self._pathexists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) - - if name != "bitbake": - layerlist.append(localdirpath) - - print "DEBUG: current layer list ", layerlist + raise Exception("Not implemented: SSH setLayers") # 3. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() @@ -171,17 +135,15 @@ class SSHBEController(BuildEnvironmentController): if not self._pathexists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - conflines = open(bblayerconf, "r").readlines() + import uuid + local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf" - bblayerconffile = open(bblayerconf, "w") - for i in xrange(len(conflines)): - if conflines[i].startswith("# line added by toaster"): - i += 2 - else: - bblayerconffile.write(conflines[i]) + self.pullFile(bblayerconf, local_bblayerconf) + + BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist) + self.pushFile(local_bblayerconf, bblayerconf) - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() + os.unlink(local_bblayerconf) self.islayerset = True return True diff --git a/bitbake/lib/toaster/bldcontrol/tests.py b/bitbake/lib/toaster/bldcontrol/tests.py index 4577c3f03b..5a9d1df37a 100644 --- a/bitbake/lib/toaster/bldcontrol/tests.py +++ b/bitbake/lib/toaster/bldcontrol/tests.py @@ -44,7 +44,7 @@ class BEControllerTests(object): # test start server and stop self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied") - bc.startBBServer() + bc.startBBServer("0:0") self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering") bc.stopBBServer() @@ -57,14 +57,8 @@ class BEControllerTests(object): bc = self._getBEController(obe) bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info - bbc = bc.getBBController() + bbc = bc.getBBController("%d:%d" % (-1, obe.pk)) self.assertTrue(isinstance(bbc, BitbakeController)) - # test set variable, use no build marker -1 for BR value - try: - bbc.setVariable("TOASTER_BRBE", "%d:%d" % (-1, obe.pk)) - except Exception as e : - self.fail("setVariable raised %s", e) - bc.stopBBServer() self._serverForceStop(bc) @@ -136,7 +130,7 @@ class RunBuildsCommandTests(TestCase): def test_br_select(self): from orm.models import Project, Release, BitbakeVersion - p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch="HEAD")[0])[0]) + p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch=Branch.objects.get_or_create(name="HEAD"))[0])[0]) obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p) command = Command() br = command._selectBuildRequest() @@ -147,3 +141,22 @@ class RunBuildsCommandTests(TestCase): self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated") # no more selections possible here self.assertRaises(IndexError, command._selectBuildRequest) + + +class UtilityTests(TestCase): + def test_reduce_path(self): + from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype + + self.assertTrue( _reduce_canon_path("/") == "/") + self.assertTrue( _reduce_canon_path("/home/..") == "/") + self.assertTrue( _reduce_canon_path("/home/../ana") == "/ana") + self.assertTrue( _reduce_canon_path("/home/../ana/..") == "/") + self.assertTrue( _reduce_canon_path("/home/ana/mihai/../maria") == "/home/ana/maria") + + def test_get_id_for_sorucetype(self): + from bldcontrol.management.commands.loadconf import _reduce_canon_path, _get_id_for_sourcetype + self.assertTrue( _get_id_for_sourcetype("layerindex") == 1) + self.assertTrue( _get_id_for_sourcetype("local") == 0) + self.assertTrue( _get_id_for_sourcetype("imported") == 2) + with self.assertRaises(Exception): + _get_id_for_sourcetype("unknown") |