path: root/lib/python2.7/site-packages/autobuilder/Autobuilder.py

'''
Created on Dec 4, 2012

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "pidge@toganlabs.com"
'''
#!/usr/bin/python
from twisted.python import log
from config import *
from buildbot.schedulers.forcesched import *
from buildbot.schedulers.triggerable import Triggerable
from buildbot.schedulers import timed
from buildbot.schedulers.basic  import SingleBranchScheduler
from buildbot.changes import filter
from buildbot.changes.pb import PBChangeSource
from buildbot.changes.gitpoller import *
from buildbot import util
from lib.ABTools import capitalize
import ast
import BuildSet

class Autobuilder:
    def __init__(self, cfile=None):
        self.yocto_sources = YOCTO_SOURCES
        self.yocto_sched = YOCTO_SCHED
        self.yocto_builders = YOCTO_BUILDERS
        self.yocto_projname = YOCTO_PROJNAME
        self.yocto_projurl = YOCTO_PROJURL
        self.config = None
        import os, config
        if cfile is None:
            try:
                self.cfile = os.environ.get("YOCTO_AB_CONFIG")
            except:
                self.cfile = "./buildset-config/yoctoAB.conf"
        else:
            self.cfile = cfile

    def createBuildsets(self):
        buildersSorted=[]
        sort = True
        if self.config.has_option('BuildSets', 'order'):
            sort = False
            for set in ast.literal_eval(self.config.get('BuildSets', 'order')):
                buildersSorted.append(set)

        for key in self.configdict:
            if key not in buildersSorted and key != "BuildSets":
                buildersSorted.append(key)

        if sort:
            util.naturalSort(buildersSorted)

        for buildset in buildersSorted:
            self.schedprops = []
            self.checkoutprops={}
            self.set_props = {}
            self.repos = []
            self.properties = []
            self.builders = None
            self.schedprops.append(NestedParameter(name='ss_' + buildset, label="<h3><div class='trigger_heading' id='"+buildset+"'>" + buildset + " defaults:</div></h3>", fields=[FixedParameter(name="dummy", default="dummy")]))
            self.parseBuildSet(buildset)
            steps = ast.literal_eval(self.configdict[buildset]['steps'])
            repos = ast.literal_eval(self.configdict[buildset].get('repos', '{}'))
            locals()['buildset_%s' % buildset] = BuildSet.BuildSet(name = buildset,
                                                                steps = steps,
                                                                builders = self.builders,
                                                                layers = repos,
                                                                set_props = self.set_props)

            self.yocto_sched.append(ForceScheduler(
                                name=str(buildset),
                                branch=FixedParameter(name="branch",
                                                      default=""),
                                reason=StringParameter(name="reason",
                                                       label="Reason (please note the reason for triggering the build and any expecations for the builds outcome):<br>",
                                                       required=False,
                                                       size=120),
                                revision=FixedParameter(name="revision",
                                                        default=""),
                                repository=FixedParameter(name="repository",
                                                          default=""),
                                project=FixedParameter(name="repository",
                                                       default=""),
                                builderNames=['%s' % buildset],
                                properties=self.schedprops))
            self.createExtraSchedulers(key=buildset, checkoutprops=self.checkoutprops)

    def parseBuildSet(self, buildset):
        log.msg("Parsing " + buildset)
        if buildset is not "BuildSets":
            self.builders=self.parseBuilders(buildset)
            self.parseProps(buildset)
            self.parseRepos(buildset)
            self.parseSteps(buildset)
            self.set_props.update(self.checkoutprops)

    def parseSteps(self, buildset):
        buildset=buildset
        try:
            for step in ast.literal_eval(self.configdict[buildset]['steps']):
                if step.has_key('TriggerBuilds'):
                    master_trigger_name = step['TriggerBuilds']['schedulerName']
                    for b in ["", "_nowait"]:
                        buildername=[]
                        if 'schedulerNames' + b in step['TriggerBuilds' ]:
                            for scheduler in step['TriggerBuilds']['schedulerNames' + b].iterkeys():
                                if scheduler not in buildername:
                                    buildername.append(scheduler)
                            self.yocto_sched.append(Triggerable(name="trigger_" + master_trigger_name + b, builderNames=buildername))
        except SyntaxError as err:
            raise ABConfigError("%s found in %s" % (err, buildset))

    def parseRepos(self, buildset=None):
        buildset=buildset
        if self.configdict[buildset].has_key('repos'):
            try:
                for layer in ast.literal_eval(self.configdict[buildset]['repos']):
                    if layer.iterkeys().next() not in self.repos:
                        schedpropstoextend, newcheckoutprops = self.CreateLayerSchedulerParams(layer=layer, trigger=buildset, triggerer=True)
                        self.schedprops.extend(schedpropstoextend)
                        self.checkoutprops.update(newcheckoutprops)
                        self.repos.append(layer.iterkeys().next())
            except SyntaxError as err:
                raise ABConfigError("%s found in %s" % (err, buildset))
        return

    def parseProps(self, buildset):
        buildset=buildset
        if self.configdict[buildset].has_key('props'):
            for props in ast.literal_eval(self.configdict[buildset]['props']):
                for prop in dict(props):
                    self.prop_name=""
                    self.prop_prop_type=""
                    self.prop_default=""
                    self.prop_choices=""
                    self.prop_label=""
                    self.prop_required=""
                    self.prop_size=""
                    setattr(self, "prop_name", prop)
                    for k, v in props[prop].iteritems():
                        setattr(self, "prop_"+str(k), v)
                    if self.prop_name not in self.properties:
                        schedpropstoextend = self.CreateExtraSchedulerParams(name=self.prop_name,
                                                                           prop_type=self.prop_prop_type,
                                                                           default=self.prop_default,
                                                                           choices=self.prop_choices,
                                                                           label=self.prop_label,
                                                                           required=self.prop_required,
                                                                           scheduler=buildset,
                                                                           size=self.prop_size)
                        self.schedprops.extend(schedpropstoextend)
                        self.properties.append(self.prop_name)

    def parseBuilders(self, buildset):
        buildset=buildset
        builders=ast.literal_eval(self.configdict[buildset]['builders'])
        return builders

    def parseConfig(self):
        import ConfigParser
        from os import listdir
        from os.path import dirname, isfile, join
        print "LOADING CONFIG FILE"
        self.config = ConfigParser.ConfigParser()
        self.configdir=os.path.dirname(self.cfile)
        self.configfiles = [ join(self.configdir,f) for f in listdir(self.configdir) if isfile(join(self.configdir,f)) and (f != 'autobuilder.conf') and f.endswith(".conf")]
        try:
            self.config.read(self.configfiles)
        except ConfigParser.Error as err:
            print("Failed parsing config files with error: %s" % err)
        self.buildsets=self.config.sections()
        self.configdict = {}
        for section in self.buildsets:
            self.configdict[section]= dict(self.config.items(section))
        return self.configdict

    def GetLayerCommitId(self, layer):
            if 'hash' in layer[layer.iterkeys().next()] :
                layercommitid=layer[layer.iterkeys().next()]['hash']
            elif 'tag' in layer[layer.iterkeys().next()]:
                layercommitid=layer[layer.iterkeys().next()]['tag']
            else:
                layercommitid="HEAD"
            return layercommitid

    def GetLayerBranch(self, layer):
            if 'branch' in layer[layer.iterkeys().next()]:
                layerbranch=layer[layer.iterkeys().next()]['branch']
            else:
                layerbranch="master"
            return layerbranch

    # Create extra schedulers requested for the given buildset using 'scheduler:'
    def createExtraSchedulers(self, key, checkoutprops):
        import ast
        schedulerkey = 'scheduler'
        if self.configdict[key].has_key(schedulerkey):
            for scheds in ast.literal_eval(self.configdict[key][schedulerkey]):
                for name,value in dict(scheds).iteritems():
                    schedtype = None
                    if value.has_key('type'):
                        schedtype = value['type']

                    # For each change based scheduler, create separate change source
                    # for each repo in the build set; this is so that each repo can have
                    # distinct authorisation credential, etc.
                    if schedtype == "SingleBranchScheduler":
                        if not value.has_key('repository'):
                            log.msg("%s has no repository property" % schedtype)
                            continue

                        reponame = value['repository']
                        repos=None
                        repo=None

                        for x in ast.literal_eval(self.configdict[key]['repos']):
                            if dict(x).has_key(reponame):
                                repos=x
                                repo=repos[reponame]
                                break

                        if repo is None:
                            log.msg("No repo %s found" % reponame)
                            continue

                        repo=repos[reponame]
                        if not repo.has_key('repourl'):
                            log.msg("No repourl for %s found" % reponame)
                            continue

                        # default user to the name of the repo
                        user=str(reponame)
                        if value.has_key('change-user'):
                            user=value['change-user']

                        # default password is None, which in the change source becomes 'changepw'
                        passwd=None
                        if value.has_key('change-password'):
                            passwd=value['change-password']

                        branch="master"
                        stabletimer=60
                        if repo.has_key('branch'):
                            branch = repo['branch']
                        if value.has_key('stable-timer'):
                            stabletimer = value['stable-timer']

                        if value.has_key('changesource') and value['changesource'] == 'GitPoller':
                            if value.has_key('interval'):
                                interval = int(value['interval'])
                            else:
                                # default to 5 minutes poll intervals
                                interval = 60*5
                            gitpoller = GitPoller(repourl=repo['repourl'], branch=branch, pollInterval=interval)
                            if gitpoller not in self.yocto_sources:
                                self.yocto_sources.append(gitpoller)
                                log.msg("Adding GitPoller as changesource for " + reponame)
                            else:
                                log.msg("GitPoller already registered as changesource for " + reponame)
                        else:
                            # default to use PBChangeSource as the changesource
                            self.yocto_sources.append(PBChangeSource(user=user, passwd=passwd))
                            log.msg("Adding PBChangeSource as changesource for " + reponame)

                        self.yocto_sched.append(SingleBranchScheduler(name=str(name),
                                                                      builderNames=['%s' % key],
                                                                      properties=checkoutprops,
                                                                      change_filter=filter.ChangeFilter(branch=branch,
                                                                                                        repository=repo['repourl']),
                                                                      treeStableTimer=stabletimer))
                        log.msg("Added %s '%s' for branch '%s' in repository '%s'" % (schedtype,str(name),branch,repo['repourl']))

                    elif not schedtype or schedtype == "Nightly":
                        # Set default values for optional nightly build
                        # parameters.
                        buildmonth="*"
                        builddayOfWeek = "*"
                        if value.has_key('month'):
                            buildmonth = value['month']
                        if value.has_key('dayOfWeek'):
                            builddayOfWeek = value['dayOfWeek']
                        self.yocto_sched.append(timed.Nightly(name=str(name),
                                                              branch='default',
                                                              builderNames=['%s' % key],
                                                              properties=checkoutprops,
                                                              hour=value['hour'],
                                                              minute=value['minute'],
                                                              month=buildmonth,
                                                              dayOfWeek=builddayOfWeek))


    def CreateExtraSchedulerParams(self, name=None,
                                         prop_type=None,
                                         default=None,
                                         choices=None,
                                         label=None,
                                         required=None,
                                         size=None,
                                         scheduler=""):
        supported_prop_types  = ["ChoiceStringParameter",
                         "StringParameter",
                         "FixedParameter",
                         "BooleanParameter"]
        schedprops = []
        prop_type=prop_type
        name=name
        choices=choices
        prop_type=prop_type
        label=label
        required=required
        size=size
        default=default
        scheduler=scheduler
        propid="custom_"+name
        if label is None:
            label=self.name

        if default is None:
            if prop_type == "FixedParameter":
                try:
                    raise ABConfigError("your config file has a fixed type property declaration that has no default")
                except ABConfigError, (instance):
                    log.msg("Caught: " + instance.parameter)
            else:
                default = ""

        if prop_type == "ChoiceStringParameter":
            if choices is None:
                try:
                    raise ABConfigError("your config file has a ChoiceStringParameter declaration that has no choices")
                except ABConfigError, (instance):
                    log.msg("Caught: " + instance.parameter)

        if required is None:
            required=True

        if size is None:
            size=80

        if prop_type is None:
            prop_type = "StringParameter"

        if prop_type=="ChoiceStringParameter":
           schedprops.append(ChoiceStringParameter(name=propid,
                                             label=label,
                                             required=required,
                                             choices=choices))
        elif prop_type=="StringParameter":
           schedprops.append(StringParameter(name=propid,
                                             label=label,
                                             required=required,
                                             default=default,
                                             size=size))
        elif prop_type=="FixedParameter":
           schedprops.append(FixedParameter(name=propid,
                                             label=label,
                                             required=required,
                                             default=default))
        elif prop_type=="BooleanParameter":
           schedprops.append(BooleanParameter(name=propid,
                                             label=label,
                                             required=required,
                                             default=default))
        else:
            try:
                raise ABConfigError("your config file has a prop declaration that is not valid")
            except ABConfigError, (instance):
                log.msg("Caught: " + instance.parameter)
        return schedprops

    def CreateLayerSchedulerParams(self, layer=None, trigger=None, triggerer=None):
            schedprops = []
            set_checkoutprops = {}
            layerrepo=layer[layer.iterkeys().next()]['repourl']
            layerbranch=self.GetLayerBranch(layer)
            layercommitid=self.GetLayerCommitId(layer)
            layername=layer.iterkeys().next()
            set_checkoutprops['repo_' + layername] = layerrepo
            set_checkoutprops['branch_' + layername] =  layerbranch
            set_checkoutprops['commit_' + layername] =  layercommitid
            schedprops.append(StringParameter(name="repo_"+ layername, label="<hr><b>" + layername + "</b> Repository:<br>", required=True, default=layerrepo, size=50))
            schedprops.append(StringParameter(name="branch_" + layername, label="Branch:<br>", required=True, default=layerbranch, size=18))
            schedprops.append(StringParameter(name="commit_" + layername, label="Tag/Commit Hash:<br>", required=True, default=layercommitid, size=20))
            return schedprops, set_checkoutprops

class ABConfigError(Exception):
       def __init__(self, value):
           self.parameter = value
       def __str__(self):
           return repr(self.parameter)