path: root/lib/python2.7/site-packages/autobuilder/buildsteps/CreateAutoConf.py

'''
Created on Dec 26, 2012

__author__ = "Elizabeth 'pidge' Flanagan"
__copyright__ = "Copyright 2013, Intel Corp."
__credits__ = ["Elizabeth Flanagan"]
__license__ = "GPL"
__version__ = "2.0"
__maintainer__ = "Elizabeth Flanagan"
__email__ = "pidge@toganlabs.com"
'''

from buildbot.steps.shell import ShellCommand
import os
from twisted.python import log
import ast, json
from buildbot.status.results import FAILURE

class CreateAutoConf(ShellCommand):
    haltOnFailure = False
    flunkOnFailure = True
    name = "CreateAutoConf"
    def __init__(self, factory, argdict=None, **kwargs):
        self.machine=""
        self.buildapp=""
        self.distro="poky"
        self.packages=None
        self.buildhistory=False
        self.gplv3=True
        self.multilib=False
        self.swabber=False
        self.x32=False
        self.atext=None
        self.emgd=False
        self.pvr=False
        self.atextprepend=None
        self.atextappend=None
        self.SDKMACHINE="i686"
        self.adtdev=False
        self.factory = factory
        self.buildappsrcrev = "${AUTOREV}"
        self.initmgr=None
        self.nosstate=False
        self.kwargs = kwargs
        self.devkernel = True
        self.tmpdir = None
        self.prserv = True
        self.lockedsigs = False
        self.buildinfo = False
        self.autoconfpath = "./build/conf/auto.conf"
        self.sdkconfpath = "./build/conf/sdk-extra.conf"
        self.rmautoconf = "True"
        for k, v in argdict.iteritems():
            if type(v) is bool:
                setattr(self, k, str(v))
            else:
                setattr(self, k, v)
        self.description = "Create Auto Configuration"
        ShellCommand.__init__(self, **kwargs)

    def start(self):
        buildername=self.getProperty("buildername")
        buildername=self.getProperty("devkernel")
        distroversion=self.getProperty("distroversion")
        layerversion = int(self.getProperty("layerversion_yoctobsp", "0"))
        layerversioncore = int(self.getProperty("layerversion_core", "0"))
        branch=self.getProperty("branch")
        sstatemirror = os.environ.get('SSTATE_MIRROR_BASE')
        release = False
        sstate_release_number = ""

        release_me = self.getProperty("custom_release_me", "None")
        if release_me != 'None':
            self.setProperty('is_release', release_me, "CreateAutoConf")

        release = (self.getProperty("is_release") == "True")
        if release:
            sstate_release_number = self.get_release_number()

        if release and sstate_release_number == "":
            self.step_status.setText(["Release build requested without release number."])
            self.finished(FAILURE)
            return None
        elif release and sstate_release_number:
            sstatemirror = "%s/%s" % (sstatemirror, sstate_release_number)

        self.setProperty('DISTRO', self.distro, "Setting DISTRO", "CreateAutoConf")
        self.command = ""
        fout = ""
        if self.rmautoconf == "True":
            self.command += "rm -rf %s;" % self.autoconfpath
            self.command += "rm -rf %s;" % self.sdkconfpath
        #check to see if we have a prepend
        if self.atextprepend:
            fout = fout +"\n"+ self.atextprepend
        #check to see if we have text override
        if self.atext == "#TOASTER":
            fout = fout+"\n"+self.getProperty("custom_atext")
        elif self.atext != "#TOASTER" and self.atext is not None:
            fout = fout+"\n"+self.atext
        else:

            if os.environ.get('ERROR_REPORT_COLLECT', 'False') == "True":
                fout += 'INHERIT += "report-error"\n'
            if self.distro != "oecore":
                fout = fout + 'DISTRO = "' + self.distro + '"\n'
            if self.tmpdir:
                if self.tmpdir.startswith('/'):
                    fout = fout + 'TMPDIR = "' + self.tmpdir + '"\n'
                else:
                    fout = fout + 'TMPDIR = "${TOPDIR}/' + self.tmpdir + '"\n'
            if self.packages:
                self.classes = [ "package_%s" % p for p in self.packages.split() ]
                fout = fout + 'PACKAGE_CLASSES = "' + " ".join(self.classes) + '"\n'
            else:
                fout = fout + 'PACKAGE_CLASSES = "package_rpm package_deb package_ipk"\n'
            repo=self.getProperty("repository")
            if os.environ.get('DEVKERNEL_MUT_REPO') is not None:
                mut_repos=ast.literal_eval(os.environ.get('DEVKERNEL_MUT_REPO').encode('utf-8'))
                if mut_repos is not None and repo in mut_repos.keys():
                    if self.getProperty("branch") in mut_repos[self.getProperty("repository")] and self.devkernel is not "False":
                        fout = fout + 'PREFERRED_PROVIDER_virtual/kernel = "' + os.environ.get('DEVKERNEL') + '"\n'
                    elif self.getProperty("custom_prefered_kernel") is not None:
                        fout = fout + 'PREFERRED_PROVIDER_virtual/kernel = "' + self.getProperty("custom_prefered_kernel") +'"\n'
            fout = fout + 'BB_NUMBER_THREADS = "' + os.environ.get('BB_NUMBER_THREADS') + '"\n'

            # We renamed this variable to prevent it affecting OE
            # configurations where the OE setting of the same name
            # has a weak assignment. In order to be less brittle to
            # changes in the config we try and read the old variable
            # when the new variable isn't set.
            pmakenum = os.environ.get('PARALLEL_MAKE_NUM')
            if not pmakenum:
                pmakenum = os.environ.get('PARALLEL_MAKE')
            fout = fout + 'PARALLEL_MAKE = "-j ' + pmakenum + '"\n'
            fout = fout + 'SDKMACHINE ?= "' + self.SDKMACHINE + '"\n'
            if os.environ.get('IMAGE_FSTYPES'):
                fout = fout + 'IMAGE_FSTYPES_append = " ' + os.environ.get('IMAGE_FSTYPES') +'"\n'
            if os.environ.get('TMP_DIR'):
                fout = fout + 'TMPDIR = "' + os.environ.get('TMP_DIR') + '"\n'
            fout = fout + 'BB_TASK_NICE_LEVEL = "5"\n'
            fout = fout + 'BB_TASK_NICE_LEVEL_task-testimage = "0"\n'
            fout = fout + 'BB_TASK_IONICE_LEVEL = "2.7"\n'
            fout = fout + 'BB_TASK_IONICE_LEVEL_task-testimage = "2.1"\n'
            buildprops = self.getProperties().asDict()
            for k, v in buildprops.iteritems():
                if "buildappsrcrev" in k:
                    self.buildappsrcrev=v[0]
            if self.buildapp == "True":
                if self.buildappsrcrev == "AUTOREV":
                    self.buildappsrcrev="${AUTOREV}"
                fout = fout + 'DL_DIR ?= "${TOPDIR}/downloads"\n'
                fout = fout + 'INHERIT += "own-mirrors"\n'
                fout = fout + 'SOURCE_MIRROR_URL = "file:///' + os.environ.get('DL_DIR') +'"\n'
                if self.buildappsrcrev != "DEFAULT":
                    fout = fout + 'SRCREV_pn-build-appliance-image = "' + self.buildappsrcrev + '"\n'
                    fout = fout + 'PV_append_pn-build-appliance-image = "+git${SRCPV}"\n'
            else:
                fout = fout + 'DL_DIR = "' + os.environ.get('DL_DIR')+'"\n'
            if self.emgd == "True":
                fout = fout + 'LICENSE_FLAGS_WHITELIST += "license_emgd-driver-bin" \n'
            if self.pvr == "True":
                fout = fout + 'LICENSE_FLAGS_WHITELIST += "license_cdv-pvr-driver" \n'
                fout = fout + 'PVR_LICENSE = "yes" \n'
            if self.multilib and self.multilib != "False":
                if self.multilib == "True" or self.multilib == "lib32":
                    fout = fout + 'require conf/multilib.conf \n'
                    fout = fout + 'MULTILIBS = "multilib:lib32" \n'
                    fout = fout + 'DEFAULTTUNE_virtclass-multilib-lib32 = "x86" \n'
                elif self.multilib == "lib64":
                    fout = fout + 'require conf/multilib.conf \n'
                    fout = fout + 'MULTILIBS = "multilib:lib64" \n'
                    fout = fout + 'DEFAULTTUNE_virtclass-multilib-lib64 = "x86-64" \n'
            if not self.nosstate:
                sstatedir = os.environ.get("SSTATE_DIR")
                if release:
                    # Use a separate SSTATE_DIR with the primary
                    # SSTATE_DIR configured as a mirror so that we
                    # have a directory of symlinks to sstate objects
                    # that can be published for the release
                    fout = fout + 'SSTATE_MIRRORS += "file://.* file://%s/PATH"\n' % sstatedir
                    pubdir = os.environ.get("SSTATE_RELEASE_DIR")
                    if pubdir:
                        sstatedir = pubdir + '/' + sstate_release_number
                fout = fout + 'SSTATE_DIR ?= "' + sstatedir + '/" \n'

            if self.gplv3 == "False":
                    fout = fout + 'INCOMPATIBLE_LICENSE = "*GPLv3" \n'
            if self.x32 == "True":
                    fout = fout + 'DEFAULTTUNE = "x86-64-x32"\n'
                    fout = fout + 'baselib = \\042${@d.getVar(\\047BASE_LIB_tune-\\047 + (d.getVar(\\047DEFAULTTUNE\\047, True) or \\047INVALID\\047), True) or \\047lib\\047}\\042 \n'
            if self.distro == "poky-rt":
                fout = fout + 'PREFERRED_PROVIDER_virtual/kernel="linux-yocto-rt" \n'
            machine=self.getProperty('custom_machine')
            if machine:
                self.machine = machine
            else:
                machine = self.machine
            if self.machine == "atom-pc":
                machine = self.machine.replace("atom-pc", "genericx86")
            if self.machine == "beagleboard":
                machine = self.machine.replace("beagleboard", "beaglebone")
            # NOTE: this is the reverse of how we'd usually do this, but we do
            # not want to break support for using meta-ti with yocto-autobuilder
            if self.machine == "beaglebone-yocto" and layerversion < 4:
                machine = self.machine.replace("beaglebone-yocto", "beaglebone")
            if self.machine == "routerstationpro":
                machine = self.machine.replace("routerstationpro", "edgerouter")

            self.setProperty('MACHINE', machine, "Setting Layer Version")
            fout = fout + 'MACHINE = "' + machine + '"\n'
            fout = fout + 'PREMIRRORS = ""\n'
            if "imx" in self.machine and "danny" in self.getProperty("branch_poky"):
                fout = fout + 'BBMASK = "udev_.*\.bbappend$"\n'
            if self.swabber == "True":
                fout = fout + 'USER_CLASSES += "image-prelink image-swab"\n'
            if os.environ.get("PUBLISH_SOURCE_MIRROR") == "True":
                fout = fout + 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
            if self.distro == "oecore":
                fout = fout + 'INHERIT += "uninative"\n'
                fout = fout + 'require conf/distro/include/yocto-uninative.inc\n'

# Setup buildhistory
# We only collect for the whitelisted repo/branch combinations. This keeps
# us from having a trashed history from "junk" builds (MUTs/release branches/etc)

            if self.buildhistory == "True" and os.environ.get("BUILD_HISTORY_COLLECT") == "True":

                if os.environ.get('BUILD_HISTORY_WHITELIST') is not None or \
                   os.environ.get('BUILD_HISTORY_THROWAWAY_WHITELIST') is not None:
                    ta_history_repos={}
                    history_repos={}
                    fout = fout + 'ERROR_QA_remove = "version-going-backwards"\n'
                    if os.environ.get('BUILD_HISTORY_THROWAWAY_WHITELIST') is not None:
                        ta_history_repos=ast.literal_eval(os.environ.get('BUILD_HISTORY_THROWAWAY_WHITELIST').encode('utf-8'))
                    if os.environ.get('BUILD_HISTORY_WHITELIST') is not None:
                        history_repos=ast.literal_eval(os.environ.get('BUILD_HISTORY_WHITELIST').encode('utf-8'))
                    if repo in history_repos.keys() and \
                       self.getProperty("branch") in history_repos[self.getProperty("repository")]:
                    # We keep this build history
                        self.setProperty('KEEP_BUILD_HIST', "True", "Keeping Buildhistory")
                    elif repo in ta_history_repos.keys() and \
                       self.getProperty("branch") in ta_history_repos[self.getProperty("repository")]:
                    # We toss the prior version of this build history
                        self.setProperty('KEEP_BUILD_HIST', "False", "Temp Buildhistory")

                    if self.getProperty("KEEP_BUILD_HIST"):
                        fout += 'INHERIT += "buildhistory"\n'
                        fout += 'BUILDHISTORY_COMMIT = "1"\n'
                        bh_path = repo.split('/')[-1] + '/' + \
                                  self.getProperty("branch") + '/' + \
                                  self.getProperty("buildername")

                        bh_repo_dir = os.environ.get('BUILD_HISTORY_DIR') + '/' + bh_path

                        fout += 'BUILDHISTORY_DIR = "' + bh_repo_dir + '"\n'
                        fout += 'BUILDHISTORY_PUSH_REPO = "' + os.environ.get('BUILD_HISTORY_REPO') + ' ' + \
                                 bh_path + ':' + bh_path + '"\n'

                        #toaster can only be run on whitelisted branches.
                        if os.environ.get('TOASTER_UPLOAD_URL', None) is not None:
                            fout += 'INHERIT += "toaster"\n'
                        self.command += "pwd=`pwd`;"

                        # Check if the local git clone dir exists
                        # If it doesn't create it and then init it
                        self.command += "if [ ! -d " + bh_repo_dir +" ];"
                        self.command += "then mkdir -p " + bh_repo_dir +";"
                        self.command += "git init " + bh_repo_dir +"; fi;"

                        # If we're tossing away the history, do it now.
                        if self.getProperty("KEEP_BUILD_HIST") is "False":
                            self.command += "rm -rf " + bh_repo_dir +";"
                            self.command += "mkdir -p " + bh_repo_dir +";"
                            self.command += "git init " + bh_repo_dir +";"
                            # We only try and delete the remote if it exists
                            self.command += "if git ls-remote --exit-code " + os.environ.get('BUILD_HISTORY_REPO') + \
                                             " refs/heads/" + bh_path + "> /dev/null;"
                            self.command += "then git push -q " + os.environ.get('BUILD_HISTORY_REPO') + \
                                            " :" + bh_path + ";fi;"

                        # Check if the remote exists
                        # If it doesn't create it and then push
                        self.command += "if ! git ls-remote --exit-code " + os.environ.get('BUILD_HISTORY_REPO') + \
                                         " refs/heads/" + bh_path + " > /dev/null;"
                        self.command += "then cd " + bh_repo_dir +"; echo 'Initializing Repo' >> README;" + \
                                        "git checkout -b " + bh_path + ";git add README; git commit -s -m 'Initializing Repo';"
                        self.command += "git push -q " + os.environ.get('BUILD_HISTORY_REPO') + \
                                        " " + bh_path + ":" + bh_path +";fi;"
                        self.command += "cd $pwd;"

            if self.prserv != "False":
                prserv_host = os.environ.get('PRSERV_HOST', None)
                prserv_port = os.environ.get('PRSERV_PORT', None)
                if prserv_host and prserv_port:
                    fout += 'PRSERV_HOST = "%s:%s"\n' % (prserv_host, prserv_port)

            # using locked-sigs whilst generating an eSDK is only
            # supported on pyro or newer
            if layerversioncore > 9 and self.lockedsigs:
                fout += 'require ../locked-sigs.inc\n'

            if self.buildinfo:
                fout += 'INHERIT += "image-buildinfo"\n'
                fout += 'IMAGE_BUILDINFO_VARS_append = " IMAGE_BASENAME IMAGE_NAME"\n'

            if self.getProperty("branch_poky"):
                if "danny" not in self.getProperty("branch_poky") and distroversion is not None:
                    if self.adtdev == "True":
                        adtrepo_url=os.environ.get("ADTREPO_DEV_URL")
                        fout=fout+'ADTREPO = "' + adtrepo_url + '/' + self.getProperty("distroversion") + '-' + self.getProperty("got_revision_poky") + '-' + self.getProperty("branch_poky") + '"\n'
                    else:
                        adtrepo_url=os.environ.get("ADTREPO_URL")
                        fout=fout+'ADTREPO = "' + adtrepo_url + '/' + self.getProperty("distroversion") + '"\n'
            if self.initmgr and "danny" not in self.getProperty("branch_poky"):
                # we don't neet to test sysvinit only, because that's the default
                if self.initmgr == "systemd":
                    fout = fout + 'DISTRO_FEATURES_append = " systemd"\n'
                    fout = fout + 'VIRTUAL-RUNTIME_init_manager = "systemd"\n'
                    fout = fout + 'DISTRO_FEATURES_BACKFILL_CONSIDERED = "sysvinit"\n'
                if self.initmgr == "systemd sysvinit":
                    fout = fout + 'DISTRO_FEATURES_append = " systemd"\n'
                    fout = fout + 'VIRTUAL-RUNTIME_init_manager = "systemd"\n'
                if self.initmgr == "sysvinit systemd":
                    fout = fout + 'DISTRO_FEATURES_append = " systemd"\n'
                    fout = fout + 'VIRTUAL-RUNTIME_init_manager = "sysvinit"\n'
            if os.environ.get("QEMU_USE_KVM") == "True":
                fout = fout + 'QEMU_USE_KVM = "True"\n'

        sout = ''
        # for pyro or newer when SSTATE_MIRROR_BASE is defined build a
        # minimal eSDK and point SSTATE_MIRRORS at the shared state mirror
        if sstatemirror and layerversioncore > 9:
            sout = 'SSTATE_MIRRORS += "\\ \\n '
            sout = sout + 'file://.* %s/PATH;downloadfilename=PATH"\n' % sstatemirror
            fout = fout + 'SDK_EXT_TYPE = "minimal"\n'
            fout = fout + 'SDK_INCLUDE_TOOLCHAIN = "1"\n'

        # The SANITY_TESTED_DISTROS values in meta-poky can easily
        # fall out of sync with the host OS on the AB workers.
        fout = fout + 'SANITY_TESTED_DISTROS = ""\n'

        # the append text should be the last thing added to the fout string so
        # that we are appending this as the final item in the generated
        # auto.conf
        if self.atextappend:
            fout = fout + self.atextappend

        self.setProperty("LOCALCONF", fout, "CreateAutoConf")
        self.command += "printf '" + fout + "'>> " + self.autoconfpath
        if sout:
            self.command += "; printf '" + sout + "' >> " + self.sdkconfpath

        ShellCommand.start(self)

    def get_release_number(self):
        release_number = self.getProperty("custom_yocto_number", "")
        if not release_number:
            return ""
        release_components = release_number.split('.', 3)
        return '.'.join(release_components).strip('.')