path: root/scripts/run-config

#!/usr/bin/env python3
#
# Copyright Linux Foundation, Richard Purdie
#
# SPDX-License-Identifier: GPL-2.0-only
#
# Iterate over a set of configurations from json.conf, calling setup-config for each one, then running the build.
#

import json
import os
import sys
import subprocess
import errno

import utils

parser = utils.ArgParser(description='Runs configurations in json.conf.')

parser.add_argument('target',
                    help="The 'nightly' target the autobuilder is running")
parser.add_argument('builddir',
                    help="The target build directory to configure")
parser.add_argument('branchname',
                    help="The poky branch name the build is running on")
parser.add_argument('reponame',
                    help="The name of the repository the build is running on")
parser.add_argument('-s', '--sstateprefix',
                    default='',
                    help="The directory prefix to publish sstate into")
parser.add_argument('-b', '--buildappsrcrev',
                    default='',
                    help="A build appliance SRCREV to use")
parser.add_argument('-p', '--publish-dir',
                    action='store',
                    help="Where to publish artefacts to (optional)")
parser.add_argument('-r', '--results-dir',
                    action='store',
                    help="Where to publish test results to (optional)")
parser.add_argument('-u', '--build-url',
                    action='store',
                    help="URL back to this build (for the error reporting system)")
parser.add_argument('--build-type',
                    action='store',
                    default="quick",
                    help="the type of build being triggered (full or quick)")
parser.add_argument('-t', '--test',
                    action='store_true',
                    default=False,
                    help="Test mode - perform setup and dry-run of commands only")
parser.add_argument('-q', '--quietlogging',
                    action='store_true',
                    default=False,
                    help="Quiet mode - don't echo bitbake logs to stdout")
parser.add_argument('--workername',
                    action='store',
                    default=None,
                    help="the name of the worker the build is running on")
parser.add_argument('-j', '--json-outputfile',
                    action='store',
                    default="",
                    help="the file to store json information about the build in")
parser.add_argument('--stepname',
                    action='store',
                    default=None,
                    help="the name of the step to run")
parser.add_argument('--phase',
                    action='store',
                    default=None,
                    help="the phase of the step to run")



args = parser.parse_args()

scriptsdir = os.path.dirname(os.path.realpath(__file__))
os.environ["SCRIPTSDIR"] = scriptsdir
ourconfig = utils.loadconfig()
ourconfig["HELPERBUILDDIR"] = args.builddir
ourconfig["HELPERTARGET"] = args.target
ourconfig["HELPERRESULTSDIR"] = (args.results_dir or "")
ourconfig["HELPERREPONAME"] = args.reponame
ourconfig["HELPERBRANCHNAME"] = args.branchname

hp = utils.HeaderPrinter()

testmode = args.test

# toolchain tests are run in system mode for x86, user mode for the other 
# arches due to speed
# toolchain tests only run on full builds
arch = args.target
arch = arch.replace("-tc", "")
if arch in ["qemuriscv32", "qemuriscv64", "qemuppc64"]:
    args.build_type = "full"
if args.build_type == "quick":
    ourconfig["HELPERSTMACHTARGS"] = "-a -t machine"
elif args.build_type == "full":
    if arch == "qemux86" or arch == "qemux86-64":
        ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-system"
    else:
        ourconfig["HELPERSTMACHTARGS"] = "-a -t machine -t toolchain-user"

# Find out the number of steps this target has
maxsteps = 0
stepnum = 0
if args.target in ourconfig['overrides']:
    maxsteps = 1
    for v in ourconfig['overrides'][args.target]:
        if v.startswith("step"):
            n = int(v[4:])
            if n <= maxsteps:
                continue
            maxsteps = n

hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))

jcfg = False
if args.json_outputfile:
    jsonconfig = []
    jcfg = True

# There is a 50 char limit on "bbname" but buildbot may append "_1", "_2" if multiple steps
# with the same name exist in a build
def addentry(name, description, phase):
    jsonconfig.append({"name" : name, "bbname" : description[:46], "phase" : phase, "description" : description})

def addstepentry(name, taskdesc, shortname, description, detail, phase, usepty=False):
    bbname = taskdesc
    if shortname:
        bbname = shortname + ": " + taskdesc
    bbdesc = taskdesc
    if description:
        bbdesc = description
    if detail:
        bbdesc = bbdesc + ": " + detail
    jsonconfig.append({"name" : name, "bbname" : bbname[:46], "phase" : phase, "description" : bbdesc, "usepty" : usepty})

if jcfg:
    buildtools = utils.setup_buildtools_tarball(ourconfig, args.workername, None, checkonly=True)
    if buildtools:
        addentry("buildtools", "Setup buildtools tarball", "init")
else:
    utils.setup_buildtools_tarball(ourconfig, args.workername, args.builddir + "/../buildtools")
    if args.phase == "init" and args.stepname == "buildtools":
        sys.exit(0)

extratools = utils.getconfigvar("extratools", ourconfig, args.target)
if jcfg:
    if extratools:
        addentry("extratools", "Setup extratools tarball", "init")
elif extratools:
    utils.setup_tools_tarball(ourconfig, args.builddir + "/../extratools", extratools, "extratools")
    if args.phase == "init" and args.stepname == "extratools":
        sys.exit(0)

logconfig = args.builddir + "/../bitbake/contrib/autobuilderlog.json"
print("Using BB_LOGCONFIG=%s" % logconfig)
os.environ["BB_LOGCONFIG"] = logconfig

finalret = 0

def flush():
    sys.stdout.flush()
    sys.stderr.flush()

def logname(path, stepnum, stepname):
    return path + "/command-%s-%s.log" % (stepnum, stepname)

utils.mkdir(args.builddir)

revision = "unknown"
report = utils.ErrorReport(ourconfig, args.target, args.builddir, args.branchname, revision)
errordir = utils.errorreportdir(args.builddir)
utils.mkdir(errordir)

errorlogs = set()

def log_file_contents(filename, builddir, stepnum, stepname):
    logfile = logname(builddir, stepnum, stepname)
    with open(logfile, "a") as outf, open(filename, "r") as f:
        def log(s):
            outf.write(s)
            sys.stdout.write(s)

        log("Contents of %s:\n" % filename)
        for line in f:
            log(line)
        log("\n")


def bitbakecmd(builddir, cmd, report, stepnum, stepname, oeenv=True):
    global finalret
    flush()
    log = logname(builddir, stepnum, stepname)
    errordir = utils.errorreportdir(builddir)
    try:
        numreports = len(os.listdir(errordir))
    except FileNotFoundError:
        numreports = 0

    def writelog(msg, a, b):
        a.write(msg)
        b.write(msg)

    if oeenv:
        cmd = ". ./oe-init-build-env; %s" % cmd

    if testmode:
        print("Would run '%s'" % cmd)
        return

    with open(log, "a") as outf:
        writelog("Running '%s' with output to %s\n" % (cmd, log), outf, sys.stdout)

    with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0) as p, open(log, 'ab') as f:
        for line in p.stdout:
            writelog(line, f, sys.stdout.buffer)
            sys.stdout.flush()
            f.flush()
        ret = p.wait()
    if ret:
        hp.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
        # No error report was written but the command failed so we should write one
        try:
            finalnumreports = len(os.listdir(errordir))
        except FileNotFoundError:
            finalnumreports = 0
        if finalnumreports == numreports:
            report.create(cmd, stepnum, log)
        finalret += 1
        errorlogs.add(log)

def runcmd(cmd, *args, **kwargs):
    if testmode:
        print("Running %s" % cmd)
        if "setup-config" not in cmd[0]:
            return
    try:
        subprocess.check_call(cmd, *args, **kwargs)
    except subprocess.CalledProcessError:
        print("ERROR: Command %s failed" % cmd)

bh_path, remoterepo, remotebranch, baseremotebranch = utils.getbuildhistoryconfig(ourconfig, args.builddir, args.target, args.reponame, args.branchname, 1)
if bh_path:
    if jcfg:
        addentry("buildhistory-init", "Initialize buildhistory", "init")
if args.phase == "init" and args.stepname == "buildhistory-init":
    if bh_path:
        runcmd([os.path.join(scriptsdir, "buildhistory-init"), bh_path, remoterepo, remotebranch, baseremotebranch])
    sys.exit(0)

def handle_stepnum(stepnum):
    shortdesc = utils.getconfigvar("shortname", ourconfig, args.target, stepnum) or ""
    desc = utils.getconfigvar("description", ourconfig, args.target, stepnum) or ""

    # Add any layers specified
    layers = utils.getconfiglist("ADDLAYER", ourconfig, args.target, stepnum)
    if jcfg:
        if layers:
            addstepentry("add-layers", "Add layers", shortdesc, desc, str(layers), str(stepnum))
    elif args.stepname == "add-layers":
        for layer in layers:
            bitbakecmd(args.builddir, "bitbake-layers add-layer %s" % layer, report, stepnum, args.stepname)
        log_file_contents(args.builddir + "/conf/bblayers.conf", args.builddir, stepnum, args.stepname)

    flush()

    # Generate the configuration files needed for this step
    if utils.getconfigvar("WRITECONFIG", ourconfig, args.target, stepnum):
        if jcfg:
            addstepentry("write-config", "Write config", shortdesc, desc, None, str(stepnum))
        elif args.stepname == "write-config":
            runcmd([scriptsdir + "/setup-config", args.target, str(stepnum - 1), args.builddir, args.branchname, args.reponame, "-s", args.sstateprefix, "-b", args.buildappsrcrev])
            log_file_contents(args.builddir + "/conf/auto.conf", args.builddir, stepnum, args.stepname)

    # Execute the targets for this configuration
    targets = utils.getconfigvar("BBTARGETS", ourconfig, args.target, stepnum)
    if targets:
        if jcfg:
            addstepentry("build-targets", "Build targets", shortdesc, desc, str(targets), str(stepnum))
        elif args.stepname == "build-targets":
            hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, targets))
            bitbakecmd(args.builddir, "bitbake %s -k" % targets, report, stepnum, args.stepname)

    # Execute the sanity targets for this configuration
    sanitytargets = utils.getconfigvar("SANITYTARGETS", ourconfig, args.target, stepnum)
    if sanitytargets:
        if jcfg:
            addstepentry("test-targets", "QA targets", shortdesc, desc, str(sanitytargets), str(stepnum))
        elif args.stepname == "test-targets":
            hp.printheader("Step %s/%s: Running bitbake %s" % (stepnum, maxsteps, sanitytargets))
            bitbakecmd(args.builddir, "%s/checkvnc; DISPLAY=:1 bitbake %s -k" % (scriptsdir, sanitytargets), report, stepnum, args.stepname)

    # Run any extra commands specified
    cmds = utils.getconfiglist("EXTRACMDS", ourconfig, args.target, stepnum)
    if jcfg:
        if cmds:
            usepty = False
            if utils.getconfigvar("USEPTY", ourconfig, args.target, stepnum):
                usepty = True
            addstepentry("cmds", "Run cmds", shortdesc, desc, str(cmds), str(stepnum), usepty=usepty)
    elif args.stepname == "cmds":
        for cmd in cmds:
            hp.printheader("Step %s/%s: Running command %s" % (stepnum, maxsteps, cmd))
            bitbakecmd(args.builddir, cmd, report, stepnum, args.stepname)

    cmds = utils.getconfiglist("EXTRAPLAINCMDS", ourconfig, args.target, stepnum)
    if jcfg:
        if cmds:
            addstepentry("plain-cmds", "Run cmds", shortdesc, desc, str(cmds), str(stepnum))
    elif args.stepname == "plain-cmds":
        for cmd in cmds:
            hp.printheader("Step %s/%s: Running 'plain' command %s" % (stepnum, maxsteps, cmd))
            bitbakecmd(args.builddir, cmd, report, stepnum, args.stepname, oeenv=False)

    if jcfg:
        if layers:
            addstepentry("remove-layers", "Remove layers", shortdesc, desc, str(layers), str(stepnum))
    elif args.stepname == "remove-layers":
        # Remove any layers we added in a reverse order
        for layer in reversed(layers):
            bitbakecmd(args.builddir, "bitbake-layers remove-layer %s" % layer, report, stepnum, args.stepname)
        log_file_contents(args.builddir + "/conf/bblayers.conf", args.builddir, stepnum, args.stepname)

    if not jcfg:
        sys.exit(finalret)

if jcfg:
    for stepnum in range(1, maxsteps + 1):
        handle_stepnum(stepnum)
else:
    try:
        stepnum = int(args.phase)
    except ValueError:
        stepnum = None

    if stepnum is not None:
        handle_stepnum(stepnum)


if jcfg:
    addentry("publish", "Publishing artefacts", "finish")
elif args.phase == "finish" and args.stepname == "publish":
    if args.publish_dir:
        hp.printheader("Running publish artefacts")
        runcmd([scriptsdir + "/publish-artefacts", args.builddir, args.publish_dir, args.target])
    sys.exit(0)

if jcfg:
    addentry("collect-results", "Collecting result files", "finish")
elif args.phase == "finish" and args.stepname == "collect-results":
    if args.results_dir:
        hp.printheader("Running results collection")
        runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])
        runcmd([scriptsdir + "/summarize_top_output.py", args.results_dir, args.target])
        runcmd([scriptsdir + "/archive_buildstats.py", args.builddir, args.results_dir, args.target])
    sys.exit(0)

if jcfg:
    addentry("send-errors", "Sending error reports", "finish")
elif args.phase == "finish" and args.stepname == "send-errors":
    if args.build_url and utils.getconfigvar("SENDERRORS", ourconfig, args.target, stepnum):
        hp.printheader("Sending any error reports")
        runcmd([scriptsdir + "/upload-error-reports", args.builddir, args.build_url])
    sys.exit(0)

if jcfg:
    addentry("builddir-cleanup", "Cleaning up build directory", "finish")
elif args.phase == "finish" and args.stepname == "builddir-cleanup":
    if args.builddir and os.path.exists(args.builddir):
        if os.path.exists("oe-init-build-env"):
            bitbakecmd(args.builddir, "bitbake -m", report, 99, args.stepname)
        runcmd(["mv", args.builddir, args.builddir + "-renamed"])

if args.json_outputfile:
    with open(args.json_outputfile, "w") as f:
        json.dump(jsonconfig, f, indent=4, sort_keys=True)

sys.exit(0)