#
# SPDX-License-Identifier: GPL-2.0-only
#
from buildbot.plugins import schedulers as sched
from buildbot.plugins import util
from yoctoabb import config
from twisted.internet import defer
from twisted.python import log
from yoctoabb.yocto_console_view.yocto_console_view import ReleaseSelector
schedulers = []
branchdefaults = {
'master': {
'branch': 'master',
'branch_poky': 'master',
'branch_bitbake': 'master',
'branch_meta-arm': 'master',
'branch_meta-agl': 'next',
'branch_meta-aws': 'master',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'master',
'branch_meta-mingw': 'master',
'branch_meta-exein': 'main',
'branch_meta-openembedded': 'master',
'branch_oecore': 'master',
},
'master-next': {
'branch': 'master',
'branch_poky': 'master-next',
'branch_bitbake': 'master-next',
'branch_meta-arm': 'master',
'branch_meta-agl': 'next',
'branch_meta-aws': 'master',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'master',
'branch_meta-mingw': 'master',
'branch_meta-openembedded': 'master',
'branch_oecore': 'master-next',
},
'abelloni-next': {
'repo_poky': 'ssh://git@push.yoctoproject.org/poky-contrib',
'branch_poky': 'abelloni/master-next',
'repo_oecore': 'ssh://git@push.openembedded.org/openembedded-core-contrib',
'branch_oecore': 'abelloni/master-next',
'repo_bitbake': 'ssh://git@push.openembedded.org/bitbake-contrib',
'branch_bitbake': 'abelloni/master-next',
},
'mut': {
'branch': 'master',
'branch_poky': 'ross/mut',
'repo_poky': 'ssh://git@push.yoctoproject.org/poky-contrib',
'branch_bitbake': 'master',
'branch_meta-arm': 'master',
'branch_meta-agl': 'next',
'branch_meta-aws': 'master',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'master',
'branch_meta-mingw': 'master',
'branch_meta-openembedded': 'master',
'branch_oecore': 'master',
},
'scarthgap': {
'branch': 'scarthgap',
'branch_poky': 'scarthgap',
'branch_bitbake': '2.8',
'branch_meta-arm': 'master',
'branch_meta-agl': 'next',
'branch_meta-aws': 'master',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'master',
'branch_meta-mingw': 'scarthgap',
'branch_meta-openembedded': 'master',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'master',
'branch_oecore': 'scarthgap',
},
'nanbield': {
'branch': 'nanbield',
'branch_poky': 'nanbield',
'branch_bitbake': '2.6',
'branch_meta-arm': 'nanbield',
'branch_meta-agl': 'next',
'branch_meta-aws': 'nanbield',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'nanbield',
'branch_meta-mingw': 'nanbield',
'branch_meta-openembedded': 'nanbield',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'nanbield',
'branch_meta-exein': 'nanbield',
'branch_oecore': 'nanbield',
},
'mickledore': {
'branch': 'mickledore',
'branch_poky': 'mickledore',
'branch_bitbake': '2.4',
'branch_meta-arm': 'mickledore',
'branch_meta-agl': 'next',
'branch_meta-aws': 'mickledore',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'mickledore',
'branch_meta-mingw': 'mickledore',
'branch_meta-openembedded': 'mickledore',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'mickledore',
'branch_oecore': 'mickledore',
},
'langdale': {
'branch': 'langdale',
'branch_poky': 'langdale',
'branch_bitbake': '2.2',
'branch_meta-arm': 'langdale',
'branch_meta-agl': 'next',
'branch_meta-aws': 'langdale',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'langdale',
'branch_meta-mingw': 'langdale',
'branch_meta-openembedded': 'langdale',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'langdale',
'branch_oecore': 'langdale',
},
'kirkstone': {
'branch': 'kirkstone',
'branch_poky': 'kirkstone',
'branch_bitbake': '2.0',
'branch_meta-arm': 'kirkstone',
'branch_meta-agl': 'master',
'branch_meta-aws': 'kirkstone',
'branch_meta-gplv2': 'kirkstone',
'branch_meta-intel': 'kirkstone',
'branch_meta-mingw': 'kirkstone',
'branch_meta-openembedded': 'kirkstone',
'branch_meta-ti': 'kirkstone',
'branch_meta-virtualization': 'kirkstone',
'branch_oecore': 'kirkstone',
},
'honister': {
'branch': 'honister',
'branch_poky': 'honister',
'branch_bitbake': '1.52',
'branch_meta-arm': 'honister',
'branch_meta-aws': 'honister',
'branch_meta-gplv2': 'honister',
'branch_meta-intel': 'honister',
'branch_meta-mingw': 'honister',
'branch_meta-openembedded': 'honister',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'honister',
'branch_oecore': 'honister',
},
'hardknott': {
'branch': 'hardknott',
'branch_poky': 'hardknott',
'branch_bitbake': '1.50',
'branch_meta-arm': 'hardknott',
'branch_meta-aws': 'hardknott',
'branch_meta-gplv2': 'hardknott',
'branch_meta-intel': 'hardknott',
'branch_meta-mingw': 'hardknott',
'branch_meta-openembedded': 'hardknott',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'hardknott',
'branch_oecore': 'hardknott',
},
'gatesgarth': {
'branch': 'gatesgarth',
'branch_poky': 'gatesgarth',
'branch_bitbake': '1.48',
'branch_meta-arm': 'gatesgarth',
'branch_meta-aws': 'gatesgarth',
'branch_meta-gplv2': 'gatesgarth',
'branch_meta-intel': 'gatesgarth',
'branch_meta-mingw': 'gatesgarth',
'branch_meta-openembedded': 'gatesgarth',
'branch_meta-ti': 'master',
'branch_meta-virtualization': 'gatesgarth',
'branch_oecore': 'gatesgarth',
},
'dunfell': {
'branch': 'dunfell',
'branch_poky': 'dunfell',
'branch_bitbake': '1.46',
'branch_meta-arm': 'dunfell',
'branch_meta-agl': 'lamprey',
'branch_meta-aws': 'dunfell',
'branch_meta-gplv2': 'dunfell',
'branch_meta-intel': 'dunfell',
'branch_meta-mingw': 'dunfell',
'branch_meta-openembedded': 'dunfell',
'branch_meta-ti': 'dunfell',
'branch_meta-virtualization': 'dunfell',
'branch_oecore': 'dunfell',
},
'zeus': {
'branch': 'zeus',
'branch_poky': 'zeus',
'branch_bitbake': '1.44',
'branch_meta-gplv2': 'zeus',
'branch_meta-intel': 'zeus',
'branch_meta-mingw': 'zeus',
'branch_oecore': 'zeus',
},
'warrior': {
'branch': 'warrior',
'branch_poky': 'warrior',
'branch_bitbake': '1.42',
'branch_meta-gplv2': 'warrior',
'branch_meta-intel': 'warrior',
'branch_meta-mingw': 'warrior',
'branch_oecore': 'warrior',
},
'thud': {
'branch': 'thud',
'branch_poky': 'thud',
'branch_bitbake': '1.40',
'branch_meta-gplv2': 'thud',
'branch_meta-intel': 'thud',
'branch_meta-mingw': 'thud',
'branch_oecore': 'thud',
},
'sumo': {
'branch': 'sumo',
'branch_poky': 'sumo',
'branch_bitbake': '1.38',
'branch_meta-gplv2': 'sumo',
'branch_meta-intel': 'sumo',
'branch_meta-mingw': 'sumo',
'branch_oecore': 'sumo',
},
'rocko': {
'branch': 'rocko',
'branch_poky': 'rocko',
'branch_bitbake': '1.36',
'branch_meta-gplv2': 'rocko',
'branch_meta-intel': 'rocko',
'branch_meta-mingw': 'rocko',
'branch_oecore': 'rocko',
},
'pyro': {
'branch': 'pyro',
'branch_poky': 'pyro',
'branch_bitbake': '1.34',
'branch_meta-gplv2': 'pyro',
'branch_meta-intel': 'pyro',
'branch_meta-mingw': 'pyro',
'branch_oecore': 'pyro',
},
'morty': {
'branch': 'morty',
'branch_poky': 'morty',
'branch_bitbake': '1.32',
'branch_meta-gplv2': 'master',
'branch_meta-intel': 'morty',
'branch_meta-mingw': 'morty',
'branch_oecore': 'morty',
}
}
def create_repo_inputs(reponame):
"""
Given the name of a repository in yoctoab.config's repo dict creates
StringParameter inputs to allow specification of alternative uri, branch
and commit/hash values
"""
repo = util.StringParameter("repo_{}".format(reponame),
label="Repository:",
default=config.repos[reponame][0])
branch = util.StringParameter("branch_{}".format(reponame),
label="Branch:",
default=config.repos[reponame][1])
commit = util.StringParameter("commit_{}".format(reponame),
label="Revision:",
default="HEAD")
return [repo, branch, commit]
def repos_for_builder(buildername):
"""
Returns a list of additional properties for a scheduler, a list of
StringParameter allowing all of the repositories used by the
builder/scheduler to be customised
"""
parameters = []
repos = config.buildertorepos.get(buildername)
if not repos:
repos = config.buildertorepos["default"]
for repo in repos:
inputs = create_repo_inputs(repo)
parameters = parameters + [util.NestedParameter(name='', label=repo, fields=inputs, columns=2)]
return parameters
def parent_default_props(buildername, branchname=None):
props = {}
props["swat_monitor"] = True
if buildername in ['reproducible-meta-oe', 'meta-oe-mirror']:
props["swat_monitor"] = False
repos = config.buildertorepos.get(buildername)
if not repos:
repos = config.buildertorepos["default"]
if branchname:
props['branch'] = branchname
for repo in repos:
props["repo_{}".format(repo)] = config.repos[repo][0]
branchkey = "branch_{}".format(repo)
if branchname and branchname in branchdefaults and branchkey in branchdefaults[branchname]:
props[branchkey] = branchdefaults[branchname][branchkey]
else:
props[branchkey] = config.repos[repo][1]
config.repos[repo][1]
props["commit_{}".format(repo)] = "HEAD"
return props
def buildappsrcrev_param():
return util.StringParameter(
name="buildappsrcrev",
label="""Build appliance source revision to use. "None" means default to the srcrev currently in the recipe, use "AUTOREV" to use latest revision or specify a revision to use:""",
default="AUTOREV")
def props_for_builder(builder):
"""
Generate an appropriate list of properties to use on a builder-specific
scheduler
"""
swat_default = True
if builder in ['auh', 'meta-oe']:
swat_default = False
props = []
props.append(util.BooleanParameter(
name="swat_monitor",
label="Should SWAT monitor this build?",
default=swat_default))
if builder == 'build-appliance':
props.append(buildappsrcrev_param())
props.append(util.BooleanParameter(
name="deploy_artefacts",
label="Do we want to deploy artefacts?",
default=False
))
props = props + repos_for_builder(builder)
worker_list = config.builder_to_workers.get(builder, config.builder_to_workers['default'])
props.append(util.ChoiceStringParameter(name="worker",
label="Worker to run the build on",
default="*",
multiple=False,
strict=True,
choices=worker_list + ["*"]))
return props
for builder in config.subbuilders:
schedulers.append(sched.ForceScheduler(
name=builder,
builderNames=[builder],
reason=util.StringParameter(
name="reason",
label="""Reason (please note the reason for triggering the
build and any expectations for the build's outcome:""",
required=False),
properties=props_for_builder(builder),
buttonName="Force Build"))
@util.renderer
def builderNamesFromConfigQuick(props):
#log.msg("builderNames: Sourcestamp %s, props %s" % (str(props.sourcestamps), str(props)))
yp_branch = props.sourcestamps[0]['branch']
builders = config.trigger_builders_wait_quick
for b in config.trigger_builders_wait_quick_releases:
if yp_branch and yp_branch.startswith(b):
log.msg("builderNames: Filtering branch %s due to entry %s" % (str(yp_branch), str(b)))
builders = config.trigger_builders_wait_quick_releases[b]
return builders
@util.renderer
def builderNamesFromConfigFull(props):
#log.msg("builderNames: Sourcestamp %s, props %s" % (str(props.sourcestamps), str(props)))
yp_branch = props.sourcestamps[0]['branch']
builders = config.trigger_builders_wait_full
for b in config.trigger_builders_wait_full_releases:
if yp_branch and yp_branch.startswith(b):
log.msg("builderNames: Filtering branch %s due to entry %s" % (str(yp_branch), str(b)))
builders = config.trigger_builders_wait_full_releases[b]
# Only run performance runs on release builds
if props.getProperty("is_release", False):
builders = builders + config.trigger_builders_wait_perf
return builders
# Upstream Triggerable class will rasise NotImplementedError() which will mean triggers abort upon reconfig
# Hack to intercept and ignore this, we'd rather they just survive in our case.
class ourTriggerable(sched.Triggerable):
def reconfigService(self, name=None, *args, **kwargs):
return
# nightly builder triggers various other builders
wait_quick = ourTriggerable(name="wait-quick",
builderNames=builderNamesFromConfigQuick)
schedulers.append(wait_quick)
wait_full = ourTriggerable(name="wait-full",
builderNames=builderNamesFromConfigFull)
schedulers.append(wait_full)
def parent_scheduler(target):
return sched.ForceScheduler(
name=target,
builderNames=[target],
buttonName="Start " + target + " Build",
codebases = [util.CodebaseParameter(codebase='', label="yocto-autobuilder-helper:", project=None)],
reason=util.StringParameter(
name="reason",
label="""Reason (please note the reason for triggering the build and
any expectations for the build's outcome:""",
required=False),
properties=[
ReleaseSelector(
name="branchselector",
default="master",
label="Release Shortcut Selector",
choices=["master", "master-next", "abelloni-next", "mut", "scarthgap", "nanbield", "mickledore", "langdale", "kirkstone", "honister", "hardknott", "gatesgarth", "dunfell", "zeus", "warrior", "thud", "sumo", "rocko", "pyro", "morty"],
selectors=branchdefaults),
util.BooleanParameter(
name="swat_monitor",
label="Should SWAT monitor this build?",
default=True),
buildappsrcrev_param(),
util.BooleanParameter(
name="is_release",
label="Generate a release?",
default=False),
util.StringParameter(
name="yocto_number", # used to form publish path
label="Yocto Project Release Number (1.5, 1.6 etc.)"),
util.ChoiceStringParameter(
name="milestone_number",
label="Milestone number",
choices=["", "M1", "M2", "M3", "M4"],
default=""
),
util.ChoiceStringParameter(
name="rc_number",
label="Release candidate number",
choices=["", "rc1", "rc2", "rc3", "rc4", "rc5", "rc6", "rc7",
"rc8", "rc9"],
default=""),
util.BooleanParameter(
name="send_email",
label="Send QA alert emails?",
default=False),
util.BooleanParameter(
name="deploy_artefacts",
label="Do we want to save build output? ",
default=False)
]+repos_for_builder(target))
schedulers.append(parent_scheduler("a-quick"))
schedulers.append(parent_scheduler("a-full"))
schedulers.append(sched.ForceScheduler(
name="docs",
builderNames=["docs"],
reason=util.StringParameter(
name="reason",
label="""Reason (please note the reason for triggering the docs build:""",
required=False),
properties=props_for_builder("docs"),
buttonName="Force Build"))
# Run a-quick at 1am each day Mon-Sat so we keep master tested and up to date in sstate and buildhistory
schedulers.append(sched.Nightly(name='nightly-quick', branch='master', properties=parent_default_props('a-quick'),
builderNames=['a-quick'], hour=1, minute=0, dayOfWeek=[0,1,2,3,4,5]))
# Run a-full at 1am Sun each Week
schedulers.append(sched.Nightly(name='nightly-full', branch='master', properties=parent_default_props('a-full'),
builderNames=['a-full'], hour=1, minute=0, dayOfWeek=6))
# Run check-layer-nightly each day for master
schedulers.append(sched.Nightly(name='nightly-check-layer', branch='master', properties=parent_default_props('check-layer-nightly'),
builderNames=['check-layer-nightly'], hour=0, minute=0))
# Run meta-oe-mirror each day for master
schedulers.append(sched.Nightly(name='nightly-meta-oe-mirror', branch='master', properties=parent_default_props('meta-oe-mirror'),
builderNames=['meta-oe-mirror'], hour=0, minute=0))
# Run reproducible-meta-oe for master once a week on weekends (2am on Saturday)
schedulers.append(sched.Nightly(name='nightly-reproducible-meta-oe', branch='master', properties=parent_default_props('reproducible-meta-oe'),
builderNames=['reproducible-meta-oe'], hour=2, minute=0, dayOfWeek=5))
# Run metrics at 3-7am each day
schedulers.append(sched.Nightly(name='nightly-metrics-dunfell', branch='dunfell', properties=parent_default_props('metrics', 'dunfell'), builderNames=['metrics'], hour=6, minute=0, codebases = {'' : {'branch' : 'dunfell'}}))
schedulers.append(sched.Nightly(name='nightly-metrics-kirkstone', branch='kirkstone', properties=parent_default_props('metrics', 'kirkstone'), builderNames=['metrics'], hour=6, minute=15, codebases = {'' : {'branch' : 'kirkstone'}}))
schedulers.append(sched.Nightly(name='nightly-metrics-mickledore', branch='mickledore', properties=parent_default_props('metrics', 'mickledore'), builderNames=['metrics'], hour=6, minute=30, codebases = {'' : {'branch' : 'mickledore'}}))
schedulers.append(sched.Nightly(name='nightly-metrics-nanbield', branch='nanbield', properties=parent_default_props('metrics', 'nanbield'), builderNames=['metrics'], hour=6, minute=45, codebases = {'' : {'branch' : 'nanbield'}}))
schedulers.append(sched.Nightly(name='nightly-metrics', branch='master', properties=parent_default_props('metrics'), builderNames=['metrics'], hour=7, minute=0))
# Run check-layer-nightly twice a week for nanbield
schedulers.append(sched.Nightly(name='nightly-check-layer-nanbield', properties=parent_default_props('check-layer-nightly', 'nanbield'),
builderNames=['check-layer-nightly'], dayOfWeek=[3, 7], hour=2, minute=0, codebases = {'' : {'branch' : 'nanbield'}}))
# Run meta-oe-mirror twice a month for nanbield
schedulers.append(sched.Nightly(name='nightly-meta-oe-mirror-nanbield', properties=parent_default_props('meta-oe-mirror', 'nanbield'),
builderNames=['meta-oe-mirror'], dayOfMonth=[1, 15], hour=2, minute=0, codebases = {'' : {'branch' : 'nanbield'}}))
# Run check-layer-nightly twice a week for kirkstone
schedulers.append(sched.Nightly(name='nightly-check-layer-kirkstone', properties=parent_default_props('check-layer-nightly', 'kirkstone'),
builderNames=['check-layer-nightly'], dayOfWeek=[0, 3], hour=2, minute=0, codebases = {'' : {'branch' : 'kirkstone'}}))
# Run meta-oe-mirror twice a month for kirkstone
schedulers.append(sched.Nightly(name='nightly-meta-oe-mirror-kirkstone', properties=parent_default_props('meta-oe-mirror', 'kirkstone'),
builderNames=['meta-oe-mirror'], dayOfMonth=[2, 16], hour=2, minute=0, codebases = {'' : {'branch' : 'kirkstone'}}))
# Run check-layer-nightly twice a week for dunfell
schedulers.append(sched.Nightly(name='nightly-check-layer-dunfell', properties=parent_default_props('check-layer-nightly', 'dunfell'),
builderNames=['check-layer-nightly'], dayOfWeek=[1, 4], hour=2, minute=0, codebases = {'' : {'branch' : 'dunfell'}}))
# Run meta-oe-mirror twice a month for dunfell
schedulers.append(sched.Nightly(name='nightly-meta-oe-mirror-dunfell', properties=parent_default_props('meta-oe-mirror', 'dunfell'),
builderNames=['meta-oe-mirror'], dayOfMonth=[3, 17], hour=2, minute=0, codebases = {'' : {'branch' : 'dunfell'}}))
# Run the build performance tests at 3am, 9am, 3pm and 9pm
schedulers.append(sched.Nightly(name='nightly-buildperf', branch='master', properties=parent_default_props('buildperf-debian11'),
builderNames=['buildperf-debian11', 'buildperf-alma8'], hour=[3,9,15,21], minute=0))
# Run the toaster tests daily
schedulers.append(sched.Nightly(name='nightly-toaster', branch='master', properties=parent_default_props('toaster'),
builderNames=['toaster'], hour=[3], minute=0))
# Run the AUH twice a month on 1st and 15th
schedulers.append(sched.Nightly(name='nightly-auh', branch='master', properties=parent_default_props('auh'),
builderNames=['auh'], dayOfMonth=[1, 15], hour=5, minute=0))
# If any of our sphinx docs branches change, trigger a build
schedulers.append(sched.AnyBranchScheduler(name="yocto-docs-changed",
change_filter=util.ChangeFilter(project=["yocto-docs"], branch=[None, "master", "master-next", "mickledore", "langdale", "kirkstone", "honister", "hardknott", "gatesgarth", "dunfell", "transition"]),
codebases = ['', 'yocto-docs', 'bitbake'],
treeStableTimer=60,
builderNames=["docs"]))
# If bitbake's sphinx docs change, trigger a build
def isbitbakeDocFile(change):
for f in change.files:
if "doc/" in f:
return True
return False
schedulers.append(sched.AnyBranchScheduler(name="bitbake-docs-changed",
change_filter=util.ChangeFilter(project=["bitbake"], branch=["master", "1.52", "1.50", "1.48", "1.46"]),
codebases = ['', 'yocto-docs', 'bitbake'],
fileIsImportant=isbitbakeDocFile,
onlyImportant=True,
treeStableTimer=60,
builderNames=["docs"]))
