aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-11-21 23:21:10 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-11-21 23:21:10 +0000
commitb7a5d4bd7a2ff44d2118a56e616d59aa01bf6c94 (patch)
tree31bbfed785215ed1bac59414b14d379c2f7fe90d
parent2174bcfba5ed0c4bf861163deefa2364fd40ff8a (diff)
downloadyocto-autobuilder2-b7a5d4bd7a2ff44d2118a56e616d59aa01bf6c94.tar.gz
yocto-autobuilder2-b7a5d4bd7a2ff44d2118a56e616d59aa01bf6c94.tar.bz2
yocto-autobuilder2-b7a5d4bd7a2ff44d2118a56e616d59aa01bf6c94.zip
config/builders/schedulers: Split nightly into quick and full targets
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--builders.py223
-rw-r--r--config.py25
-rw-r--r--schedulers.py28
3 files changed, 145 insertions, 131 deletions
diff --git a/builders.py b/builders.py
index 47f34be..92fb3eb 100644
--- a/builders.py
+++ b/builders.py
@@ -188,7 +188,7 @@ def create_builder_factory():
# regular builders
f = create_builder_factory()
-for builder in config.triggered_builders:
+for builder in config.subbuilders:
workers = config.builder_to_workers.get(builder, None)
if not workers:
workers = config.builder_to_workers['default']
@@ -196,115 +196,114 @@ for builder in config.triggered_builders:
workernames=workers,
factory=f, env=extra_env))
-factory = util.BuildFactory()
-# NOTE: Assumes that yocto-autobuilder repo has been cloned to home
-# directory of the user running buildbot.
-clob = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
-factory.addStep(steps.ShellCommand(
- command=[clob, util.Interpolate("%(prop:builddir)s/")],
- haltOnFailure=True,
- name="Clobber build dir"))
-# check out the source
-factory.addStep(steps.Git(
- repourl='git://git.yoctoproject.org/yocto-autobuilder-helper',
- workdir=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper"),
- mode='incremental',
- haltOnFailure=True,
- name='Fetch yocto-autobuilder-helper'))
-factory.addStep(WriteLayerInfo(name='Write main layerinfo.json', haltOnFailure=True))
-factory.addStep(steps.ShellCommand(
- command=[
- util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/prepare-shared-repos"),
- util.Interpolate("%(prop:builddir)s/layerinfo.json"),
- util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
- "-p", get_publish_dest],
- haltOnFailure=True,
- name="Prepare shared repositories"))
-factory.addStep(steps.SetProperty(
- property="sharedrepolocation",
- value=util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir))
-))
-
-# shared-repo-unpack
-factory.addStep(steps.ShellCommand(
- command=[
- util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"),
- util.Interpolate("%(prop:builddir)s/layerinfo.json"),
- util.Interpolate("%(prop:builddir)s/build"),
- util.Property("buildername"),
- "-c", util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
- "-p", util.Property("is_release")],
- haltOnFailure=True,
- name="Unpack shared repositories"))
-
-factory.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop:sharedrepolocation)s/poky; git rev-parse HEAD"),
- property="yp_build_revision",
- haltOnFailure=True,
- name='Set build revision'))
-
-factory.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop:sharedrepolocation)s/poky; git rev-parse --abbrev-ref HEAD"),
- property="yp_build_branch",
- haltOnFailure=True,
- name='Set build branch'))
-
-# run-config
-factory.addStep(RunConfigLogObserver(
- command=[
- util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
- util.Property("buildername"),
- util.Interpolate("%(prop:builddir)s/build/build"),
- util.Interpolate("%(prop:branch_poky)s"),
- util.Interpolate("%(prop:repo_poky)s"),
- "-s", get_sstate_release_number,
- "-p", get_publish_dest,
- "-u", util.URLForBuild,
- "-r", get_publish_resultdir,
- "-q"],
- name="run-config",
- logfiles=get_buildlogs(maxsteps),
- lazylogfiles=True,
- maxsteps=maxsteps,
- timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
-
-# trigger the buildsets contained in the nightly set
-def get_props_set():
- set_props = {
- "sharedrepolocation": util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
- "is_release": util.Property("is_release"),
- "buildappsrcrev": "",
- "deploy_artefacts": util.Property("deploy_artefacts"),
- "publish_destination": util.Property("publish_destination"),
- "yocto_number": util.Property("yocto_number"),
- "milestone_number": util.Property("milestone_number"),
- "rc_number": util.Property("rc_number")
- }
+def create_parent_builder_factory(waitname):
+ factory = util.BuildFactory()
+ # NOTE: Assumes that yocto-autobuilder repo has been cloned to home
+ # directory of the user running buildbot.
+ clob = os.path.expanduser("~/yocto-autobuilder-helper/janitor/clobberdir")
+ factory.addStep(steps.ShellCommand(
+ command=[clob, util.Interpolate("%(prop:builddir)s/")],
+ haltOnFailure=True,
+ name="Clobber build dir"))
+ # check out the source
+ factory.addStep(steps.Git(
+ repourl='git://git.yoctoproject.org/yocto-autobuilder-helper',
+ workdir=util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper"),
+ mode='incremental',
+ haltOnFailure=True,
+ name='Fetch yocto-autobuilder-helper'))
+ factory.addStep(WriteLayerInfo(name='Write main layerinfo.json', haltOnFailure=True))
+ factory.addStep(steps.ShellCommand(
+ command=[
+ util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/prepare-shared-repos"),
+ util.Interpolate("%(prop:builddir)s/layerinfo.json"),
+ util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
+ "-p", get_publish_dest],
+ haltOnFailure=True,
+ name="Prepare shared repositories"))
+ factory.addStep(steps.SetProperty(
+ property="sharedrepolocation",
+ value=util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir))
+ ))
+
+ # shared-repo-unpack
+ factory.addStep(steps.ShellCommand(
+ command=[
+ util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/shared-repo-unpack"),
+ util.Interpolate("%(prop:builddir)s/layerinfo.json"),
+ util.Interpolate("%(prop:builddir)s/build"),
+ util.Property("buildername"),
+ "-c", util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
+ "-p", util.Property("is_release")],
+ haltOnFailure=True,
+ name="Unpack shared repositories"))
+
+ factory.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop:sharedrepolocation)s/poky; git rev-parse HEAD"),
+ property="yp_build_revision",
+ haltOnFailure=True,
+ name='Set build revision'))
+
+ factory.addStep(steps.SetPropertyFromCommand(command=util.Interpolate("cd %(prop:sharedrepolocation)s/poky; git rev-parse --abbrev-ref HEAD"),
+ property="yp_build_branch",
+ haltOnFailure=True,
+ name='Set build branch'))
+
+ # run-config
+ factory.addStep(RunConfigLogObserver(
+ command=[
+ util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config"),
+ util.Property("buildername"),
+ util.Interpolate("%(prop:builddir)s/build/build"),
+ util.Interpolate("%(prop:branch_poky)s"),
+ util.Interpolate("%(prop:repo_poky)s"),
+ "-s", get_sstate_release_number,
+ "-p", get_publish_dest,
+ "-u", util.URLForBuild,
+ "-r", get_publish_resultdir,
+ "-q"],
+ name="run-config",
+ logfiles=get_buildlogs(maxsteps),
+ lazylogfiles=True,
+ maxsteps=maxsteps,
+ timeout=16200)) # default of 1200s/20min is too short, use 4.5hrs
- for repo in config.repos:
- set_props["branch_%s" % repo] = util.Property("branch_%s" % repo)
- set_props["commit_%s" % repo] = util.Property("commit_%s" % repo)
- set_props["repo_%s" % repo] = util.Property("repo_%s" % repo)
- set_props["yocto_number"] = util.Property("yocto_number")
- set_props["milestone_number"] = util.Property("milestone_number")
- set_props["rc_number"] = util.Property("rc_number")
-
- return set_props
-
-factory.addStep(steps.Trigger(schedulerNames=['wait'],
- waitForFinish=True,
- set_properties=get_props_set()))
-
-factory.addStep(steps.ShellCommand(
- command=[
- util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/send-qa-email"),
- util.Property("send_email"),
- util.Interpolate("%(prop:builddir)s/layerinfo.json"),
- util.Interpolate("%(prop:sharedrepolocation)s"),
- "-p", get_publish_dest,
- "-r", get_publish_name
- ],
- name="Send QA Email"))
-
-builders.append(
- util.BuilderConfig(name="nightly",
- workernames=config.workers,
- factory=factory, env=extra_env))
+ # trigger the buildsets contained in the nightly set
+ def get_props_set():
+ set_props = {
+ "sharedrepolocation": util.Interpolate("{}/%(prop:buildername)s-%(prop:buildnumber)s".format(config.sharedrepodir)),
+ "is_release": util.Property("is_release"),
+ "buildappsrcrev": "",
+ "deploy_artefacts": util.Property("deploy_artefacts"),
+ "publish_destination": util.Property("publish_destination"),
+ "yocto_number": util.Property("yocto_number"),
+ "milestone_number": util.Property("milestone_number"),
+ "rc_number": util.Property("rc_number")
+ }
+
+ for repo in config.repos:
+ set_props["branch_%s" % repo] = util.Property("branch_%s" % repo)
+ set_props["commit_%s" % repo] = util.Property("commit_%s" % repo)
+ set_props["repo_%s" % repo] = util.Property("repo_%s" % repo)
+ set_props["yocto_number"] = util.Property("yocto_number")
+ set_props["milestone_number"] = util.Property("milestone_number")
+ set_props["rc_number"] = util.Property("rc_number")
+
+ return set_props
+
+ factory.addStep(steps.Trigger(schedulerNames=[waitname],
+ waitForFinish=True,
+ set_properties=get_props_set()))
+
+ factory.addStep(steps.ShellCommand(
+ command=[
+ util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/send-qa-email"),
+ util.Property("send_email"),
+ util.Interpolate("%(prop:builddir)s/layerinfo.json"),
+ util.Interpolate("%(prop:sharedrepolocation)s"),
+ "-p", get_publish_dest,
+ "-r", get_publish_name
+ ],
+ name="Send QA Email"))
+
+builders.append(util.BuilderConfig(name="quick", workernames=config.workers, factory=create_parent_builder_factory("wait-quick"), env=extra_env))
+builders.append(util.BuilderConfig(name="full", workernames=config.workers, factory=create_parent_builder_factory("wait-full"), env=extra_env))
diff --git a/config.py b/config.py
index 295f739..4a387f8 100644
--- a/config.py
+++ b/config.py
@@ -3,7 +3,10 @@
buildertorepos = {
"eclipse-plugin-neon": ["eclipse-poky-neon"],
"eclipse-plugin-oxygen": ["eclipse-poky-oxygen"],
- "nightly": ["poky", "meta-intel", "oecore", "bitbake",
+ "quick": ["poky", "meta-intel", "oecore", "bitbake",
+ "eclipse-poky-neon", "eclipse-poky-oxygen", "meta-qt4",
+ "meta-qt3", "meta-mingw", "meta-gplv2"],
+ "full": ["poky", "meta-intel", "oecore", "bitbake",
"eclipse-poky-neon", "eclipse-poky-oxygen", "meta-qt4",
"meta-qt3", "meta-mingw", "meta-gplv2"],
"non-gpl3": ["poky", "meta-gplv2"],
@@ -35,16 +38,16 @@ repos = {
"meta-gplv2": ["git://git.yoctoproject.org/meta-gplv2", "master"]
}
-trigger_builders_wait = [
+trigger_builders_wait_quick = [
"qemuarm", "qemuarm-lsb", "qemuarm64", "qemuarm-oecore",
- "qemumips", "qemumips-lsb", "qemumips64",
+ "qemumips", "qemumips64",
"multilib",
- "qemuppc", "qemuppc-lsb",
+ "qemuppc",
"qemux86", "qemux86-lsb",
"qemux86-64", "qemux86-64-lsb",
- "qemux86-64-x32", "qemux86-world", "qemux86-world-lsb",
- "edgerouter", "edgerouter-lsb",
- "mpc8315e-rdb", "mpc8315e-rdb-lsb",
+ "qemux86-64-x32", "qemux86-world",
+ "edgerouter",
+ "mpc8315e-rdb",
"genericx86", "genericx86-lsb",
"genericx86-64", "genericx86-64-lsb",
"beaglebone", "beaglebone-lsb",
@@ -57,8 +60,12 @@ trigger_builders_wait = [
"check-layer"
]
-triggered_builders = trigger_builders_wait
-builders = ["nightly"] + triggered_builders
+trigger_builders_wait_full = [
+ "qemumips-lsb", "edgerouter-lsb", "mpc8315e-rdb-lsb", "qemuppc-lsb", "qemux86-world-lsb"
+]
+
+subbuilders = trigger_builders_wait_quick + trigger_builders_wait_full
+builders = ["quick", "full"]
# ## Cluster configuration
# Publishing settings
diff --git a/schedulers.py b/schedulers.py
index b6c1e24..1f2d96f 100644
--- a/schedulers.py
+++ b/schedulers.py
@@ -68,7 +68,7 @@ def props_for_builder(builder):
return props
-for builder in config.triggered_builders:
+for builder in config.subbuilders:
schedulers.append(sched.ForceScheduler(
name=builder,
builderNames=[builder],
@@ -81,14 +81,18 @@ for builder in config.triggered_builders:
buttonName="Force Build"))
# nightly builder triggers various other builders
-wait = sched.Triggerable(name="wait",
- builderNames=config.trigger_builders_wait)
-schedulers.append(wait)
-
-schedulers.append(sched.ForceScheduler(
- name="nightly",
- builderNames=["nightly"],
- buttonName="Start Nightly Build",
+wait_quick = sched.Triggerable(name="wait-quick",
+ builderNames=config.trigger_builders_wait_quick)
+schedulers.append(wait_quick)
+wait_full = sched.Triggerable(name="wait-full",
+ builderNames=config.trigger_builders_wait_quick + config.trigger_builders_wait_full)
+schedulers.append(wait_full)
+
+def parent_scheduler(target):
+ return sched.ForceScheduler(
+ name=target,
+ builderNames=[target],
+ buttonName="Start " + target + " Build",
codebases = [util.CodebaseParameter(codebase='', label="yocto-autobuilder-helper:", project=None)],
reason=util.StringParameter(
name="reason",
@@ -236,4 +240,8 @@ schedulers.append(sched.ForceScheduler(
name="deploy_artefacts",
label="Do we want to save build output? ",
default=False)
- ]+repos_for_builder("nightly")))
+ ]+repos_for_builder(target))
+
+schedulers.append(parent_scheduler("quick"))
+schedulers.append(parent_scheduler("full"))
+