1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
#
# SPDX-License-Identifier: GPL-2.0-only
#
from twisted.internet import defer
from buildbot.plugins import steps, util
from buildbot.process import buildstep, logobserver
from buildbot.process.results import Results, SUCCESS, FAILURE, CANCELLED, WARNINGS, SKIPPED, EXCEPTION, RETRY
from buildbot.steps import shell
from yoctoabb.steps.observer import RunConfigLogObserver, SimpleLogObserver
import json
import datetime
import os
import sys
from yoctoabb import config
maxsteps = 9
def get_publish_internal(props):
"""
Calculate the location to which artefacts should be published and store it
as a property for use by other workers.
"""
# Cache the value in the publish_detination property
dest = props.getProperty("publish_destination", "")
if dest:
return dest
if props.getProperty("is_release", False):
milestone = props.getProperty("milestone_number", "")
rc_number = props.getProperty("rc_number", "")
snapshot = ""
if milestone:
snapshot += "_" + milestone
if rc_number:
snapshot += "." + rc_number
rel_name = "yocto-" + props.getProperty("yocto_number", "") + snapshot
dest = os.path.join(config.publish_dest, "releases", rel_name)
else:
dest_base = os.path.join(config.publish_dest, 'non-release',
datetime.datetime.now().strftime("%Y%m%d"))
# We want to make sure that we aren't writing artefacts to a publish
# directory which already exists, therefore we keep a list of used
# publish paths to prevent re-use. We store that in a JSON file.
useddests = {}
# NOTE: we make a strong assumption here that this code lives in a
# directory which is an immediate child of the buildbot master's
# working directory.
basedir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"..")
persist = os.path.join(basedir, "pub_locations.json")
if os.path.exists(persist):
with open(persist) as f:
useddests = json.load(f)
rev = useddests.get(dest_base, "")
if rev: # incremenent and use
rev = int(rev) + 1
else: # use the base path and store rev 0
rev = 1
dest = "%s-%s" % (dest_base, rev)
# need to update the used destinations
useddests[dest_base] = rev
# save the info, overwriting the file if it exists
with open(persist, 'w') as out:
json.dump(useddests, out)
# set the destination as a property to be inherited by workers, so that
# all workers in a triggered set publish to the same location
props.setProperty("publish_destination", dest,
"get_publish_dest")
return dest
@util.renderer
def get_publish_dest(props):
deploy = props.getProperty("deploy_artefacts", False)
if not deploy:
return ""
return get_publish_internal(props)
@util.renderer
def get_publish_resultdir(props):
return get_publish_internal(props) + "/testresults"
@util.renderer
def get_publish_name(props):
dest = get_publish_internal(props)
if dest:
return os.path.basename(dest)
return dest
@util.renderer
def get_sstate_release_number(props):
"""
Uses the values submitted to the scheduler to determine the major number
of the release for the purposes of publishing per-major release
shared-state artefacts.
"""
release_number = props.getProperty("yocto_number")
if not release_number:
return ""
release_components = release_number.split('.', 3)
return '.'.join(release_components).strip('.')
def get_runconfig_command(posttrigger=False):
runconfig_command = [util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/run-config")]
if posttrigger:
runconfig_command.append(util.Interpolate("%(prop:buildername)s-posttrigger"))
else:
runconfig_command.append(util.Property("buildername"))
runconfig_command.extend([
util.Interpolate("%(prop:builddir)s/build/build"),
util.Interpolate("%(prop:branch_poky)s"),
util.Interpolate("%(prop:repo_poky)s"),
"--sstateprefix", get_sstate_release_number,
"--buildappsrcrev", util.Interpolate("%(prop:buildappsrcrev)s"),
"--publish-dir", get_publish_dest,
"--build-type", util.Interpolate("%(prop:build_type)s"),
"--workername", util.Interpolate("%(prop:workername)s"),
"--build-url", util.URLForBuild,
"--results-dir", get_publish_resultdir,
"--quietlogging"])
return runconfig_command
def get_buildlogs(maxsteps):
logfiles = {}
for i in range(1, maxsteps):
for j in ['a', 'b', 'c', 'd']:
logfiles["step" + str(i) + str(j)] = "build/command.log." + str(i) + str(j)
return logfiles
def get_runconfig_legacy_step(posttrigger):
step = RunConfigLogObserver(
command=get_runconfig_command(posttrigger),
name="run-config",
logfiles=get_buildlogs(maxsteps),
lazylogfiles=True,
maxsteps=maxsteps,
timeout=16200) # default of 1200s/20min is too short, use 4.5hrs
return step
def get_runconfig_step(name, stepname, phase, description, usepty, posttrigger):
step = SimpleLogObserver(
command=get_runconfig_command(posttrigger) + ['--stepname', stepname, '--phase', phase],
name=name,
description=description,
lazylogfiles=True,
usePTY=usepty,
timeout=16200) # default of 1200s/20min is too short, use 4.5hrs
return step
class RunConfigCheckSteps(shell.ShellCommand):
name = 'Check run-config steps to use'
descriptionDone = ['Checked which run-config approach to use']
haltOnFailure = False
flunkOnFailure = True
jsonFileName = util.Interpolate("%(prop:builddir)s/runconfig.json")
logfiles = {'json': jsonFileName}
def __init__(self, *args, **kwargs):
self.posttrigger = kwargs.pop("posttrigger")
self.command = get_runconfig_command(self.posttrigger)
self.command.append("--json-outputfile")
self.command.append(self.jsonFileName)
super().__init__(*args, **kwargs)
self.log_observer_json = logobserver.BufferLogObserver()
self.addLogObserver('json', self.log_observer_json)
@defer.inlineCallbacks
def run(self):
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# Steal from shell.py (new style)
stdio_log = yield self.getLog('json')
yield stdio_log.finish()
yield self.evaluateCommand(cmd)
return cmd.results()
def evaluateCommand(self, cmd):
# If the command fails, fall back to old style run-config execution
rc = cmd.results()
logLines = self.log_observer_json.getStdout()
jsonconfig = None
try:
jsonconfig = json.loads(logLines)
except Exception as ex:
self._addToLog('stderr', 'ERROR: unable to parse data, exception {}: {}'.format(ex.__class__, ex))
if rc == FAILURE or not jsonconfig:
steps = [get_runconfig_legacy_step(self.posttrigger)]
else:
steps = []
for s in jsonconfig:
if 'usepty' not in s:
s['usepty'] = False
if 'bbname' in s:
name = s['bbname']
else:
name = "run-config-" + s['name'] + "-" + s['phase']
steps.append(get_runconfig_step(name, s['name'], s['phase'], s['description'], s['usepty'], self.posttrigger))
self.build.addStepsAfterCurrentStep(steps)
return SUCCESS
@defer.inlineCallbacks
def _addToLog(self, logName, message):
try:
log = self.getLog(logName)
except KeyError:
log = yield self.addLog(logName)
log.addStdout(message)
class TargetPresent(shell.ShellCommand):
name = "Check if branch needs this target"
command=[util.Interpolate("%(prop:builddir)s/yocto-autobuilder-helper/scripts/target-present"), util.Property("buildername")]
def evaluateCommand(self, cmd):
# If the command fails, fall back to old style run-config execution
rc = cmd.results()
if rc != SUCCESS:
self.descriptionDone = "Target not present in branch configuration"
self.build.results = SKIPPED
# Run no further steps, terminate the build
self.build.terminate = True
return SKIPPED
return SUCCESS
@defer.inlineCallbacks
def run(self):
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
return self.evaluateCommand(cmd)
|
