1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
|
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
import os
from base64 import b64encode
import sys
import shutil
from zope.interface import implements
from twisted.internet import reactor, threads, defer
from twisted.python import log, failure, runtime
from buildslave.interfaces import ISlaveCommand
from buildslave import runprocess
from buildslave.exceptions import AbandonChain
from buildslave.commands import utils
from buildslave import util
# this used to be a CVS $-style "Revision" auto-updated keyword, but since I
# moved to Darcs as the primary repository, this is updated manually each
# time this file is changed. The last cvs_ver that was here was 1.51 .
command_version = "2.15"
# version history:
# >=1.17: commands are interruptable
# >=1.28: Arch understands 'revision', added Bazaar
# >=1.33: Source classes understand 'retry'
# >=1.39: Source classes correctly handle changes in branch (except Git)
# Darcs accepts 'revision' (now all do but Git) (well, and P4Sync)
# Arch/Baz should accept 'build-config'
# >=1.51: (release 0.7.3)
# >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open',
# and 'logfiles'. It now sends 'log' messages in addition to
# stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods,
# but these are not remotely callable yet.
# (not externally visible: ShellCommandPP has writeStdin/closeStdin.
# ShellCommand accepts new arguments (logfiles=, initialStdin=,
# keepStdinOpen=) and no longer accepts stdin=)
# (release 0.7.4)
# >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5)
# >= 2.3: added bzr (release 0.7.6)
# >= 2.4: Git understands 'revision' and branches
# >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2
# >= 2.6: added uploadDirectory
# >= 2.7: added usePTY option to SlaveShellCommand
# >= 2.8: added username and password args to SVN class
# >= 2.9: add depth arg to SVN class
# >= 2.10: CVS can handle 'extra_options' and 'export_options'
# >= 2.11: Arch, Bazaar, and Monotone removed
# >= 2.12: SlaveShellCommand no longer accepts 'keep_stdin_open'
# >= 2.13: SlaveFileUploadCommand supports option 'keepstamp'
# >= 2.14: RemoveDirectory can delete multiple directories
# >= 2.15: 'interruptSignal' option is added to SlaveShellCommand
class Command:
implements(ISlaveCommand)
"""This class defines one command that can be invoked by the build master.
The command is executed on the slave side, and always sends back a
completion message when it finishes. It may also send intermediate status
as it runs (by calling builder.sendStatus). Some commands can be
interrupted (either by the build master or a local timeout), in which
case the step is expected to complete normally with a status message that
indicates an error occurred.
These commands are used by BuildSteps on the master side. Each kind of
BuildStep uses a single Command. The slave must implement all the
Commands required by the set of BuildSteps used for any given build:
this is checked at startup time.
All Commands are constructed with the same signature:
c = CommandClass(builder, stepid, args)
where 'builder' is the parent SlaveBuilder object, and 'args' is a
dict that is interpreted per-command.
The setup(args) method is available for setup, and is run from __init__.
The Command is started with start(). This method must be implemented in a
subclass, and it should return a Deferred. When your step is done, you
should fire the Deferred (the results are not used). If the command is
interrupted, it should fire the Deferred anyway.
While the command runs. it may send status messages back to the
buildmaster by calling self.sendStatus(statusdict). The statusdict is
interpreted by the master-side BuildStep however it likes.
A separate completion message is sent when the deferred fires, which
indicates that the Command has finished, but does not carry any status
data. If the Command needs to return an exit code of some sort, that
should be sent as a regular status message before the deferred is fired .
Once builder.commandComplete has been run, no more status messages may be
sent.
If interrupt() is called, the Command should attempt to shut down as
quickly as possible. Child processes should be killed, new ones should
not be started. The Command should send some kind of error status update,
then complete as usual by firing the Deferred.
.interrupted should be set by interrupt(), and can be tested to avoid
sending multiple error status messages.
If .running is False, the bot is shutting down (or has otherwise lost the
connection to the master), and should not send any status messages. This
is checked in Command.sendStatus .
"""
# builder methods:
# sendStatus(dict) (zero or more)
# commandComplete() or commandInterrupted() (one, at end)
debug = False
interrupted = False
running = False # set by Builder, cleared on shutdown or when the
# Deferred fires
_reactor = reactor
def __init__(self, builder, stepId, args):
self.builder = builder
self.stepId = stepId # just for logging
self.args = args
self.startTime = None
self.setup(args)
def setup(self, args):
"""Override this in a subclass to extract items from the args dict."""
pass
def doStart(self):
self.running = True
self.startTime = util.now(self._reactor)
d = defer.maybeDeferred(self.start)
def commandComplete(res):
self.sendStatus({"elapsed": util.now(self._reactor) - self.startTime})
self.running = False
return res
d.addBoth(commandComplete)
return d
def start(self):
"""Start the command. This method should return a Deferred that will
fire when the command has completed. The Deferred's argument will be
ignored.
This method should be overridden by subclasses."""
raise NotImplementedError, "You must implement this in a subclass"
def sendStatus(self, status):
"""Send a status update to the master."""
if self.debug:
log.msg("sendStatus", status)
if not self.running:
log.msg("would sendStatus but not .running")
return
self.builder.sendUpdate(status)
def doInterrupt(self):
self.running = False
self.interrupt()
def interrupt(self):
"""Override this in a subclass to allow commands to be interrupted.
May be called multiple times, test and set self.interrupted=True if
this matters."""
pass
# utility methods, mostly used by SlaveShellCommand and the like
def _abandonOnFailure(self, rc):
if type(rc) is not int:
log.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \
(rc, type(rc)))
assert isinstance(rc, int)
if rc != 0:
raise AbandonChain(rc)
return rc
def _sendRC(self, res):
self.sendStatus({'rc': 0})
def _checkAbandoned(self, why):
log.msg("_checkAbandoned", why)
why.trap(AbandonChain)
log.msg(" abandoning chain", why.value)
self.sendStatus({'rc': why.value.args[0]})
return None
class SourceBaseCommand(Command):
"""Abstract base class for Version Control System operations (checkout
and update). This class extracts the following arguments from the
dictionary received from the master:
- ['workdir']: (required) the subdirectory where the buildable sources
should be placed
- ['mode']: one of update/copy/clobber/export, defaults to 'update'
- ['revision']: (required) If not None, this is an int or string which indicates
which sources (along a time-like axis) should be used.
It is the thing you provide as the CVS -r or -D
argument.
- ['patch']: If not None, this is a tuple of (striplevel, patch)
which contains a patch that should be applied after the
checkout has occurred. Once applied, the tree is no
longer eligible for use with mode='update', and it only
makes sense to use this in conjunction with a
['revision'] argument. striplevel is an int, and patch
is a string in standard unified diff format. The patch
will be applied with 'patch -p%d <PATCH', with
STRIPLEVEL substituted as %d. The command will fail if
the patch process fails (rejected hunks).
- ['timeout']: seconds of silence tolerated before we kill off the
command
- ['maxTime']: seconds before we kill off the command
- ['retry']: If not None, this is a tuple of (delay, repeats)
which means that any failed VC updates should be
reattempted, up to REPEATS times, after a delay of
DELAY seconds. This is intended to deal with slaves
that experience transient network failures.
"""
sourcedata = ""
def setup(self, args):
# if we need to parse the output, use this environment. Otherwise
# command output will be in whatever the buildslave's native language
# has been set to.
self.env = os.environ.copy()
self.env['LC_MESSAGES'] = "C"
self.checkoutdir = args['srcdir']
self.workdir = args['workdir']
self.mode = args.get('mode', "update")
self.revision = args.get('revision')
self.patch = args.get('patch')
self.timeout = args.get('timeout', 120)
self.maxTime = args.get('maxTime', None)
self.retry = args.get('retry')
self.logEnviron = args.get('logEnviron',True)
self._commandPaths = {}
# VC-specific subclasses should override this to extract more args.
# Make sure to upcall!
def getCommand(self, name):
"""Wrapper around utils.getCommand that will output a resonable
error message and raise AbandonChain if the command cannot be
found"""
if name not in self._commandPaths:
try:
self._commandPaths[name] = utils.getCommand(name)
except RuntimeError:
self.sendStatus({'stderr' : "could not find '%s'\n" % name})
self.sendStatus({'stderr' : "PATH is '%s'\n" % os.environ.get('PATH', '')})
raise AbandonChain(-1)
return self._commandPaths[name]
def start(self):
self.sendStatus({'header': "starting " + self.header + "\n"})
self.command = None
# self.srcdir is where the VC system should put the sources
if self.mode == "copy":
self.srcdir = "source" # hardwired directory name, sorry
else:
self.srcdir = self.workdir
self.sourcedatafile = os.path.join(self.builder.basedir,
".buildbot-sourcedata-" + b64encode(self.srcdir))
# upgrade older versions to the new sourcedata location
old_sd_path = os.path.join(self.builder.basedir, self.srcdir, ".buildbot-sourcedata")
if os.path.exists(old_sd_path) and not os.path.exists(self.sourcedatafile):
os.rename(old_sd_path, self.sourcedatafile)
# also upgrade versions that didn't include the encoded version of the
# source directory
old_sd_path = os.path.join(self.builder.basedir, ".buildbot-sourcedata")
if os.path.exists(old_sd_path) and not os.path.exists(self.sourcedatafile):
os.rename(old_sd_path, self.sourcedatafile)
d = defer.succeed(None)
self.maybeClobber(d)
if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()):
# the directory cannot be updated, so we have to clobber it.
# Perhaps the master just changed modes from 'export' to
# 'update'.
d.addCallback(self.doClobber, self.srcdir)
d.addCallback(self.doVC)
if self.mode == "copy":
d.addCallback(self.doCopy)
if self.mode == "movecopy":
d.addCallback(self.doCopy)
if self.patch:
d.addCallback(self.doPatch)
d.addCallbacks(self._sendRC, self._checkAbandoned)
return d
def maybeClobber(self, d):
# do we need to clobber anything?
if self.mode in ("copy", "clobber", "export"):
d.addCallback(self.doClobber, self.workdir)
def interrupt(self):
self.interrupted = True
if self.command:
self.command.kill("command interrupted")
def doVC(self, res):
if self.interrupted:
raise AbandonChain(1)
if self.sourcedirIsUpdateable() and self.sourcedataMatches():
d = self.doVCUpdate()
d.addBoth(self.maybeDoVCFallback)
else:
d = self.doVCFull()
d.addBoth(self.maybeDoVCRetry)
d.addCallback(self._abandonOnFailure)
d.addCallback(self._handleGotRevision)
d.addCallback(self.writeSourcedata)
return d
def sourcedataMatches(self):
try:
olddata = self.readSourcedata()
if olddata != self.sourcedata:
return False
except IOError:
return False
return True
def sourcedirIsPatched(self):
return os.path.exists(os.path.join(self.builder.basedir,
self.workdir,
".buildbot-patched"))
def _handleGotRevision(self, res):
d = defer.maybeDeferred(self.parseGotRevision)
d.addCallback(lambda got_revision:
self.sendStatus({'got_revision': got_revision}))
return d
def parseGotRevision(self):
"""Override this in a subclass. It should return a string that
represents which revision was actually checked out, or a Deferred
that will fire with such a string. If, in a future build, you were to
pass this 'got_revision' string in as the 'revision' component of a
SourceStamp, you should wind up with the same source code as this
checkout just obtained.
It is probably most useful to scan self.command.stdout for a string
of some sort. Be sure to set keepStdout=True on the VC command that
you run, so that you'll have something available to look at.
If this information is unavailable, just return None."""
return None
def readSourcedata(self):
"""
Read the sourcedata file and return its contents
@returns: source data
@raises: IOError if the file does not exist
"""
return open(self.sourcedatafile, "r").read()
def writeSourcedata(self, res):
open(self.sourcedatafile, "w").write(self.sourcedata)
return res
def sourcedirIsUpdateable(self):
"""Returns True if the tree can be updated."""
raise NotImplementedError("this must be implemented in a subclass")
def doVCUpdate(self):
"""Returns a deferred with the steps to update a checkout."""
raise NotImplementedError("this must be implemented in a subclass")
def doVCFull(self):
"""Returns a deferred with the steps to do a fresh checkout."""
raise NotImplementedError("this must be implemented in a subclass")
def maybeDoVCFallback(self, rc):
if type(rc) is int and rc == 0:
return rc
if self.interrupted:
raise AbandonChain(1)
# allow AssertionErrors to fall through, for benefit of the tests; for
# all other errors, carry on to try the fallback
if isinstance(rc, failure.Failure) and rc.check(AssertionError):
return rc
# Let VCS subclasses have an opportunity to handle
# unrecoverable errors without having to clobber the repo
self.maybeNotDoVCFallback(rc)
msg = "update failed, clobbering and trying again"
self.sendStatus({'header': msg + "\n"})
log.msg(msg)
d = self.doClobber(None, self.srcdir)
d.addCallback(self.doVCFallback2)
return d
def doVCFallback2(self, res):
msg = "now retrying VC operation"
self.sendStatus({'header': msg + "\n"})
log.msg(msg)
d = self.doVCFull()
d.addBoth(self.maybeDoVCRetry)
d.addCallback(self._abandonOnFailure)
return d
def maybeNotDoVCFallback(self, rc):
"""Override this in a subclass if you want to detect unrecoverable
checkout errors where clobbering the repo wouldn't help, and stop
the current VC chain before it clobbers the repo for future builds.
Use 'raise AbandonChain' to pass up a halt if you do detect such."""
pass
def maybeDoVCRetry(self, res):
"""We get here somewhere after a VC chain has finished. res could
be::
- 0: the operation was successful
- nonzero: the operation failed. retry if possible
- AbandonChain: the operation failed, someone else noticed. retry.
- Failure: some other exception, re-raise
"""
if isinstance(res, failure.Failure):
if self.interrupted:
return res # don't re-try interrupted builds
res.trap(AbandonChain)
else:
if type(res) is int and res == 0:
return res
if self.interrupted:
raise AbandonChain(1)
# if we get here, we should retry, if possible
if self.retry:
delay, repeats = self.retry
if repeats >= 0:
self.retry = (delay, repeats-1)
msg = ("update failed, trying %d more times after %d seconds"
% (repeats, delay))
self.sendStatus({'header': msg + "\n"})
log.msg(msg)
d = defer.Deferred()
# we are going to do a full checkout, so a clobber is
# required first
self.doClobber(d, self.workdir)
if self.srcdir:
self.doClobber(d, self.srcdir)
d.addCallback(lambda res: self.doVCFull())
d.addBoth(self.maybeDoVCRetry)
self._reactor.callLater(delay, d.callback, None)
return d
return res
def doClobber(self, dummy, dirname, chmodDone=False):
d = os.path.join(self.builder.basedir, dirname)
if runtime.platformType != "posix":
d = threads.deferToThread(utils.rmdirRecursive, d)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus({'header' : 'exception from rmdirRecursive\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
return d
command = ["rm", "-rf", d]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
# sendRC=0 means the rm command will send stdout/stderr to the
# master, but not the rc=0 when it finishes. That job is left to
# _sendRC
d = c.start()
# The rm -rf may fail if there is a left-over subdir with chmod 000
# permissions. So if we get a failure, we attempt to chmod suitable
# permissions and re-try the rm -rf.
if chmodDone:
d.addCallback(self._abandonOnFailure)
else:
d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname))
return d
def doClobberTryChmodIfFail(self, rc, dirname):
assert isinstance(rc, int)
if rc == 0:
return defer.succeed(0)
# Attempt a recursive chmod and re-try the rm -rf after.
command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, dirname)]
if sys.platform.startswith('freebsd'):
# Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a
# directory for which it doesn't have permission, before changing that
# permission) by running 'find' instead
command = ["find", os.path.join(self.builder.basedir, dirname),
'-exec', 'chmod', 'u+rwx', '{}', ';' ]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
d.addCallback(lambda dummy: self.doClobber(dummy, dirname, True))
return d
def doCopy(self, res):
# now copy tree to workdir
fromdir = os.path.join(self.builder.basedir, self.srcdir)
todir = os.path.join(self.builder.basedir, self.workdir)
if runtime.platformType != "posix":
d = threads.deferToThread(shutil.copytree, fromdir, todir)
def cb(_):
return 0 # rc=0
def eb(f):
self.sendStatus({'header' : 'exception from copytree\n' + f.getTraceback()})
return -1 # rc=-1
d.addCallbacks(cb, eb)
return d
if not os.path.exists(os.path.dirname(todir)):
os.makedirs(os.path.dirname(todir))
if os.path.exists(todir):
# I don't think this happens, but just in case..
log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir)
command = ['cp', '-R', '-P', '-p', fromdir, todir]
c = runprocess.RunProcess(self.builder, command, self.builder.basedir,
sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
logEnviron=self.logEnviron, usePTY=False)
self.command = c
d = c.start()
d.addCallback(self._abandonOnFailure)
return d
def doPatch(self, res):
patchlevel = self.patch[0]
diff = self.patch[1]
root = None
if len(self.patch) >= 3:
root = self.patch[2]
command = [
utils.getCommand("patch"),
'-p%d' % patchlevel,
'--remove-empty-files',
'--force',
'--forward',
'-i', '.buildbot-diff',
]
dir = os.path.join(self.builder.basedir, self.workdir)
# Mark the directory so we don't try to update it later, or at least try
# to revert first.
open(os.path.join(dir, ".buildbot-patched"), "w").write("patched\n")
# write the diff to a file, for reading later
open(os.path.join(dir, ".buildbot-diff"), "w").write(diff)
# Update 'dir' with the 'root' option. Make sure it is a subdirectory
# of dir.
if (root and
os.path.abspath(os.path.join(dir, root)
).startswith(os.path.abspath(dir))):
dir = os.path.join(dir, root)
# now apply the patch
c = runprocess.RunProcess(self.builder, command, dir,
sendRC=False, timeout=self.timeout,
maxTime=self.maxTime, logEnviron=self.logEnviron,
usePTY=False)
self.command = c
d = c.start()
# clean up the temp file
def cleanup(x):
try:
os.unlink(os.path.join(dir, ".buildbot-diff"))
except:
pass
return x
d.addBoth(cleanup)
d.addCallback(self._abandonOnFailure)
return d
def setFileContents(self, filename, contents):
"""Put the given C{contents} in C{filename}; this is a bit more
succinct than opening, writing, and closing, and has the advantage of
being patchable in tests. Note that the enclosing directory is
not automatically created, nor is this an "atomic" overwrite."""
f = open(filename, 'w')
f.write(contents)
f.close()
|