1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
|
# This file is part of Buildbot. Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members
from twisted.python import log
from buildbot.status import testresult
from buildbot.status.results import SUCCESS, FAILURE, WARNINGS, SKIPPED
from buildbot.process.buildstep import LogLineObserver, OutputProgressObserver
from buildbot.steps.shell import ShellCommand
try:
import cStringIO
StringIO = cStringIO
except ImportError:
import StringIO
import re
# BuildSteps that are specific to the Twisted source tree
class HLint(ShellCommand):
"""I run a 'lint' checker over a set of .xhtml files. Any deviations
from recommended style is flagged and put in the output log.
This step looks at .changes in the parent Build to extract a list of
Lore XHTML files to check."""
name = "hlint"
description = ["running", "hlint"]
descriptionDone = ["hlint"]
warnOnWarnings = True
warnOnFailure = True
# TODO: track time, but not output
warnings = 0
def __init__(self, python=None, **kwargs):
ShellCommand.__init__(self, **kwargs)
self.python = python
def start(self):
# create the command
htmlFiles = {}
for f in self.build.allFiles():
if f.endswith(".xhtml") and not f.startswith("sandbox/"):
htmlFiles[f] = 1
# remove duplicates
hlintTargets = htmlFiles.keys()
hlintTargets.sort()
if not hlintTargets:
return SKIPPED
self.hlintFiles = hlintTargets
c = []
if self.python:
c.append(self.python)
c += ["bin/lore", "-p", "--output", "lint"] + self.hlintFiles
self.setCommand(c)
# add an extra log file to show the .html files we're checking
self.addCompleteLog("files", "\n".join(self.hlintFiles)+"\n")
ShellCommand.start(self)
def commandComplete(self, cmd):
# TODO: remove the 'files' file (a list of .xhtml files that were
# submitted to hlint) because it is available in the logfile and
# mostly exists to give the user an idea of how long the step will
# take anyway).
lines = cmd.logs['stdio'].getText().split("\n")
warningLines = filter(lambda line:':' in line, lines)
if warningLines:
self.addCompleteLog("warnings", "".join(warningLines))
warnings = len(warningLines)
self.warnings = warnings
def evaluateCommand(self, cmd):
# warnings are in stdout, rc is always 0, unless the tools break
if cmd.didFail():
return FAILURE
if self.warnings:
return WARNINGS
return SUCCESS
def getText2(self, cmd, results):
if cmd.didFail():
return ["hlint"]
return ["%d hlin%s" % (self.warnings,
self.warnings == 1 and 't' or 'ts')]
def countFailedTests(output):
# start scanning 10kb from the end, because there might be a few kb of
# import exception tracebacks between the total/time line and the errors
# line
chunk = output[-10000:]
lines = chunk.split("\n")
lines.pop() # blank line at end
# lines[-3] is "Ran NN tests in 0.242s"
# lines[-2] is blank
# lines[-1] is 'OK' or 'FAILED (failures=1, errors=12)'
# or 'FAILED (failures=1)'
# or "PASSED (skips=N, successes=N)" (for Twisted-2.0)
# there might be other lines dumped here. Scan all the lines.
res = {'total': None,
'failures': 0,
'errors': 0,
'skips': 0,
'expectedFailures': 0,
'unexpectedSuccesses': 0,
}
for l in lines:
out = re.search(r'Ran (\d+) tests', l)
if out:
res['total'] = int(out.group(1))
if (l.startswith("OK") or
l.startswith("FAILED ") or
l.startswith("PASSED")):
# the extra space on FAILED_ is to distinguish the overall
# status from an individual test which failed. The lack of a
# space on the OK is because it may be printed without any
# additional text (if there are no skips,etc)
out = re.search(r'failures=(\d+)', l)
if out: res['failures'] = int(out.group(1))
out = re.search(r'errors=(\d+)', l)
if out: res['errors'] = int(out.group(1))
out = re.search(r'skips=(\d+)', l)
if out: res['skips'] = int(out.group(1))
out = re.search(r'expectedFailures=(\d+)', l)
if out: res['expectedFailures'] = int(out.group(1))
out = re.search(r'unexpectedSuccesses=(\d+)', l)
if out: res['unexpectedSuccesses'] = int(out.group(1))
# successes= is a Twisted-2.0 addition, and is not currently used
out = re.search(r'successes=(\d+)', l)
if out: res['successes'] = int(out.group(1))
return res
class TrialTestCaseCounter(LogLineObserver):
_line_re = re.compile(r'^(?:Doctest: )?([\w\.]+) \.\.\. \[([^\]]+)\]$')
numTests = 0
finished = False
def outLineReceived(self, line):
# different versions of Twisted emit different per-test lines with
# the bwverbose reporter.
# 2.0.0: testSlave (buildbot.test.test_runner.Create) ... [OK]
# 2.1.0: buildbot.test.test_runner.Create.testSlave ... [OK]
# 2.4.0: buildbot.test.test_runner.Create.testSlave ... [OK]
# Let's just handle the most recent version, since it's the easiest.
# Note that doctests create lines line this:
# Doctest: viff.field.GF ... [OK]
if self.finished:
return
if line.startswith("=" * 40):
self.finished = True
return
m = self._line_re.search(line.strip())
if m:
testname, result = m.groups()
self.numTests += 1
self.step.setProgress('tests', self.numTests)
UNSPECIFIED=() # since None is a valid choice
class Trial(ShellCommand):
"""
There are some class attributes which may be usefully overridden
by subclasses. 'trialMode' and 'trialArgs' can influence the trial
command line.
"""
name = "trial"
progressMetrics = ('output', 'tests', 'test.log')
# note: the slash only works on unix buildslaves, of course, but we have
# no way to know what the buildslave uses as a separator.
# TODO: figure out something clever.
logfiles = {"test.log": "_trial_temp/test.log"}
# we use test.log to track Progress at the end of __init__()
renderables = ['tests', 'jobs']
flunkOnFailure = True
python = None
trial = "trial"
trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer
# for Twisted-2.0.0 or 1.3.0, use ["-o"] instead
trialArgs = []
jobs = None
testpath = UNSPECIFIED # required (but can be None)
testChanges = False # TODO: needs better name
recurse = False
reactor = None
randomly = False
tests = None # required
def __init__(self, reactor=UNSPECIFIED, python=None, trial=None,
testpath=UNSPECIFIED,
tests=None, testChanges=None,
recurse=None, randomly=None,
trialMode=None, trialArgs=None, jobs=None,
**kwargs):
"""
@type testpath: string
@param testpath: use in PYTHONPATH when running the tests. If
None, do not set PYTHONPATH. Setting this to '.' will
cause the source files to be used in-place.
@type python: string (without spaces) or list
@param python: which python executable to use. Will form the start of
the argv array that will launch trial. If you use this,
you should set 'trial' to an explicit path (like
/usr/bin/trial or ./bin/trial). Defaults to None, which
leaves it out entirely (running 'trial args' instead of
'python ./bin/trial args'). Likely values are 'python',
['python2.2'], ['python', '-Wall'], etc.
@type trial: string
@param trial: which 'trial' executable to run.
Defaults to 'trial', which will cause $PATH to be
searched and probably find /usr/bin/trial . If you set
'python', this should be set to an explicit path (because
'python2.3 trial' will not work).
@type trialMode: list of strings
@param trialMode: a list of arguments to pass to trial, specifically
to set the reporting mode. This defaults to ['-to']
which means 'verbose colorless output' to the trial
that comes with Twisted-2.0.x and at least -2.1.0 .
Newer versions of Twisted may come with a trial
that prefers ['--reporter=bwverbose'].
@type trialArgs: list of strings
@param trialArgs: a list of arguments to pass to trial, available to
turn on any extra flags you like. Defaults to [].
@type jobs: integer
@param jobs: integer to be used as trial -j/--jobs option (for
running tests on several workers). Only supported
since Twisted-12.3.0.
@type tests: list of strings
@param tests: a list of test modules to run, like
['twisted.test.test_defer', 'twisted.test.test_process'].
If this is a string, it will be converted into a one-item
list.
@type testChanges: boolean
@param testChanges: if True, ignore the 'tests' parameter and instead
ask the Build for all the files that make up the
Changes going into this build. Pass these filenames
to trial and ask it to look for test-case-name
tags, running just the tests necessary to cover the
changes.
@type recurse: boolean
@param recurse: If True, pass the --recurse option to trial, allowing
test cases to be found in deeper subdirectories of the
modules listed in 'tests'. This does not appear to be
necessary when using testChanges.
@type reactor: string
@param reactor: which reactor to use, like 'gtk' or 'java'. If not
provided, the Twisted's usual platform-dependent
default is used.
@type randomly: boolean
@param randomly: if True, add the --random=0 argument, which instructs
trial to run the unit tests in a random order each
time. This occasionally catches problems that might be
masked when one module always runs before another
(like failing to make registerAdapter calls before
lookups are done).
@type kwargs: dict
@param kwargs: parameters. The following parameters are inherited from
L{ShellCommand} and may be useful to set: workdir,
haltOnFailure, flunkOnWarnings, flunkOnFailure,
warnOnWarnings, warnOnFailure, want_stdout, want_stderr,
timeout.
"""
ShellCommand.__init__(self, **kwargs)
if python:
self.python = python
if self.python is not None:
if type(self.python) is str:
self.python = [self.python]
for s in self.python:
if " " in s:
# this is not strictly an error, but I suspect more
# people will accidentally try to use python="python2.3
# -Wall" than will use embedded spaces in a python flag
log.msg("python= component '%s' has spaces")
log.msg("To add -Wall, use python=['python', '-Wall']")
why = "python= value has spaces, probably an error"
raise ValueError(why)
if trial:
self.trial = trial
if " " in self.trial:
raise ValueError("trial= value has spaces")
if trialMode is not None:
self.trialMode = trialMode
if trialArgs is not None:
self.trialArgs = trialArgs
if jobs is not None:
self.jobs = jobs
if testpath is not UNSPECIFIED:
self.testpath = testpath
if self.testpath is UNSPECIFIED:
raise ValueError("You must specify testpath= (it can be None)")
assert isinstance(self.testpath, str) or self.testpath is None
if reactor is not UNSPECIFIED:
self.reactor = reactor
if tests is not None:
self.tests = tests
if type(self.tests) is str:
self.tests = [self.tests]
if testChanges is not None:
self.testChanges = testChanges
#self.recurse = True # not sure this is necessary
if not self.testChanges and self.tests is None:
raise ValueError("Must either set testChanges= or provide tests=")
if recurse is not None:
self.recurse = recurse
if randomly is not None:
self.randomly = randomly
# build up most of the command, then stash it until start()
command = []
if self.python:
command.extend(self.python)
command.append(self.trial)
command.extend(self.trialMode)
if self.recurse:
command.append("--recurse")
if self.reactor:
command.append("--reactor=%s" % reactor)
if self.randomly:
command.append("--random=0")
command.extend(self.trialArgs)
self.command = command
if self.reactor:
self.description = ["testing", "(%s)" % self.reactor]
self.descriptionDone = ["tests"]
# commandComplete adds (reactorname) to self.text
else:
self.description = ["testing"]
self.descriptionDone = ["tests"]
# this counter will feed Progress along the 'test cases' metric
self.addLogObserver('stdio', TrialTestCaseCounter())
def setupEnvironment(self, cmd):
ShellCommand.setupEnvironment(self, cmd)
if self.testpath != None:
e = cmd.args['env']
if e is None:
cmd.args['env'] = {'PYTHONPATH': self.testpath}
else:
#this bit produces a list, which can be used
#by buildslave.runprocess.RunProcess
ppath = e.get('PYTHONPATH', self.testpath)
if isinstance(ppath, str):
ppath = [ppath]
if self.testpath not in ppath:
ppath.insert(0, self.testpath)
e['PYTHONPATH'] = ppath
def start(self):
# choose progressMetrics and logfiles based on whether trial is being
# run with multiple workers or not.
output_observer = OutputProgressObserver('test.log')
if self.jobs is not None:
self.jobs = int(self.jobs)
self.command.append("--jobs=%d" % self.jobs)
# using -j/--jobs flag produces more than one test log.
self.logfiles = {}
for i in xrange(self.jobs):
self.logfiles['test.%d.log' % i] = '_trial_temp/%d/test.log' % i
self.logfiles['err.%d.log' % i] = '_trial_temp/%d/err.log' % i
self.logfiles['out.%d.log' % i] = '_trial_temp/%d/out.log' % i
self.addLogObserver('test.%d.log' % i, output_observer)
else:
# this one just measures bytes of output in _trial_temp/test.log
self.addLogObserver('test.log', output_observer)
# now that self.build.allFiles() is nailed down, finish building the
# command
if self.testChanges:
for f in self.build.allFiles():
if f.endswith(".py"):
self.command.append("--testmodule=%s" % f)
else:
self.command.extend(self.tests)
log.msg("Trial.start: command is", self.command)
ShellCommand.start(self)
def commandComplete(self, cmd):
# figure out all status, then let the various hook functions return
# different pieces of it
# 'cmd' is the original trial command, so cmd.logs['stdio'] is the
# trial output. We don't have access to test.log from here.
output = cmd.logs['stdio'].getText()
counts = countFailedTests(output)
total = counts['total']
failures, errors = counts['failures'], counts['errors']
parsed = (total != None)
text = []
text2 = ""
if not cmd.didFail():
if parsed:
results = SUCCESS
if total:
text += ["%d %s" % \
(total,
total == 1 and "test" or "tests"),
"passed"]
else:
text += ["no tests", "run"]
else:
results = FAILURE
text += ["testlog", "unparseable"]
text2 = "tests"
else:
# something failed
results = FAILURE
if parsed:
text.append("tests")
if failures:
text.append("%d %s" % \
(failures,
failures == 1 and "failure" or "failures"))
if errors:
text.append("%d %s" % \
(errors,
errors == 1 and "error" or "errors"))
count = failures + errors
text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts'))
else:
text += ["tests", "failed"]
text2 = "tests"
if counts['skips']:
text.append("%d %s" % \
(counts['skips'],
counts['skips'] == 1 and "skip" or "skips"))
if counts['expectedFailures']:
text.append("%d %s" % \
(counts['expectedFailures'],
counts['expectedFailures'] == 1 and "todo"
or "todos"))
if 0: # TODO
results = WARNINGS
if not text2:
text2 = "todo"
if 0:
# ignore unexpectedSuccesses for now, but it should really mark
# the build WARNING
if counts['unexpectedSuccesses']:
text.append("%d surprises" % counts['unexpectedSuccesses'])
results = WARNINGS
if not text2:
text2 = "tests"
if self.reactor:
text.append(self.rtext('(%s)'))
if text2:
text2 = "%s %s" % (text2, self.rtext('(%s)'))
self.results = results
self.text = text
self.text2 = [text2]
def rtext(self, fmt='%s'):
if self.reactor:
rtext = fmt % self.reactor
return rtext.replace("reactor", "")
return ""
def addTestResult(self, testname, results, text, tlog):
if self.reactor is not None:
testname = (self.reactor,) + testname
tr = testresult.TestResult(testname, results, text, logs={'log': tlog})
#self.step_status.build.addTestResult(tr)
self.build.build_status.addTestResult(tr)
def createSummary(self, loog):
output = loog.getText()
problems = ""
sio = StringIO.StringIO(output)
warnings = {}
while 1:
line = sio.readline()
if line == "":
break
if line.find(" exceptions.DeprecationWarning: ") != -1:
# no source
warning = line # TODO: consider stripping basedir prefix here
warnings[warning] = warnings.get(warning, 0) + 1
elif (line.find(" DeprecationWarning: ") != -1 or
line.find(" UserWarning: ") != -1):
# next line is the source
warning = line + sio.readline()
warnings[warning] = warnings.get(warning, 0) + 1
elif line.find("Warning: ") != -1:
warning = line
warnings[warning] = warnings.get(warning, 0) + 1
if line.find("=" * 60) == 0 or line.find("-" * 60) == 0:
problems += line
problems += sio.read()
break
if problems:
self.addCompleteLog("problems", problems)
# now parse the problems for per-test results
pio = StringIO.StringIO(problems)
pio.readline() # eat the first separator line
testname = None
done = False
while not done:
while 1:
line = pio.readline()
if line == "":
done = True
break
if line.find("=" * 60) == 0:
break
if line.find("-" * 60) == 0:
# the last case has --- as a separator before the
# summary counts are printed
done = True
break
if testname is None:
# the first line after the === is like:
# EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase)
# SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer)
# FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile)
r = re.search(r'^([^:]+): (\w+) \(([\w\.]+)\)', line)
if not r:
# TODO: cleanup, if there are no problems,
# we hit here
continue
result, name, case = r.groups()
testname = tuple(case.split(".") + [name])
results = {'SKIPPED': SKIPPED,
'EXPECTED FAILURE': SUCCESS,
'UNEXPECTED SUCCESS': WARNINGS,
'FAILURE': FAILURE,
'ERROR': FAILURE,
'SUCCESS': SUCCESS, # not reported
}.get(result, WARNINGS)
text = result.lower().split()
loog = line
# the next line is all dashes
loog += pio.readline()
else:
# the rest goes into the log
loog += line
if testname:
self.addTestResult(testname, results, text, loog)
testname = None
if warnings:
lines = warnings.keys()
lines.sort()
self.addCompleteLog("warnings", "".join(lines))
def evaluateCommand(self, cmd):
return self.results
def getText(self, cmd, results):
return self.text
def getText2(self, cmd, results):
return self.text2
class RemovePYCs(ShellCommand):
name = "remove-.pyc"
command = ['find', '.', '-name', '*.pyc', '-exec', 'rm', '{}', ';']
description = ["removing", ".pyc", "files"]
descriptionDone = ["remove", ".pycs"]
|