diff options
Diffstat (limited to 'lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py')
| -rw-r--r-- | lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py | 435 |
1 files changed, 0 insertions, 435 deletions
diff --git a/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py b/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py deleted file mode 100644 index e51e8b90..00000000 --- a/lib/python2.7/site-packages/buildbot_slave-0.8.8-py2.7.egg/buildslave/commands/transfer.py +++ /dev/null @@ -1,435 +0,0 @@ -# This file is part of Buildbot. Buildbot is free software: you can -# redistribute it and/or modify it under the terms of the GNU General Public -# License as published by the Free Software Foundation, version 2. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more -# details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., 51 -# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Copyright Buildbot Team Members - -import os, tarfile, tempfile - -from twisted.python import log -from twisted.internet import defer - -from buildslave.commands.base import Command - -class TransferCommand(Command): - - def finished(self, res): - if self.debug: - log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) - - # don't use self.sendStatus here, since we may no longer be running - # if we have been interrupted - upd = {'rc': self.rc} - if self.stderr: - upd['stderr'] = self.stderr - self.builder.sendUpdate(upd) - return res - - def interrupt(self): - if self.debug: - log.msg('interrupted') - if self.interrupted: - return - self.rc = 1 - self.interrupted = True - # now we wait for the next trip around the loop. It abandon the file - # when it sees self.interrupted set. - - -class SlaveFileUploadCommand(TransferCommand): - """ - Upload a file from slave to build master - Arguments: - - - ['workdir']: base directory to use - - ['slavesrc']: name of the slave-side file to read from - - ['writer']: RemoteReference to a transfer._FileWriter object - - ['maxsize']: max size (in bytes) of file to write - - ['blocksize']: max size for each data block - - ['keepstamp']: whether to preserve file modified and accessed times - """ - debug = False - - def setup(self, args): - self.workdir = args['workdir'] - self.filename = args['slavesrc'] - self.writer = args['writer'] - self.remaining = args['maxsize'] - self.blocksize = args['blocksize'] - self.keepstamp = args.get('keepstamp', False) - self.stderr = None - self.rc = 0 - - def start(self): - if self.debug: - log.msg('SlaveFileUploadCommand started') - - # Open file - self.path = os.path.join(self.builder.basedir, - self.workdir, - os.path.expanduser(self.filename)) - accessed_modified = None - try: - if self.keepstamp: - accessed_modified = (os.path.getatime(self.path), - os.path.getmtime(self.path)) - - self.fp = open(self.path, 'rb') - if self.debug: - log.msg("Opened '%s' for upload" % self.path) - except: - self.fp = None - self.stderr = "Cannot open file '%s' for upload" % self.path - self.rc = 1 - if self.debug: - log.msg("Cannot open file '%s' for upload" % self.path) - - self.sendStatus({'header': "sending %s" % self.path}) - - d = defer.Deferred() - self._reactor.callLater(0, self._loop, d) - def _close_ok(res): - self.fp = None - d1 = self.writer.callRemote("close") - def _utime_ok(res): - return self.writer.callRemote("utime", accessed_modified) - if self.keepstamp: - d1.addCallback(_utime_ok) - return d1 - def _close_err(f): - self.rc = 1 - self.fp = None - # call remote's close(), but keep the existing failure - d1 = self.writer.callRemote("close") - def eb(f2): - log.msg("ignoring error from remote close():") - log.err(f2) - d1.addErrback(eb) - d1.addBoth(lambda _ : f) # always return _loop failure - return d1 - - d.addCallbacks(_close_ok, _close_err) - d.addBoth(self.finished) - return d - - def _loop(self, fire_when_done): - d = defer.maybeDeferred(self._writeBlock) - def _done(finished): - if finished: - fire_when_done.callback(None) - else: - self._loop(fire_when_done) - def _err(why): - fire_when_done.errback(why) - d.addCallbacks(_done, _err) - return None - - def _writeBlock(self): - """Write a block of data to the remote writer""" - - if self.interrupted or self.fp is None: - if self.debug: - log.msg('SlaveFileUploadCommand._writeBlock(): end') - return True - - length = self.blocksize - if self.remaining is not None and length > self.remaining: - length = self.remaining - - if length <= 0: - if self.stderr is None: - self.stderr = 'Maximum filesize reached, truncating file \'%s\'' \ - % self.path - self.rc = 1 - data = '' - else: - data = self.fp.read(length) - - if self.debug: - log.msg('SlaveFileUploadCommand._writeBlock(): '+ - 'allowed=%d readlen=%d' % (length, len(data))) - if len(data) == 0: - log.msg("EOF: callRemote(close)") - return True - - if self.remaining is not None: - self.remaining = self.remaining - len(data) - assert self.remaining >= 0 - d = self.writer.callRemote('write', data) - d.addCallback(lambda res: False) - return d - - -class SlaveDirectoryUploadCommand(SlaveFileUploadCommand): - debug = False - - def setup(self, args): - self.workdir = args['workdir'] - self.dirname = args['slavesrc'] - self.writer = args['writer'] - self.remaining = args['maxsize'] - self.blocksize = args['blocksize'] - self.compress = args['compress'] - self.stderr = None - self.rc = 0 - - def start(self): - if self.debug: - log.msg('SlaveDirectoryUploadCommand started') - - self.path = os.path.join(self.builder.basedir, - self.workdir, - os.path.expanduser(self.dirname)) - if self.debug: - log.msg("path: %r" % self.path) - - # Create temporary archive - fd, self.tarname = tempfile.mkstemp() - fileobj = os.fdopen(fd, 'w') - if self.compress == 'bz2': - mode='w|bz2' - elif self.compress == 'gz': - mode='w|gz' - else: - mode = 'w' - archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) - archive.add(self.path, '') - archive.close() - fileobj.close() - - # Transfer it - self.fp = open(self.tarname, 'rb') - - self.sendStatus({'header': "sending %s" % self.path}) - - d = defer.Deferred() - self._reactor.callLater(0, self._loop, d) - def unpack(res): - d1 = self.writer.callRemote("unpack") - def unpack_err(f): - self.rc = 1 - return f - d1.addErrback(unpack_err) - d1.addCallback(lambda ignored: res) - return d1 - d.addCallback(unpack) - d.addBoth(self.finished) - return d - - def finished(self, res): - self.fp.close() - os.remove(self.tarname) - return TransferCommand.finished(self, res) - - -class SlaveFileDownloadCommand(TransferCommand): - """ - Download a file from master to slave - Arguments: - - - ['workdir']: base directory to use - - ['slavedest']: name of the slave-side file to be created - - ['reader']: RemoteReference to a transfer._FileReader object - - ['maxsize']: max size (in bytes) of file to write - - ['blocksize']: max size for each data block - - ['mode']: access mode for the new file - """ - debug = False - - def setup(self, args): - self.workdir = args['workdir'] - self.filename = args['slavedest'] - self.reader = args['reader'] - self.bytes_remaining = args['maxsize'] - self.blocksize = args['blocksize'] - self.mode = args['mode'] - self.stderr = None - self.rc = 0 - - def start(self): - if self.debug: - log.msg('SlaveFileDownloadCommand starting') - - # Open file - self.path = os.path.join(self.builder.basedir, - self.workdir, - os.path.expanduser(self.filename)) - - dirname = os.path.dirname(self.path) - if not os.path.exists(dirname): - os.makedirs(dirname) - - try: - self.fp = open(self.path, 'wb') - if self.debug: - log.msg("Opened '%s' for download" % self.path) - if self.mode is not None: - # note: there is a brief window during which the new file - # will have the buildslave's default (umask) mode before we - # set the new one. Don't use this mode= feature to keep files - # private: use the buildslave's umask for that instead. (it - # is possible to call os.umask() before and after the open() - # call, but cleaning up from exceptions properly is more of a - # nuisance that way). - os.chmod(self.path, self.mode) - except IOError: - # TODO: this still needs cleanup - self.fp = None - self.stderr = "Cannot open file '%s' for download" % self.path - self.rc = 1 - if self.debug: - log.msg("Cannot open file '%s' for download" % self.path) - - d = defer.Deferred() - self._reactor.callLater(0, self._loop, d) - def _close(res): - # close the file, but pass through any errors from _loop - d1 = self.reader.callRemote('close') - d1.addErrback(log.err, 'while trying to close reader') - d1.addCallback(lambda ignored: res) - return d1 - d.addBoth(_close) - d.addBoth(self.finished) - return d - - def _loop(self, fire_when_done): - d = defer.maybeDeferred(self._readBlock) - def _done(finished): - if finished: - fire_when_done.callback(None) - else: - self._loop(fire_when_done) - def _err(why): - fire_when_done.errback(why) - d.addCallbacks(_done, _err) - return None - - def _readBlock(self): - """Read a block of data from the remote reader.""" - - if self.interrupted or self.fp is None: - if self.debug: - log.msg('SlaveFileDownloadCommand._readBlock(): end') - return True - - length = self.blocksize - if self.bytes_remaining is not None and length > self.bytes_remaining: - length = self.bytes_remaining - - if length <= 0: - if self.stderr is None: - self.stderr = "Maximum filesize reached, truncating file '%s'" \ - % self.path - self.rc = 1 - return True - else: - d = self.reader.callRemote('read', length) - d.addCallback(self._writeData) - return d - - def _writeData(self, data): - if self.debug: - log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' % - len(data)) - if len(data) == 0: - return True - - if self.bytes_remaining is not None: - self.bytes_remaining = self.bytes_remaining - len(data) - assert self.bytes_remaining >= 0 - self.fp.write(data) - return False - - def finished(self, res): - if self.fp is not None: - self.fp.close() - - return TransferCommand.finished(self, res) - -class SlaveDirectoryDownloadCommand(SlaveFileDownloadCommand): - debug = False - - def setup(self, args): - self.workdir = args['workdir'] - - self.dirname = os.path.expanduser(args['slavedest']) - self.reader = args['reader'] - self.bytes_remaining = args['maxsize'] - self.blocksize = args['blocksize'] - self.compress = args['compress'] - self.mode = args['mode'] - self.stderr = None - self.rc = 0 - - def start(self): - if self.debug: - log.msg('SlaveDirectoryDownloadCommand starting') - - if not os.path.exists(self.dirname): - os.makedirs(self.dirname) - - try: - fd, self.tarname = tempfile.mkstemp() - self.fp = os.fdopen(fd, 'w') - if self.debug: - log.msg("Opened '%s' for download" % self.tarname) - except IOError: - # TODO: this still needs cleanup - self.fp = None - self.stderr = "Cannot open file '%s' for download" % self.tarname - self.rc = 1 - if self.debug: - log.msg("Cannot open file '%s' for download" % self.tarname) - - d = defer.Deferred() - self._reactor.callLater(0, self._loop, d) - def _close(res): - # close the file, but pass through any errors from _loop - d1 = self.reader.callRemote('close') - d1.addErrback(log.err, 'while trying to close reader') - d1.addCallback(lambda ignored: res) - return d1 - d.addBoth(_close) - d.addBoth(self.finished) - return d - - def finished(self, res): - if self.fp is not None: - self.fp.close() - - # decompress - if self.compress == 'bz2': - mode='r|bz2' - elif self.compress == 'gz': - mode='r|gz' - else: - mode = 'r' - if not hasattr(tarfile.TarFile, 'extractall'): - tarfile.TarFile.extractall = _extractall - archive = tarfile.open(name=self.tarname, mode=mode) - archive.extractall(path=self.dirname) - archive.close() - os.remove(self.tarname) - - # note: there is a brief window during which the new file - # will have the buildslave's default (umask) mode before we - # set the new one. Don't use this mode= feature to keep files - # private: use the buildslave's umask for that instead. (it - # is possible to call os.umask() before and after the open() - # call, but cleaning up from exceptions properly is more of a - # nuisance that way). - if self.mode is not None: - for f in os.listdir(self.dirname): - full_f = os.path.join(self.dirname, f) - os.chmod(full_f, self.mode) - - return TransferCommand.finished(self, res) |