path: root/lib/python2.7/site-packages/buildbot-0.8.8-py2.7.egg/buildbot/changes/mail.py

# This file is part of Buildbot.  Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members

"""
Parse various kinds of 'CVS notify' email.
"""
import re
import time, calendar
import datetime
from email import message_from_file
from email.Utils import parseaddr, parsedate_tz, mktime_tz
from email.Iterators import body_line_iterator

from zope.interface import implements
from twisted.python import log
from twisted.internet import defer
from buildbot import util
from buildbot.interfaces import IChangeSource
from buildbot.util.maildir import MaildirService

class MaildirSource(MaildirService, util.ComparableMixin):
    """Generic base class for Maildir-based change sources"""
    implements(IChangeSource)

    compare_attrs = ["basedir", "pollinterval", "prefix"]

    def __init__(self, maildir, prefix=None, category='', repository=''):
        MaildirService.__init__(self, maildir)
        self.prefix = prefix
        self.category = category
        self.repository = repository
        if prefix and not prefix.endswith("/"):
            log.msg("%s: you probably want your prefix=('%s') to end with "
                    "a slash")

    def describe(self):
        return "%s watching maildir '%s'" % (self.__class__.__name__, self.basedir)

    def messageReceived(self, filename):
        d = defer.succeed(None)
        def parse_file(_):
            f = self.moveToCurDir(filename)
            return self.parse_file(f, self.prefix)
        d.addCallback(parse_file)

        def add_change(chtuple):
            src, chdict = None, None
            if chtuple:
                src, chdict = chtuple
            if chdict:
                return self.master.addChange(src=src, **chdict)
            else:
                log.msg("no change found in maildir file '%s'" % filename)
        d.addCallback(add_change)

        return d

    def parse_file(self, fd, prefix=None):
        m = message_from_file(fd)
        return self.parse(m, prefix)

class CVSMaildirSource(MaildirSource):
    name = "CVSMaildirSource"

    def __init__(self, maildir, prefix=None, category='',
                 repository='', properties={}):
        MaildirSource.__init__(self, maildir, prefix, category, repository)
        self.properties = properties
        
    def parse(self, m, prefix=None):
        """Parse messages sent by the 'buildbot-cvs-mail' program.
        """
        # The mail is sent from the person doing the checkin. Assume that the
        # local username is enough to identify them (this assumes a one-server
        # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
        # model)
        name, addr = parseaddr(m["from"])
        if not addr:
            return None # no From means this message isn't from buildbot-cvs-mail
        at = addr.find("@")
        if at == -1:
            author = addr # might still be useful
        else:
            author = addr[:at]

        # CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as
        # part of the mail header, so use that.
        # This assumes cvs is being access via ssh or pserver, so the time
        # will be the CVS server's time.
        
        # calculate a "revision" based on that timestamp, or the current time
        # if we're unable to parse the date.
        log.msg('Processing CVS mail')
        dateTuple = parsedate_tz(m["date"])
        if dateTuple == None:
            when = util.now()
        else:
            when = mktime_tz(dateTuple)
            
        theTime =  datetime.datetime.utcfromtimestamp(float(when))
        rev = theTime.strftime('%Y-%m-%d %H:%M:%S')

        catRE           = re.compile( '^Category:\s*(\S.*)')
        cvsRE           = re.compile( '^CVSROOT:\s*(\S.*)')
        cvsmodeRE       = re.compile( '^Cvsmode:\s*(\S.*)')
        filesRE         = re.compile( '^Files:\s*(\S.*)')
        modRE           = re.compile( '^Module:\s*(\S.*)')
        pathRE          = re.compile( '^Path:\s*(\S.*)')
        projRE          = re.compile( '^Project:\s*(\S.*)')
        singleFileRE    = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
        tagRE           = re.compile( '^\s+Tag:\s*(\S.*)')
        updateRE        = re.compile( '^Update of:\s*(\S.*)')
        comments = ""
        branch = None
        cvsroot = None
        fileList = None
        files = []
        isdir = 0
        path = None
        project = None

        lines = list(body_line_iterator(m))
        while lines:
            line = lines.pop(0)
            m = catRE.match(line)
            if m:
                category = m.group(1)
                continue
            m = cvsRE.match(line)
            if m:
                cvsroot = m.group(1)
                continue
            m = cvsmodeRE.match(line)
            if m:
                cvsmode = m.group(1)
                continue
            m = filesRE.match(line)
            if m:
                fileList = m.group(1)
                continue
            m = modRE.match(line)
            if m:
                # We don't actually use this
                #module = m.group(1)
                continue
            m = pathRE.match(line)
            if m:
                path = m.group(1)
                continue
            m = projRE.match(line)
            if m:
                project = m.group(1)
                continue
            m = tagRE.match(line)
            if m:
                branch = m.group(1)
                continue
            m = updateRE.match(line)
            if m:
                # We don't actually use this
                #updateof = m.group(1)
                continue
            if line == "Log Message:\n":
                break

        # CVS 1.11 lists files as:
        #   repo/path file,old-version,new-version file2,old-version,new-version
        # Version 1.12 lists files as:
        #   file1 old-version new-version file2 old-version new-version
        # 
        # files consists of tuples of 'file-name old-version new-version'
        # The versions are either dotted-decimal version numbers, ie 1.1
        # or NONE. New files are of the form 'NONE NUMBER', while removed
        # files are 'NUMBER NONE'. 'NONE' is a literal string
        # Parsing this instead of files list in 'Added File:' etc
        # makes it possible to handle files with embedded spaces, though
        # it could fail if the filename was 'bad 1.1 1.2'
        # For cvs version 1.11, we expect
        #  my_module new_file.c,NONE,1.1
        #  my_module removed.txt,1.2,NONE
        #  my_module modified_file.c,1.1,1.2
        # While cvs version 1.12 gives us       
        #  new_file.c NONE 1.1
        #  removed.txt 1.2 NONE
        #  modified_file.c 1.1,1.2

        if fileList is None:
           log.msg('CVSMaildirSource Mail with no files. Ignoring')
           return None       # We don't have any files. Email not from CVS

        if cvsmode == '1.11':
            # Please, no repo paths with spaces!
            m = re.search('([^ ]*) ', fileList)
            if m:
                path = m.group(1)
            else:
                log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail')
                return
            fileList = fileList[len(path):].strip()
            singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
        elif cvsmode == '1.12':
            singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
            if path is None:
                raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config')
        else:
            raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode)
        
        log.msg("CVSMaildirSource processing filelist: %s" % fileList)
        while(fileList):
            m = singleFileRE.match(fileList)
            if m:
                curFile = path + '/' + m.group(1)
                files.append( curFile )
                fileList = fileList[m.end():]
            else:
                log.msg('CVSMaildirSource no files matched regex. Ignoring')
                return None   # bail - we couldn't parse the files that changed
        # Now get comments    
        while lines:
            line = lines.pop(0)
            comments += line
            
        comments = comments.rstrip() + "\n"
        if comments == '\n':
            comments = None
        return ('cvs', dict(author=author, files=files, comments=comments,
                            isdir=isdir, when=when, branch=branch,
                            revision=rev, category=category,
                            repository=cvsroot, project=project,
                            properties=self.properties))

# svn "commit-email.pl" handler.  The format is very similar to freshcvs mail;
# here's a sample:

#  From: username [at] apache.org    [slightly obfuscated to avoid spam here]
#  To: commits [at] spamassassin.apache.org
#  Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail
#  ...
#
#  Author: username
#  Date: Sat Nov 20 00:17:49 2004      [note: TZ = local tz on server!]
#  New Revision: 105955
#
#  Modified:   [also Removed: and Added:]
#    [filename]
#    ...
#  Log:
#  [log message]
#  ...
#
#
#  Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm
#  [unified diff]
#
#  [end of mail]

class SVNCommitEmailMaildirSource(MaildirSource):
    name = "SVN commit-email.pl"

    def parse(self, m, prefix=None):
        """Parse messages sent by the svn 'commit-email.pl' trigger.
        """

        # The mail is sent from the person doing the checkin. Assume that the
        # local username is enough to identify them (this assumes a one-server
        # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
        # model)
        name, addr = parseaddr(m["from"])
        if not addr:
            return None # no From means this message isn't from svn
        at = addr.find("@")
        if at == -1:
            author = addr # might still be useful
        else:
            author = addr[:at]

        # we take the time of receipt as the time of checkin. Not correct (it
        # depends upon the email latency), but it avoids the
        # out-of-order-changes issue. Also syncmail doesn't give us anything
        # better to work with, unless you count pulling the v1-vs-v2
        # timestamp out of the diffs, which would be ugly. TODO: Pulling the
        # 'Date:' header from the mail is a possibility, and
        # email.Utils.parsedate_tz may be useful. It should be configurable,
        # however, because there are a lot of broken clocks out there.
        when = util.now()

        files = []
        comments = ""
        lines = list(body_line_iterator(m))
        rev = None
        while lines:
            line = lines.pop(0)

            # "Author: jmason"
            match = re.search(r"^Author: (\S+)", line)
            if match:
                author = match.group(1)

            # "New Revision: 105955"
            match = re.search(r"^New Revision: (\d+)", line)
            if match:
                rev = match.group(1)

            # possible TODO: use "Date: ..." data here instead of time of
            # commit message receipt, above. however, this timestamp is
            # specified *without* a timezone, in the server's local TZ, so to
            # be accurate buildbot would need a config setting to specify the
            # source server's expected TZ setting! messy.

            # this stanza ends with the "Log:"
            if (line == "Log:\n"):
                break

        # commit message is terminated by the file-listing section
        while lines:
            line = lines.pop(0)
            if (line == "Modified:\n" or
                line == "Added:\n" or
                line == "Removed:\n"):
                break
            comments += line
        comments = comments.rstrip() + "\n"

        while lines:
            line = lines.pop(0)
            if line == "\n":
                break
            if line.find("Modified:\n") == 0:
                continue            # ignore this line
            if line.find("Added:\n") == 0:
                continue            # ignore this line
            if line.find("Removed:\n") == 0:
                continue            # ignore this line
            line = line.strip()

            thesefiles = line.split(" ")
            for f in thesefiles:
                if prefix:
                    # insist that the file start with the prefix: we may get
                    # changes we don't care about too
                    if f.startswith(prefix):
                        f = f[len(prefix):]
                    else:
                        log.msg("ignored file from svn commit: prefix '%s' "
                                "does not match filename '%s'" % (prefix, f))
                        continue

                # TODO: figure out how new directories are described, set
                # .isdir
                files.append(f)

        if not files:
            log.msg("no matching files found, ignoring commit")
            return None

        return ('svn', dict(author=author, files=files, comments=comments,
                            when=when, revision=rev))

# bzr Launchpad branch subscription mails. Sample mail:
#
#   From: noreply@launchpad.net
#   Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file
#   To: Joe <joe@acme.com>
#   ...
#   
#   ------------------------------------------------------------
#   revno: 2701
#   committer: Joe <joe@acme.com>
#   branch nick: tmpbb
#   timestamp: Fri 2009-05-15 10:35:43 +0200
#   message:
#     test add file
#   added:
#     test-add-file
#   
#   
#   --
#   
#   https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test
#   
#   You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test.
#   To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription.
# 
# [end of mail]

class BzrLaunchpadEmailMaildirSource(MaildirSource):
    name = "Launchpad"

    compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"]

    def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):
        self.branchMap = branchMap
        self.defaultBranch = defaultBranch
        MaildirSource.__init__(self, maildir, prefix, **kwargs)

    def parse(self, m, prefix=None):
        """Parse branch notification messages sent by Launchpad.
        """

        subject = m["subject"]
        match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
        if match:
            repository = match.group(1)
        else:
            repository = None

        # Put these into a dictionary, otherwise we cannot assign them
        # from nested function definitions.
        d = { 'files': [], 'comments': u"" }
        gobbler = None
        rev = None
        author = None
        when = util.now()
        def gobble_comment(s):
            d['comments'] += s + "\n"
        def gobble_removed(s):
            d['files'].append('%s REMOVED' % s)
        def gobble_added(s):
            d['files'].append('%s ADDED' % s)
        def gobble_modified(s):
            d['files'].append('%s MODIFIED' % s)
        def gobble_renamed(s):
            match = re.search(r"^(.+) => (.+)$", s)
            if match:
                d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2)))
            else:
                d['files'].append('%s RENAMED' % s)

        lines = list(body_line_iterator(m, True))
        rev = None
        while lines:
            line = unicode(lines.pop(0), "utf-8", errors="ignore")

            # revno: 101
            match = re.search(r"^revno: ([0-9.]+)", line)
            if match:
                rev = match.group(1)

            # committer: Joe <joe@acme.com>
            match = re.search(r"^committer: (.*)$", line)
            if match:
                author = match.group(1)

            # timestamp: Fri 2009-05-15 10:35:43 +0200
            # datetime.strptime() is supposed to support %z for time zone, but
            # it does not seem to work. So handle the time zone manually.
            match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line)
            if match:
                datestr = match.group(1)
                tz_sign = match.group(2)
                tz_hours = match.group(3)
                tz_minutes = match.group(4)
                when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes)

            if re.search(r"^message:\s*$", line):
                gobbler = gobble_comment
            elif re.search(r"^removed:\s*$", line):
                gobbler = gobble_removed
            elif re.search(r"^added:\s*$", line):
                gobbler = gobble_added
            elif re.search(r"^renamed:\s*$", line):
                gobbler = gobble_renamed
            elif re.search(r"^modified:\s*$", line):
                gobbler = gobble_modified
            elif re.search(r"^  ", line) and gobbler:
                gobbler(line[2:-1]) # Use :-1 to gobble trailing newline

        # Determine the name of the branch.
        branch = None
        if self.branchMap and repository:
            if self.branchMap.has_key(repository):
                branch = self.branchMap[repository]
            elif self.branchMap.has_key('lp:' + repository):
                branch = self.branchMap['lp:' + repository]
        if not branch:
            if self.defaultBranch:
                branch = self.defaultBranch
            else:
                if repository:
                    branch = 'lp:' + repository
                else:
                    branch = None

        if rev and author:
            return ('bzr', dict(author=author, files=d['files'],
                                comments=d['comments'],
                                when=when, revision=rev,
                                branch=branch, repository=repository or ''))
        else:
            return None

def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):
    time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
    tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes)
    return time_no_tz - tz_delta