[PATCH v3 04/10] parsemail: Move parsing to 'parser'
Stephen Finucane
stephenfinucane at hotmail.com
Tue Aug 30 08:50:24 AEST 2016
From: Stephen Finucane <stephen.finucane at intel.com>
Separate the parsing of mails from the CLI aspects of parsemail. Since
there is already a 'parser' module, it makes sense to place the parsing
functionality here. This will allow for additional uses of this parsing
functionality in the future.
Signed-off-by: Stephen Finucane <stephen.finucane at intel.com>
Reviewed-by: Andy Doan <andy.doan at linaro.org>
---
patchwork/bin/parsearchive.py | 4 +-
patchwork/bin/parsemail.py | 525 +---------------------------------------
patchwork/parser.py | 530 ++++++++++++++++++++++++++++++++++++++++-
patchwork/tests/test_parser.py | 17 +-
4 files changed, 540 insertions(+), 536 deletions(-)
diff --git a/patchwork/bin/parsearchive.py b/patchwork/bin/parsearchive.py
index 30bca13..8986b22 100755
--- a/patchwork/bin/parsearchive.py
+++ b/patchwork/bin/parsearchive.py
@@ -29,7 +29,7 @@ import mailbox
import django
-from patchwork.bin import parsemail
+from patchwork.parser import parse_mail
from patchwork import models
LOGGER = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ def parse_mbox(path, list_id):
mbox = mailbox.mbox(path)
for msg in mbox:
try:
- obj = parsemail.parse_mail(msg, list_id)
+ obj = parse_mail(msg, list_id)
if obj:
results[type(obj)] += 1
else:
diff --git a/patchwork/bin/parsemail.py b/patchwork/bin/parsemail.py
index 8913076..b35b1f6 100755
--- a/patchwork/bin/parsemail.py
+++ b/patchwork/bin/parsemail.py
@@ -22,29 +22,15 @@
from __future__ import absolute_import
import argparse
-import codecs
-import datetime
from email import message_from_file
-from email.header import Header, decode_header
-from email.utils import parsedate_tz, mktime_tz
-from fnmatch import fnmatch
-from functools import reduce
import logging
-import operator
-import re
import sys
import django
from django.conf import settings
-from django.contrib.auth.models import User
from django.utils.log import AdminEmailHandler
-from django.utils import six
-from django.utils.six.moves import map
-from patchwork.models import (Patch, Project, Person, Comment, State,
- DelegationRule, Submission, CoverLetter,
- get_default_initial_patch_state)
-from patchwork.parser import parse_patch, find_filenames
+from patchwork.parser import parse_mail
LOGGER = logging.getLogger(__name__)
@@ -56,515 +42,6 @@ VERBOSITY_LEVELS = {
'critical': logging.CRITICAL
}
-list_id_headers = ['List-ID', 'X-Mailing-List', 'X-list']
-
-
-def normalise_space(str):
- whitespace_re = re.compile(r'\s+')
- return whitespace_re.sub(' ', str).strip()
-
-
-def clean_header(header):
- """Decode (possibly non-ascii) headers."""
- def decode(fragment):
- (frag_str, frag_encoding) = fragment
- if frag_encoding:
- return frag_str.decode(frag_encoding)
- elif isinstance(frag_str, six.binary_type): # python 2
- return frag_str.decode()
- return frag_str
-
- fragments = list(map(decode, decode_header(header)))
-
- return normalise_space(u' '.join(fragments))
-
-
-def find_project_by_id(list_id):
- """Find a `project` object with given `list_id`."""
- project = None
- try:
- project = Project.objects.get(listid=list_id)
- except Project.DoesNotExist:
- pass
- return project
-
-
-def find_project_by_header(mail):
- project = None
- listid_res = [re.compile(r'.*<([^>]+)>.*', re.S),
- re.compile(r'^([\S]+)$', re.S)]
-
- for header in list_id_headers:
- if header in mail:
-
- for listid_re in listid_res:
- match = listid_re.match(mail.get(header))
- if match:
- break
-
- if not match:
- continue
-
- listid = match.group(1)
-
- project = find_project_by_id(listid)
- if project:
- break
-
- return project
-
-
-def find_author(mail):
-
- from_header = clean_header(mail.get('From'))
- name, email = (None, None)
-
- # tuple of (regex, fn)
- # - where fn returns a (name, email) tuple from the match groups resulting
- # from re.match().groups()
- from_res = [
- # for "Firstname Lastname" <example at example.com> style addresses
- (re.compile(r'"?(.*?)"?\s*<([^>]+)>'), (lambda g: (g[0], g[1]))),
-
- # for example at example.com (Firstname Lastname) style addresses
- (re.compile(r'"?(.*?)"?\s*\(([^\)]+)\)'), (lambda g: (g[1], g[0]))),
-
- # for example at example.com (Firstname Lastname) style addresses
- (re.compile(r'(.*?)\sat\s(.*?)\s*\(([^\)]+)\)'),
- (lambda g: (g[2], '@'.join(g[0:2])))),
-
- # everything else
- (re.compile(r'(.*)'), (lambda g: (None, g[0]))),
- ]
-
- for regex, fn in from_res:
- match = regex.match(from_header)
- if match:
- (name, email) = fn(match.groups())
- break
-
- if email is None:
- raise ValueError("Invalid 'From' header")
-
- email = email.strip()
- if name is not None:
- name = name.strip()
-
- try:
- person = Person.objects.get(email__iexact=email)
- if name: # use the latest provided name
- person.name = name
- except Person.DoesNotExist:
- person = Person(name=name, email=email)
-
- return person
-
-
-def find_date(mail):
- t = parsedate_tz(mail.get('Date', ''))
- if not t:
- return datetime.datetime.utcnow()
- return datetime.datetime.utcfromtimestamp(mktime_tz(t))
-
-
-def find_headers(mail):
- return reduce(operator.__concat__,
- ['%s: %s\n' % (k, Header(v, header_name=k,
- continuation_ws='\t').encode())
- for (k, v) in list(mail.items())])
-
-
-def find_pull_request(content):
- git_re = re.compile(r'^The following changes since commit.*' +
- r'^are available in the git repository at:\n'
- r'^\s*([\S]+://[^\n]+)$',
- re.DOTALL | re.MULTILINE)
- match = git_re.search(content)
- if match:
- return match.group(1)
- return None
-
-
-def find_references(mail):
- """Construct a list of possible reply message ids."""
- refs = []
-
- if 'In-Reply-To' in mail:
- refs.append(mail.get('In-Reply-To'))
-
- if 'References' in mail:
- rs = mail.get('References').split()
- rs.reverse()
- for r in rs:
- if r not in refs:
- refs.append(r)
-
- return refs
-
-
-def parse_series_marker(subject_prefixes):
- """Extract series markers from subject.
-
- Extract the markers of multi-patches series, i.e. 'x/n', from the
- provided subject series.
-
- Args:
- subject_prefixes: List of subject prefixes to extract markers
- from
-
- Returns:
- (x, n) if markers found, else (None, None)
- """
-
- regex = re.compile('^([0-9]+)/([0-9]+)$')
- for prefix in subject_prefixes:
- m = regex.match(prefix)
- if not m:
- continue
- return (int(m.group(1)), int(m.group(2)))
- return (None, None)
-
-
-def find_content(project, mail):
- patchbuf = None
- commentbuf = ''
-
- for part in mail.walk():
- if part.get_content_maintype() != 'text':
- continue
-
- payload = part.get_payload(decode=True)
- subtype = part.get_content_subtype()
-
- if not isinstance(payload, six.text_type):
- charset = part.get_content_charset()
-
- # Check that we have a charset that we understand. Otherwise,
- # ignore it and fallback to our standard set.
- if charset is not None:
- try:
- codecs.lookup(charset)
- except LookupError:
- charset = None
-
- # If there is no charset or if it is unknown, then try some common
- # charsets before we fail.
- if charset is None:
- try_charsets = ['utf-8', 'windows-1252', 'iso-8859-1']
- else:
- try_charsets = [charset]
-
- for cset in try_charsets:
- try:
- payload = six.text_type(payload, cset)
- break
- except UnicodeDecodeError:
- payload = None
-
- # Could not find a valid decoded payload. Fail.
- if payload is None:
- return None, None
-
- if subtype in ['x-patch', 'x-diff']:
- patchbuf = payload
- elif subtype == 'plain':
- c = payload
-
- if not patchbuf:
- patchbuf, c = parse_patch(payload)
-
- if c is not None:
- commentbuf += c.strip() + '\n'
-
- commentbuf = clean_content(commentbuf)
-
- return patchbuf, commentbuf
-
-
-def find_submission_for_comment(project, refs):
- for ref in refs:
- # first, check for a direct reply
- try:
- submission = Submission.objects.get(project=project, msgid=ref)
- return submission
- except Submission.DoesNotExist:
- pass
-
- # see if we have comments that refer to a patch
- try:
- comment = Comment.objects.get(submission__project=project,
- msgid=ref)
- return comment.submission
- except Comment.MultipleObjectsReturned:
- # NOTE(stephenfin): This is a artifact of prior lack of support
- # for cover letters in Patchwork. Previously all replies to
- # patches were saved as comments. However, it's possible that
- # someone could have created a new series as a reply to one of the
- # comments on the original patch series. For example,
- # '2015-November/002096.html' from the Patchwork archives. In this
- # case, reparsing the archives will result in creation of a cover
- # letter with the same message ID as the existing comment. Follow
- # up comments will then apply to both this cover letter and the
- # linked patch from the comment previously created. We choose to
- # apply the comment to the cover letter. Note that this only
- # happens when running 'parsearchive' or similar, so it should not
- # affect every day use in any way.
- comments = Comment.objects.filter(submission__project=project,
- msgid=ref)
- # The latter item will be the cover letter
- return comments.reverse()[0].submission
- except Comment.DoesNotExist:
- pass
-
- return None
-
-
-def split_prefixes(prefix):
- """Turn a prefix string into a list of prefix tokens."""
- split_re = re.compile(r'[,\s]+')
- matches = split_re.split(prefix)
-
- return [s for s in matches if s != '']
-
-
-def clean_subject(subject, drop_prefixes=None):
- """Clean a Subject: header from an incoming patch.
-
- Removes Re: and Fwd: strings, as well as [PATCH]-style prefixes. By
- default, only [PATCH] is removed, and we keep any other bracketed
- data in the subject. If drop_prefixes is provided, remove those
- too, comparing case-insensitively.
-
- Args:
- subject: Subject to be cleaned
- drop_prefixes: Additional, case-insensitive prefixes to remove
- from the subject
- """
- re_re = re.compile(r'^(re|fwd?)[:\s]\s*', re.I)
- prefix_re = re.compile(r'^\[([^\]]*)\]\s*(.*)$')
- subject = clean_header(subject)
-
- if drop_prefixes is None:
- drop_prefixes = []
- else:
- drop_prefixes = [s.lower() for s in drop_prefixes]
-
- drop_prefixes.append('patch')
-
- # remove Re:, Fwd:, etc
- subject = re_re.sub(' ', subject)
-
- subject = normalise_space(subject)
-
- prefixes = []
-
- match = prefix_re.match(subject)
-
- while match:
- prefix_str = match.group(1)
- prefixes += [p for p in split_prefixes(prefix_str)
- if p.lower() not in drop_prefixes]
-
- subject = match.group(2)
- match = prefix_re.match(subject)
-
- subject = normalise_space(subject)
-
- subject = subject.strip()
- if prefixes:
- subject = '[%s] %s' % (','.join(prefixes), subject)
-
- return (subject, prefixes)
-
-
-def clean_content(content):
- """Remove cruft from the email message.
-
- Catch signature (-- ) and list footer (_____) cruft.
- """
- sig_re = re.compile(r'^(-- |_+)\n.*', re.S | re.M)
- content = sig_re.sub('', content)
-
- return content.strip()
-
-
-def find_state(mail):
- """Return the state with the given name or the default."""
- state_name = mail.get('X-Patchwork-State', '').strip()
- if state_name:
- try:
- return State.objects.get(name__iexact=state_name)
- except State.DoesNotExist:
- pass
- return get_default_initial_patch_state()
-
-
-def auto_delegate(project, filenames):
- if not filenames:
- return None
-
- rules = list(DelegationRule.objects.filter(project=project))
-
- patch_delegate = None
-
- for filename in filenames:
- file_delegate = None
- for rule in rules:
- if fnmatch(filename, rule.path):
- file_delegate = rule.user
- break
-
- if file_delegate is None:
- return None
-
- if patch_delegate is not None and file_delegate != patch_delegate:
- return None
-
- patch_delegate = file_delegate
-
- return patch_delegate
-
-
-def find_delegate(mail):
- """Return the delegate with the given email or None."""
- delegate_email = mail.get('X-Patchwork-Delegate', '').strip()
- if delegate_email:
- try:
- return User.objects.get(email__iexact=delegate_email)
- except User.DoesNotExist:
- pass
- return None
-
-
-def parse_mail(mail, list_id=None):
- """Parse a mail and add to the database.
-
- Args:
- mail (`mbox.Mail`): Mail to parse and add.
- list_id (str): Mailing list ID
-
- Returns:
- None
- """
- # some basic sanity checks
- if 'From' not in mail:
- raise ValueError("Missing 'From' header")
-
- if 'Subject' not in mail:
- raise ValueError("Missing 'Subject' header")
-
- if 'Message-Id' not in mail:
- raise ValueError("Missing 'Message-Id' header")
-
- hint = mail.get('X-Patchwork-Hint', '').lower()
- if hint == 'ignore':
- LOGGER.debug("Ignoring email due to 'ignore' hint")
- return
-
- if list_id:
- project = find_project_by_id(list_id)
- else:
- project = find_project_by_header(mail)
-
- if project is None:
- LOGGER.error('Failed to find a project for email')
- return
-
- # parse content
-
- diff, message = find_content(project, mail)
-
- if not (diff or message):
- return # nothing to work with
-
- msgid = mail.get('Message-Id').strip()
- author = find_author(mail)
- name, prefixes = clean_subject(mail.get('Subject'), [project.linkname])
- x, n = parse_series_marker(prefixes)
- refs = find_references(mail)
- date = find_date(mail)
- headers = find_headers(mail)
- pull_url = find_pull_request(message)
-
- # build objects
-
- if diff or pull_url: # patches or pull requests
- # we delay the saving until we know we have a patch.
- author.save()
-
- delegate = find_delegate(mail)
- if not delegate and diff:
- filenames = find_filenames(diff)
- delegate = auto_delegate(project, filenames)
-
- patch = Patch(
- msgid=msgid,
- project=project,
- name=name,
- date=date,
- headers=headers,
- submitter=author,
- content=message,
- diff=diff,
- pull_url=pull_url,
- delegate=delegate,
- state=find_state(mail))
- patch.save()
- LOGGER.debug('Patch saved')
-
- return patch
- elif x == 0: # (potential) cover letters
- # if refs are empty, it's implicitly a cover letter. If not,
- # however, we need to see if a match already exists and, if
- # not, assume that it is indeed a new cover letter
- is_cover_letter = False
- if not refs == []:
- try:
- CoverLetter.objects.all().get(name=name)
- except CoverLetter.DoesNotExist: # no match => new cover
- is_cover_letter = True
- except CoverLetter.MultipleObjectsReturned:
- # if multiple cover letters are found, just ignore
- pass
- else:
- is_cover_letter = True
-
- if is_cover_letter:
- author.save()
-
- cover_letter = CoverLetter(
- msgid=msgid,
- project=project,
- name=name,
- date=date,
- headers=headers,
- submitter=author,
- content=message)
- cover_letter.save()
- LOGGER.debug('Cover letter saved')
-
- return cover_letter
-
- # comments
-
- # we only save comments if we have the parent email
- submission = find_submission_for_comment(project, refs)
- if not submission:
- return
-
- author.save()
-
- comment = Comment(
- submission=submission,
- msgid=msgid,
- date=date,
- headers=headers,
- submitter=author,
- content=message)
- comment.save()
- LOGGER.debug('Comment saved')
-
- return comment
extra_error_message = '''
== Mail
diff --git a/patchwork/parser.py b/patchwork/parser.py
index c9c058d..b5f2ed6 100644
--- a/patchwork/parser.py
+++ b/patchwork/parser.py
@@ -19,13 +19,359 @@
# along with Patchwork; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+import codecs
+import datetime
+from email.header import Header, decode_header
+from email.utils import parsedate_tz, mktime_tz
+from fnmatch import fnmatch
+from functools import reduce
+import logging
+import operator
import re
+from django.contrib.auth.models import User
+from django.utils import six
from django.utils.six.moves import map
+from patchwork.models import (Patch, Project, Person, Comment, State,
+ DelegationRule, Submission, CoverLetter,
+ get_default_initial_patch_state)
-_hunk_re = re.compile('^\@\@ -\d+(?:,(\d+))? \+\d+(?:,(\d+))? \@\@')
-_filename_re = re.compile('^(---|\+\+\+) (\S+)')
+
+_hunk_re = re.compile(r'^\@\@ -\d+(?:,(\d+))? \+\d+(?:,(\d+))? \@\@')
+_filename_re = re.compile(r'^(---|\+\+\+) (\S+)')
+list_id_headers = ['List-ID', 'X-Mailing-List', 'X-list']
+
+LOGGER = logging.getLogger(__name__)
+
+
+def normalise_space(str):
+ whitespace_re = re.compile(r'\s+')
+ return whitespace_re.sub(' ', str).strip()
+
+
+def clean_header(header):
+ """Decode (possibly non-ascii) headers."""
+ def decode(fragment):
+ (frag_str, frag_encoding) = fragment
+ if frag_encoding:
+ return frag_str.decode(frag_encoding)
+ elif isinstance(frag_str, six.binary_type): # python 2
+ return frag_str.decode()
+ return frag_str
+
+ fragments = list(map(decode, decode_header(header)))
+
+ return normalise_space(u' '.join(fragments))
+
+
+def find_project_by_id(list_id):
+ """Find a `project` object with given `list_id`."""
+ project = None
+ try:
+ project = Project.objects.get(listid=list_id)
+ except Project.DoesNotExist:
+ pass
+ return project
+
+
+def find_project_by_header(mail):
+ project = None
+ listid_res = [re.compile(r'.*<([^>]+)>.*', re.S),
+ re.compile(r'^([\S]+)$', re.S)]
+
+ for header in list_id_headers:
+ if header in mail:
+
+ for listid_re in listid_res:
+ match = listid_re.match(mail.get(header))
+ if match:
+ break
+
+ if not match:
+ continue
+
+ listid = match.group(1)
+
+ project = find_project_by_id(listid)
+ if project:
+ break
+
+ return project
+
+
+def find_author(mail):
+ from_header = clean_header(mail.get('From'))
+ name, email = (None, None)
+
+ # tuple of (regex, fn)
+ # - where fn returns a (name, email) tuple from the match groups resulting
+ # from re.match().groups()
+ from_res = [
+ # for "Firstname Lastname" <example at example.com> style addresses
+ (re.compile(r'"?(.*?)"?\s*<([^>]+)>'), (lambda g: (g[0], g[1]))),
+
+ # for example at example.com (Firstname Lastname) style addresses
+ (re.compile(r'"?(.*?)"?\s*\(([^\)]+)\)'), (lambda g: (g[1], g[0]))),
+
+ # for example at example.com (Firstname Lastname) style addresses
+ (re.compile(r'(.*?)\sat\s(.*?)\s*\(([^\)]+)\)'),
+ (lambda g: (g[2], '@'.join(g[0:2])))),
+
+ # everything else
+ (re.compile(r'(.*)'), (lambda g: (None, g[0]))),
+ ]
+
+ for regex, fn in from_res:
+ match = regex.match(from_header)
+ if match:
+ (name, email) = fn(match.groups())
+ break
+
+ if email is None:
+ raise ValueError("Invalid 'From' header")
+
+ email = email.strip()
+ if name is not None:
+ name = name.strip()
+
+ try:
+ person = Person.objects.get(email__iexact=email)
+ if name: # use the latest provided name
+ person.name = name
+ except Person.DoesNotExist:
+ person = Person(name=name, email=email)
+
+ return person
+
+
+def find_date(mail):
+ t = parsedate_tz(mail.get('Date', ''))
+ if not t:
+ return datetime.datetime.utcnow()
+ return datetime.datetime.utcfromtimestamp(mktime_tz(t))
+
+
+def find_headers(mail):
+ return reduce(operator.__concat__,
+ ['%s: %s\n' % (k, Header(v, header_name=k,
+ continuation_ws='\t').encode())
+ for (k, v) in list(mail.items())])
+
+
+def find_pull_request(content):
+ git_re = re.compile(r'^The following changes since commit.*' +
+ r'^are available in the git repository at:\n'
+ r'^\s*([\S]+://[^\n]+)$',
+ re.DOTALL | re.MULTILINE)
+ match = git_re.search(content)
+ if match:
+ return match.group(1)
+ return None
+
+
+def find_references(mail):
+ """Construct a list of possible reply message ids."""
+ refs = []
+
+ if 'In-Reply-To' in mail:
+ refs.append(mail.get('In-Reply-To'))
+
+ if 'References' in mail:
+ rs = mail.get('References').split()
+ rs.reverse()
+ for r in rs:
+ if r not in refs:
+ refs.append(r)
+
+ return refs
+
+
+def parse_series_marker(subject_prefixes):
+ """Extract series markers from subject.
+
+ Extract the markers of multi-patches series, i.e. 'x/n', from the
+ provided subject series.
+
+ Args:
+ subject_prefixes: List of subject prefixes to extract markers
+ from
+
+ Returns:
+ (x, n) if markers found, else (None, None)
+ """
+
+ regex = re.compile('^([0-9]+)/([0-9]+)$')
+ for prefix in subject_prefixes:
+ m = regex.match(prefix)
+ if not m:
+ continue
+ return (int(m.group(1)), int(m.group(2)))
+ return (None, None)
+
+
+def find_content(project, mail):
+ """Extract a comment and potential diff from a mail."""
+ patchbuf = None
+ commentbuf = ''
+
+ for part in mail.walk():
+ if part.get_content_maintype() != 'text':
+ continue
+
+ payload = part.get_payload(decode=True)
+ subtype = part.get_content_subtype()
+
+ if not isinstance(payload, six.text_type):
+ charset = part.get_content_charset()
+
+ # Check that we have a charset that we understand. Otherwise,
+ # ignore it and fallback to our standard set.
+ if charset is not None:
+ try:
+ codecs.lookup(charset)
+ except LookupError:
+ charset = None
+
+ # If there is no charset or if it is unknown, then try some common
+ # charsets before we fail.
+ if charset is None:
+ try_charsets = ['utf-8', 'windows-1252', 'iso-8859-1']
+ else:
+ try_charsets = [charset]
+
+ for cset in try_charsets:
+ try:
+ payload = six.text_type(payload, cset)
+ break
+ except UnicodeDecodeError:
+ payload = None
+
+ # Could not find a valid decoded payload. Fail.
+ if payload is None:
+ return None, None
+
+ if subtype in ['x-patch', 'x-diff']:
+ patchbuf = payload
+ elif subtype == 'plain':
+ c = payload
+
+ if not patchbuf:
+ patchbuf, c = parse_patch(payload)
+
+ if c is not None:
+ commentbuf += c.strip() + '\n'
+
+ commentbuf = clean_content(commentbuf)
+
+ return patchbuf, commentbuf
+
+
+def find_submission_for_comment(project, refs):
+ for ref in refs:
+ # first, check for a direct reply
+ try:
+ submission = Submission.objects.get(project=project, msgid=ref)
+ return submission
+ except Submission.DoesNotExist:
+ pass
+
+ # see if we have comments that refer to a patch
+ try:
+ comment = Comment.objects.get(submission__project=project,
+ msgid=ref)
+ return comment.submission
+ except Comment.MultipleObjectsReturned:
+ # NOTE(stephenfin): This is a artifact of prior lack of support
+ # for cover letters in Patchwork. Previously all replies to
+ # patches were saved as comments. However, it's possible that
+ # someone could have created a new series as a reply to one of the
+ # comments on the original patch series. For example,
+ # '2015-November/002096.html' from the Patchwork archives. In this
+ # case, reparsing the archives will result in creation of a cover
+ # letter with the same message ID as the existing comment. Follow
+ # up comments will then apply to both this cover letter and the
+ # linked patch from the comment previously created. We choose to
+ # apply the comment to the cover letter. Note that this only
+ # happens when running 'parsearchive' or similar, so it should not
+ # affect every day use in any way.
+ comments = Comment.objects.filter(submission__project=project,
+ msgid=ref)
+ # The latter item will be the cover letter
+ return comments.reverse()[0].submission
+ except Comment.DoesNotExist:
+ pass
+
+ return None
+
+
+def split_prefixes(prefix):
+ """Turn a prefix string into a list of prefix tokens."""
+ split_re = re.compile(r'[,\s]+')
+ matches = split_re.split(prefix)
+
+ return [s for s in matches if s != '']
+
+
+def clean_subject(subject, drop_prefixes=None):
+ """Clean a Subject: header from an incoming patch.
+
+ Removes Re: and Fwd: strings, as well as [PATCH]-style prefixes. By
+ default, only [PATCH] is removed, and we keep any other bracketed
+ data in the subject. If drop_prefixes is provided, remove those
+ too, comparing case-insensitively.
+
+ Args:
+ subject: Subject to be cleaned
+ drop_prefixes: Additional, case-insensitive prefixes to remove
+ from the subject
+ """
+ re_re = re.compile(r'^(re|fwd?)[:\s]\s*', re.I)
+ prefix_re = re.compile(r'^\[([^\]]*)\]\s*(.*)$')
+ subject = clean_header(subject)
+
+ if drop_prefixes is None:
+ drop_prefixes = []
+ else:
+ drop_prefixes = [s.lower() for s in drop_prefixes]
+
+ drop_prefixes.append('patch')
+
+ # remove Re:, Fwd:, etc
+ subject = re_re.sub(' ', subject)
+
+ subject = normalise_space(subject)
+
+ prefixes = []
+
+ match = prefix_re.match(subject)
+
+ while match:
+ prefix_str = match.group(1)
+ prefixes += [p for p in split_prefixes(prefix_str)
+ if p.lower() not in drop_prefixes]
+
+ subject = match.group(2)
+ match = prefix_re.match(subject)
+
+ subject = normalise_space(subject)
+
+ subject = subject.strip()
+ if prefixes:
+ subject = '[%s] %s' % (','.join(prefixes), subject)
+
+ return (subject, prefixes)
+
+
+def clean_content(content):
+ """Remove cruft from the email message.
+
+ Catch signature (-- ) and list footer (_____) cruft.
+ """
+ sig_re = re.compile(r'^(-- |_+)\n.*', re.S | re.M)
+ content = sig_re.sub('', content)
+
+ return content.strip()
def parse_patch(content):
@@ -181,6 +527,186 @@ def parse_patch(content):
return patchbuf, commentbuf
+def find_state(mail):
+ """Return the state with the given name or the default."""
+ state_name = mail.get('X-Patchwork-State', '').strip()
+ if state_name:
+ try:
+ return State.objects.get(name__iexact=state_name)
+ except State.DoesNotExist:
+ pass
+ return get_default_initial_patch_state()
+
+
+def auto_delegate(project, filenames):
+ if not filenames:
+ return None
+
+ rules = list(DelegationRule.objects.filter(project=project))
+
+ patch_delegate = None
+
+ for filename in filenames:
+ file_delegate = None
+ for rule in rules:
+ if fnmatch(filename, rule.path):
+ file_delegate = rule.user
+ break
+
+ if file_delegate is None:
+ return None
+
+ if patch_delegate is not None and file_delegate != patch_delegate:
+ return None
+
+ patch_delegate = file_delegate
+
+ return patch_delegate
+
+
+def find_delegate(mail):
+ """Return the delegate with the given email or None."""
+ delegate_email = mail.get('X-Patchwork-Delegate', '').strip()
+ if delegate_email:
+ try:
+ return User.objects.get(email__iexact=delegate_email)
+ except User.DoesNotExist:
+ pass
+ return None
+
+
+def parse_mail(mail, list_id=None):
+ """Parse a mail and add to the database.
+
+ Args:
+ mail (`mbox.Mail`): Mail to parse and add.
+ list_id (str): Mailing list ID
+
+ Returns:
+ None
+ """
+ # some basic sanity checks
+ if 'From' not in mail:
+ raise ValueError("Missing 'From' header")
+
+ if 'Subject' not in mail:
+ raise ValueError("Missing 'Subject' header")
+
+ if 'Message-Id' not in mail:
+ raise ValueError("Missing 'Message-Id' header")
+
+ hint = mail.get('X-Patchwork-Hint', '').lower()
+ if hint == 'ignore':
+ LOGGER.debug("Ignoring email due to 'ignore' hint")
+ return
+
+ if list_id:
+ project = find_project_by_id(list_id)
+ else:
+ project = find_project_by_header(mail)
+
+ if project is None:
+ LOGGER.error('Failed to find a project for email')
+ return
+
+ # parse content
+
+ diff, message = find_content(project, mail)
+
+ if not (diff or message):
+ return # nothing to work with
+
+ msgid = mail.get('Message-Id').strip()
+ author = find_author(mail)
+ name, prefixes = clean_subject(mail.get('Subject'), [project.linkname])
+ x, n = parse_series_marker(prefixes)
+ refs = find_references(mail)
+ date = find_date(mail)
+ headers = find_headers(mail)
+ pull_url = find_pull_request(message)
+
+ # build objects
+
+ if diff or pull_url: # patches or pull requests
+ # we delay the saving until we know we have a patch.
+ author.save()
+
+ delegate = find_delegate(mail)
+ if not delegate and diff:
+ filenames = find_filenames(diff)
+ delegate = auto_delegate(project, filenames)
+
+ patch = Patch(
+ msgid=msgid,
+ project=project,
+ name=name,
+ date=date,
+ headers=headers,
+ submitter=author,
+ content=message,
+ diff=diff,
+ pull_url=pull_url,
+ delegate=delegate,
+ state=find_state(mail))
+ patch.save()
+ LOGGER.debug('Patch saved')
+
+ return patch
+ elif x == 0: # (potential) cover letters
+ # if refs are empty, it's implicitly a cover letter. If not,
+ # however, we need to see if a match already exists and, if
+ # not, assume that it is indeed a new cover letter
+ is_cover_letter = False
+ if not refs == []:
+ try:
+ CoverLetter.objects.all().get(name=name)
+ except CoverLetter.DoesNotExist:
+ # if no match, this is a new cover letter
+ is_cover_letter = True
+ except CoverLetter.MultipleObjectsReturned:
+ # if multiple cover letters are found, just ignore
+ pass
+ else:
+ is_cover_letter = True
+
+ if is_cover_letter:
+ author.save()
+
+ cover_letter = CoverLetter(
+ msgid=msgid,
+ project=project,
+ name=name,
+ date=date,
+ headers=headers,
+ submitter=author,
+ content=message)
+ cover_letter.save()
+ LOGGER.debug('Cover letter saved')
+
+ return cover_letter
+
+ # comments
+
+ # we only save comments if we have the parent email
+ submission = find_submission_for_comment(project, refs)
+ if not submission:
+ return
+
+ author.save()
+
+ comment = Comment(
+ submission=submission,
+ msgid=msgid,
+ date=date,
+ headers=headers,
+ submitter=author,
+ content=message)
+ comment.save()
+ LOGGER.debug('Comment saved')
+
+ return comment
+
+
def find_filenames(diff):
"""Find files changes in a given diff."""
# normalise spaces
diff --git a/patchwork/tests/test_parser.py b/patchwork/tests/test_parser.py
index eca05a0..684a667 100644
--- a/patchwork/tests/test_parser.py
+++ b/patchwork/tests/test_parser.py
@@ -26,18 +26,19 @@ import os
from django.test import TestCase
-from patchwork.bin.parsemail import clean_subject
-from patchwork.bin.parsemail import find_author
-from patchwork.bin.parsemail import find_content
-from patchwork.bin.parsemail import find_project_by_header
-from patchwork.bin.parsemail import find_pull_request
-from patchwork.bin.parsemail import parse_mail as _parse_mail
-from patchwork.bin.parsemail import parse_series_marker
-from patchwork.bin.parsemail import split_prefixes
from patchwork.models import Comment
from patchwork.models import Patch
from patchwork.models import Person
from patchwork.models import State
+from patchwork.parser import clean_subject
+from patchwork.parser import find_author
+from patchwork.parser import find_content
+from patchwork.parser import find_project_by_header
+from patchwork.parser import find_pull_request
+from patchwork.parser import parse_mail as _parse_mail
+from patchwork.parser import parse_series_marker
+from patchwork.parser import split_prefixes
+from patchwork.tests.utils import create_email
from patchwork.tests.utils import create_project
from patchwork.tests.utils import create_state
from patchwork.tests.utils import create_user
--
2.7.4
More information about the Patchwork
mailing list