[PATCH] Implement list filtering
Daniel Axtens
dja at axtens.net
Thu Jan 25 17:57:47 AEDT 2018
vkabatov at redhat.com writes:
> From: Veronika Kabatova <vkabatov at redhat.com>
>
> Sometimes, multiple projects reside at the same mailing list. So far,
> Patchwork only allowed a single project per mailing list, which made it
> impossible for these projects to use Patchwork (unless they did some
> dirty hacks).
>
> Add a new property `subject_match` to projects and implement filtering
> on (list_id, subject_match) match instead of solely list_id. Instance
> admin can specify a regex on a per-project basis when the project is
> created.
(I haven't thought a great deal about this, so don't take this as a nack)
Two general thoughts:
- netdev has several projects sharing a mailing list and they seem to
get along fine...
- I am pretty sure Snowpatch deals with this somehow - Snowpatch does
CI on patchwork, so they need to know what project is used. Andrew,
Russell - how do you deal with this there?
I suppose to put a finer point on it - what is your usecase here? What
are you trying to achieve, and can we help you do that in a way that
requires smaller changes to Patchwork, and is less fragile? (For example
this is going to break if someone mis-spells the keyword you're looking
for in the subject_match.)
Regards,
Daniel
>
> Signed-off-by: Veronika Kabatova <vkabatov at redhat.com>
> ---
> docs/api.yaml | 3 +++
> docs/deployment/management.rst | 4 +--
> patchwork/api/project.py | 5 ++--
> .../0021_add_subject_match_to_project.py | 29 ++++++++++++++++++++++
> patchwork/models.py | 9 ++++++-
> patchwork/parser.py | 19 +++++++++++++-
> .../notes/list-filtering-4643d98b4064367a.yaml | 10 ++++++++
> 7 files changed, 73 insertions(+), 6 deletions(-)
> create mode 100644 patchwork/migrations/0021_add_subject_match_to_project.py
> create mode 100644 releasenotes/notes/list-filtering-4643d98b4064367a.yaml
>
> diff --git a/docs/api.yaml b/docs/api.yaml
> index 3e79f0b..3373226 100644
> --- a/docs/api.yaml
> +++ b/docs/api.yaml
> @@ -374,6 +374,9 @@ definitions:
> list_id:
> type: string
> description: Mailing list identifier for project.
> + subject_match:
> + type: string
> + description: Regex used for email filtering.
> list_email:
> type: string
> description: Mailing list email address for project.
> diff --git a/docs/deployment/management.rst b/docs/deployment/management.rst
> index c50b7b6..870d7ee 100644
> --- a/docs/deployment/management.rst
> +++ b/docs/deployment/management.rst
> @@ -63,7 +63,7 @@ due to, for example, an outage.
> .. option:: --list-id <list-id>
>
> mailing list ID. If not supplied, this will be extracted from the mail
> - headers.
> + headers. Don't use this option if you require filtering based on subjects.
>
> .. option:: infile
>
> @@ -88,7 +88,7 @@ the :ref:`deployment installation guide <deployment-parsemail>`.
> .. option:: --list-id <list-id>
>
> mailing list ID. If not supplied, this will be extracted from the mail
> - headers.
> + headers. Don't use this option if you require filtering based on subjects.
>
> .. option:: infile
>
> diff --git a/patchwork/api/project.py b/patchwork/api/project.py
> index 446c473..597f605 100644
> --- a/patchwork/api/project.py
> +++ b/patchwork/api/project.py
> @@ -39,8 +39,9 @@ class ProjectSerializer(HyperlinkedModelSerializer):
> class Meta:
> model = Project
> fields = ('id', 'url', 'name', 'link_name', 'list_id', 'list_email',
> - 'web_url', 'scm_url', 'webscm_url', 'maintainers')
> - read_only_fields = ('name', 'maintainers')
> + 'web_url', 'scm_url', 'webscm_url', 'maintainers',
> + 'subject_match')
> + read_only_fields = ('name', 'maintainers', 'subject_match')
> extra_kwargs = {
> 'url': {'view_name': 'api-project-detail'},
> }
> diff --git a/patchwork/migrations/0021_add_subject_match_to_project.py b/patchwork/migrations/0021_add_subject_match_to_project.py
> new file mode 100644
> index 0000000..6066266
> --- /dev/null
> +++ b/patchwork/migrations/0021_add_subject_match_to_project.py
> @@ -0,0 +1,29 @@
> +# -*- coding: utf-8 -*-
> +# Generated by Django 1.10.8 on 2018-01-19 18:16
> +from __future__ import unicode_literals
> +
> +from django.db import migrations, models
> +
> +
> +class Migration(migrations.Migration):
> +
> + dependencies = [
> + ('patchwork', '0020_tag_show_column'),
> + ]
> +
> + operations = [
> + migrations.AddField(
> + model_name='project',
> + name='subject_match',
> + field=models.CharField(blank=True, default=b'', help_text=b'Regex to match the subject against if only part of emails sent to the list belongs to this project. Will be used with IGNORECASE and MULTILINE flags. If rules for more projects match the first one returned from DB is chosen.', max_length=64),
> + ),
> + migrations.AlterField(
> + model_name='project',
> + name='listid',
> + field=models.CharField(max_length=255),
> + ),
> + migrations.AlterUniqueTogether(
> + name='project',
> + unique_together=set([('listid', 'subject_match')]),
> + ),
> + ]
> diff --git a/patchwork/models.py b/patchwork/models.py
> index 11886f1..907707f 100644
> --- a/patchwork/models.py
> +++ b/patchwork/models.py
> @@ -71,8 +71,14 @@ class Project(models.Model):
>
> linkname = models.CharField(max_length=255, unique=True)
> name = models.CharField(max_length=255, unique=True)
> - listid = models.CharField(max_length=255, unique=True)
> + listid = models.CharField(max_length=255)
> listemail = models.CharField(max_length=200)
> + subject_match = models.CharField(
> + max_length=64, blank=True, default='', help_text='Regex to match the '
> + 'subject against if only part of emails sent to the list belongs to '
> + 'this project. Will be used with IGNORECASE and MULTILINE flags. If '
> + 'rules for more projects match the first one returned from DB is '
> + 'chosen.')
>
> # url metadata
>
> @@ -100,6 +106,7 @@ class Project(models.Model):
> return self.name
>
> class Meta:
> + unique_together = (('listid', 'subject_match'),)
> ordering = ['linkname']
>
>
> diff --git a/patchwork/parser.py b/patchwork/parser.py
> index ac7dc5f..015f709 100644
> --- a/patchwork/parser.py
> +++ b/patchwork/parser.py
> @@ -157,9 +157,25 @@ def find_project_by_id(list_id):
> project = Project.objects.get(listid=list_id)
> except Project.DoesNotExist:
> logger.debug("'%s' if not a valid project list-id", list_id)
> + except Project.MultipleObjectsReturned:
> + logger.debug("Multiple projects with list-id '%s' found", list_id)
> return project
>
>
> +def find_project_by_id_and_subject(list_id, subject):
> + """Find a `project` object based on `list_id` and subject prefix match."""
> + projects = Project.objects.filter(listid=list_id)
> + for project in projects:
> + if (not project.subject_match or
> + re.search(project.subject_match, subject,
> + re.MULTILINE | re.IGNORECASE)):
> + return project
> +
> + logger.debug("No project to match email with list-id '%s', subject '%s' "
> + "found", list_id, subject)
> + return None
> +
> +
> def find_project_by_header(mail):
> project = None
> listid_res = [re.compile(r'.*<([^>]+)>.*', re.S),
> @@ -181,7 +197,8 @@ def find_project_by_header(mail):
>
> listid = match.group(1)
>
> - project = find_project_by_id(listid)
> + project = find_project_by_id_and_subject(listid,
> + mail.get('Subject'))
> if project:
> break
>
> diff --git a/releasenotes/notes/list-filtering-4643d98b4064367a.yaml b/releasenotes/notes/list-filtering-4643d98b4064367a.yaml
> new file mode 100644
> index 0000000..21c1680
> --- /dev/null
> +++ b/releasenotes/notes/list-filtering-4643d98b4064367a.yaml
> @@ -0,0 +1,10 @@
> +---
> +features:
> + - |
> + Allow list filtering into multiple projects (and email dropping) based on
> + subject prefixes. Disabled by default, enable by specifying a regular
> + expression which needs to be matched in the subject on a per-project basis
> + (field `subject_match`).
> +api:
> + - |
> + Expose `subject_match` in REST API.
> --
> 2.13.6
>
> _______________________________________________
> Patchwork mailing list
> Patchwork at lists.ozlabs.org
> https://lists.ozlabs.org/listinfo/patchwork
More information about the Patchwork
mailing list