Merge branch 'develop'

Author: manfred-kaiser

.github/workflows/python-publish.yml

@@ -0,0 +1,31 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        python setup.py sdist bdist_wheel
+        twine upload dist/*

MANIFEST.in

@@ -1,4 +1,3 @@
 include LICENSE.txt
 include MANIFEST.in
 include README.rst
-include requirements.txt

README.md

@@ -0,0 +1,84 @@
+# Sifter3 - Sieve email filter (RFC 5228)
+
+Sifter3 is a Python 3 implementation of the Sieve email filter language (RFC 5228) and is based on the Python 2 version from <https://github.com/garyp/sifter>
+
+![Python package](https://github.com/manfred-kaiser/sifter3/workflows/Python%20package/badge.svg)
+[![Documentation Status](https://readthedocs.org/projects/sifter3/badge/?version=master)](https://sifter3.readthedocs.io/de/master/?badge=master)
+[![CodeFactor](https://www.codefactor.io/repository/github/manfred-kaiser/sifter3/badge)](https://www.codefactor.io/repository/github/manfred-kaiser/sifter3)
+[![Github version](https://img.shields.io/github/v/release/manfred-kaiser/sifter3?label=github&logo=github)](https://github.com/manfred-kaiser/sifter3/releases)
+[![PyPI version](https://img.shields.io/pypi/v/sifter3.svg?logo=pypi&logoColor=FFE873)](https://pypi.org/project/sifter3/)
+[![Supported Python versions](https://img.shields.io/pypi/pyversions/sifter3.svg?logo=python&logoColor=FFE873)](https://pypi.org/project/sifter3/)
+[![PyPI downloads](https://pepy.tech/badge/sifter3/month)](https://pepy.tech/project/sifter3/month)
+[![GitHub](https://img.shields.io/github/license/manfred-kaiser/sifter3.svg)](LICENSE)
+
+
+
+FEATURES
+========
+
+-   Supports all of the base Sieve spec from RFC 5228, except for
+    features still listed under TODO below
+-   Extensions supported:
+    -   regex (draft-ietf-sieve-regex-01)
+
+EXAMPLE
+=======
+
+    import email
+    import sifter.parser
+    rules = sifter.parser.parse_file(open('my_rules.sieve'))
+    msg = email.message_from_file(open('an_email_to_me.eml'))
+    msg_actions = rules.evaluate(msg)
+
+In the above example, `msg_actions` is a list of actions to apply to the
+email message. Each action is a tuple consisting of the action name and
+action-specific arguments. It is up to the caller to manipulate the
+message and message store based on the actions returned.
+
+WARNINGS
+========
+
+-   No thought has been given yet to hardening against malicious user
+    input. The current implementation is aimed at users that are running
+    their own sieve scripts.
+-   The current implementation is not optimized for performance, though
+    hopefully it's not too slow for normal inputs.
+
+TODO
+====
+
+In rough order of importance:
+
+-   An example adaptor that provides Unix LDA behavior using sieve for
+    filtering
+-   Base spec features not yet implemented:
+    -   encoded characters (section 2.4.2.4)
+    -   multi-line strings (section 2.4.2)
+    -   bracketed comments (section 2.3)
+    -   message uniqueness (section 2.10.3)
+    -   envelope test (section 5.4)
+    -   handle message loops (section 10)
+    -   limit abuse of redirect action (section 10)
+    -   address test should limit allowed headers to those that contain
+        addresses (section 5.1)
+-   Make sure character sets are actually handled according to the spec
+-   Make string parsing comply with the grammar in section 8.1 and the
+    features described in section 2.4.2
+-   Check that python's `email.message` implements header comparisons
+    the same way as the sieve spec
+-   Make sure regular expressions are actually handled according to the
+    extension spec
+-   Add support for various extensions:
+    -   variables (RFC 5229)
+    -   externally stored lists (draft-melnikov-sieve-external-lists)
+    -   body (RFC 5173)
+    -   relational (RFC 5231)
+    -   subaddress (RFC 5233)
+    -   copy (RFC 3894)
+    -   environment (RFC 5183)
+    -   date and index (RFC 5260)
+    -   editheader (RFC 5293)
+    -   ihave (RFC 5463)
+    -   mailbox metadata (RFC 5490)
+    -   notifications (RFC 5435), mailto notifications (RFC 5436), xmpp
+        notifications (RFC 5437)

README.rst

@@ -5,27 +5,28 @@
 # read the contents of your README file
 from os import path
 this_directory = path.abspath(path.dirname(__file__))
-with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
+with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
     long_description = f.read()
 
-with open('requirements.txt') as f:
-    requirements = f.read().splitlines()
-
 setup(
-    name="sifter",
+    name="sifter3",
     version="0.2.0",
-    author="Gary Peck, Manfred Kaiser",
-    author_email="gary@realify.com, manfred.kaiser@logfile.at",
+    author="Manfred Kaiser, Gary Peck",
+    author_email="manfred.kaiser@logfile.at, gary@realify.com",
     url="https://github.com/garyp/sifter",
     license="BSD",
+    description='Parser/evaluator for the Sieve filtering language (RFC 5228) - Python3 version',
     long_description=long_description,
-    long_description_content_type='text/x-rst',
+    long_description_content_type='text/markdown',
+    keywords="sieve email filter parser",
     project_urls={
-        'Source': 'https://github.com/garyp/sifter',
-        'Tracker': 'https://github.com/garyp/sifter/issues',
+        'Source': 'https://github.com/manfred-kaiser/sifter3',
+        'Tracker': 'https://github.com/manfred-kaiser/sifter3/issues',
     },
     python_requires='>= 3.6',
-    install_requires=requirements,
+    install_requires=[
+        'ply'
+    ],
     classifiers=[
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
@@ -46,6 +47,9 @@
         "sifter": ['py.typed']
     },
     entry_points={
+        'console_scripts': [
+            'sifter = sifter.cli:main'
+        ],
         'sifter_extensions': [
             # sifter commands
             'discard = sifter.commands.discard:CommandDiscard',
@@ -54,11 +58,14 @@
             'elseif = sifter.commands.if_cmd:CommandElsIf',
             'else = sifter.commands.if_cmd:CommandElse',
             'keep = sifter.commands.keep:CommandKeep',
+            'notify = sifter.commands.notify:CommandNotify',
             'redirect = sifter.commands.redirect:CommandRedirect',
             'require = sifter.commands.require:CommandRequire',
+            'set = sifter.commands.variables:CommandSet',
             'stop = sifter.commands.stop:CommandStop',
             # sifter tests
             'address = sifter.tests.address:TestAddress',
+            'body = sifter.tests.body:TestBody',
             'allof = sifter.tests.allof:TestAllOf',
             'anyof = sifter.tests.anyof:TestAnyOf',
             'exists = sifter.tests.exists:TestExists',
@@ -67,9 +74,14 @@
             'not_test = sifter.tests.not_test:TestNot',
             'size = sifter.tests.size:TestSize',
             'true = sifter.tests.true:TestTrue',
+            'valid_notify_method = sifter.tests.notify:TestValidNotifyMethod',
+            'notify_method_capability = sifter.tests.notify:TestValidNotifyMethod',
             # sifter comparators
             'ascii_casemap = sifter.comparators.ascii_casemap:ComparatorASCIICasemap',
-            'octed = sifter.comparators.octet:ComparatorOctet'
-        ],
+            'ascii_casemap_noi = sifter.comparators.ascii_casemap:ComparatorASCIICasemapnoi',
+            'octed = sifter.comparators.octet:ComparatorOctet',
+            # notification methods
+            'mailto = sifter.notificationmethods.mailto:MailtoNotificationMethod',
+        ]
     }
 )

requirements.txt

@@ -0,0 +1,27 @@
+from argparse import ArgumentParser
+import email
+import os
+import logging
+import sys
+
+import sifter.parser
+
+
+def main() -> None:
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument('rulefile')
+    arg_parser.add_argument('messagefile')
+    args = arg_parser.parse_args()
+
+    if not os.path.isfile(args.rulefile):
+        logging.error("rulefile '%s' does not exist", args.rulefile)
+        sys.exit(1)
+
+    if not os.path.isfile(args.messagefile):
+        logging.error("mail message '%s' does not exist", args.messagefile)
+        sys.exit(1)
+
+    rules = sifter.parser.parse_file(open(args.rulefile))
+    msg = email.message_from_file(open(args.messagefile))
+    msg_actions = rules.evaluate(msg)
+    print(msg_actions)

setup.py

@@ -4,6 +4,7 @@
 )
 
 from sifter.grammar.command import Command
+from sifter.grammar.string import expand_variables
 from sifter.validators.stringlist import StringList
 from sifter.grammar.state import EvaluationState
 from sifter.grammar.actions import Actions
@@ -16,9 +17,11 @@ class CommandFileInto(Command):
     POSITIONAL_ARGS = [StringList(length=1)]
 
     def evaluate(self, message: Message, state: EvaluationState) -> Optional[Actions]:
+        state.check_required_extension('fileinto', 'FILEINTO')
+
         file_dest = self.positional_args[0]
+        file_dest = list(map(lambda s: expand_variables(s, state), file_dest))  # type: ignore
 
-        state.check_required_extension('fileinto', 'FILEINTO')
-        state.actions.append('fileinto', file_dest)  # type: ignore
+        state.actions.append('fileinto', file_dest)
         state.actions.cancel_implicit_keep()
         return None

sifter/cli.py

@@ -0,0 +1,52 @@
+from email.message import Message
+from typing import Optional
+
+from sifter.grammar.command import Command
+from sifter.validators.stringlist import StringList
+from sifter.grammar.string import expand_variables
+from sifter.grammar.state import EvaluationState
+from sifter.grammar.actions import Actions
+
+# This implements the RFC5232 imap4flags extension
+# commands: addflag, removeflag, setflag
+# tests: :hasflag
+# tagged arguments: :flag to 'fileinto'
+
+
+class CommandSetFlag(Command):
+
+    RULE_IDENTIFIER = 'SETFLAG'
+    POSITIONAL_ARGS = [StringList()]
+
+    def evaluate(self, message: Message, state: EvaluationState) -> Optional[Actions]:
+        state.check_required_extension('imap4flags', 'imapflags')
+        flag_list = self.positional_args[0]
+        flag_list = list(map(lambda s: expand_variables(s, state), flag_list))  # type: ignore
+        state.actions.append('setflag', flag_list)
+        return None
+
+
+class CommandRemoveFlag(Command):
+
+    RULE_IDENTIFIER = 'REMOVEFLAG'
+    POSITIONAL_ARGS = [StringList()]
+
+    def evaluate(self, message: Message, state: EvaluationState) -> Optional[Actions]:
+        state.check_required_extension('imap4flags', 'imapflags')
+        flag_list = self.positional_args[0]
+        flag_list = list(map(lambda s: expand_variables(s, state), flag_list))  # type: ignore
+        state.actions.append('removeflag', flag_list)
+        return None
+
+
+class CommandAddFlag(Command):
+
+    RULE_IDENTIFIER = 'ADDFLAG'
+    POSITIONAL_ARGS = [StringList()]
+
+    def evaluate(self, message: Message, state: EvaluationState) -> Optional[Actions]:
+        state.check_required_extension('imap4flags', 'imapflags')
+        flag_list = self.positional_args[0]
+        flag_list = list(map(lambda s: expand_variables(s, state), flag_list))  # type: ignore
+        state.actions.append('addflag', flag_list)
+        return None