allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kentontay...@apache.org
Subject [1/2] allura git commit: [#8192] StopForumSpam filter, and chained spam filters
Date Wed, 07 Mar 2018 20:13:24 GMT
Repository: allura
Updated Branches:
  refs/heads/master f597b2e67 -> 6742f0f15


[#8192] StopForumSpam filter, and chained spam filters


Project: http://git-wip-us.apache.org/repos/asf/allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/allura/commit/407559b0
Tree: http://git-wip-us.apache.org/repos/asf/allura/tree/407559b0
Diff: http://git-wip-us.apache.org/repos/asf/allura/diff/407559b0

Branch: refs/heads/master
Commit: 407559b02d77284b839e3ea9a26ef48dc1193c32
Parents: f597b2e
Author: Dave Brondsema <dave@brondsema.net>
Authored: Thu Mar 1 15:33:12 2018 -0500
Committer: Kenton Taylor <ktaylor@slashdotmedia.com>
Committed: Wed Mar 7 14:38:33 2018 -0500

----------------------------------------------------------------------
 Allura/allura/lib/app_globals.py                |  3 +-
 Allura/allura/lib/spam/__init__.py              | 51 +++++++++++++---
 Allura/allura/lib/spam/akismetfilter.py         |  9 +--
 Allura/allura/lib/spam/mollomfilter.py          |  9 +--
 Allura/allura/lib/spam/stopforumspamfilter.py   | 63 ++++++++++++++++++++
 .../allura/tests/unit/spam/test_spam_filter.py  | 30 +++++++++-
 .../tests/unit/spam/test_stopforumspam.py       | 51 ++++++++++++++++
 Allura/development.ini                          |  5 +-
 Allura/setup.py                                 |  1 +
 9 files changed, 197 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 5453055..fc06acc 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -256,7 +256,8 @@ class Globals(object):
                     value = ep.load()
                 except Exception:
                     log.exception('Could not load entry point [%s] %s', section_name, ep)
-                d[ep.name] = value
+                else:
+                    d[ep.name] = value
             return d
 
         class entry_point_loading_dict(dict):

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/__init__.py b/Allura/allura/lib/spam/__init__.py
index a4405e3..aaa4412 100644
--- a/Allura/allura/lib/spam/__init__.py
+++ b/Allura/allura/lib/spam/__init__.py
@@ -16,6 +16,9 @@
 #       under the License.
 
 import logging
+from copy import copy
+
+from paste.deploy.converters import aslist
 
 from allura.lib.helpers import exceptionless
 from allura.model.artifact import SpamCheckResult
@@ -30,20 +33,23 @@ class SpamFilter(object):
     def __init__(self, config):
         pass
 
+    @property
+    def filter_name(self):
+        return self.__class__.__name__.replace('SpamFilter', '').lower()
+
     def check(self, text, artifact=None, user=None, content_type='comment', **kw):
         """Return True if ``text`` is spam, else False."""
         log.info("No spam checking enabled")
         return False
 
     def submit_spam(self, text, artifact=None, user=None, content_type='comment', **kw):
-        log.info("No spam checking enabled")
+        log.info("No submit_spam available for %s", self.filter_name)
 
     def submit_ham(self, text, artifact=None, user=None, content_type='comment', **kw):
-        log.info("No spam checking enabled")
+        log.info("No submit_ham available for %s", self.filter_name)
 
     def record_result(self, result, artifact, user):
-        filter_name = self.__class__.__name__.replace('SpamFilter', '').lower()
-        log.info("spam=%s (%s): %s" % (str(result), filter_name, artifact.url() if artifact
else ''))
+        log.info("spam=%s (%s): %s" % (str(result), self.filter_name, artifact.url() if artifact
else ''))
         r = SpamCheckResult(
             ref=artifact.ref if artifact else None,
             project_id=artifact.project_id if artifact else None,
@@ -60,7 +66,36 @@ class SpamFilter(object):
         method = config.get('spam.method')
         if not method:
             return cls(config)
-        result = entry_points[method]
-        filter_obj = result(config)
-        filter_obj.check = exceptionless(False, log=log)(filter_obj.check)
-        return filter_obj
+        elif ' ' in method:
+            return ChainedSpamFilter(method, entry_points, config)
+        else:
+            result = entry_points[method]
+            filter_obj = result(config)
+            filter_obj.check = exceptionless(False, log=log)(filter_obj.check)
+            return filter_obj
+
+
+class ChainedSpamFilter(SpamFilter):
+
+    def __init__(self, methods_string, entry_points, config):
+        methods = aslist(methods_string)
+        self.filters = []
+        for m in methods:
+            config = copy(config)
+            config['spam.method'] = m
+            spam_filter = SpamFilter.get(config=config, entry_points=entry_points)
+            self.filters.append(spam_filter)
+
+    def check(self, *a, **kw):
+        for spam_filter in self.filters:
+            if spam_filter.check(*a, **kw):
+                return True
+        return False
+
+    def submit_spam(self, *a, **kw):
+        for spam_filter in self.filters:
+            spam_filter.submit_spam(*a, **kw)
+
+    def submit_ham(self, *a, **kw):
+        for spam_filter in self.filters:
+            spam_filter.submit_ham(*a, **kw)

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/akismetfilter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/akismetfilter.py b/Allura/allura/lib/spam/akismetfilter.py
index 7a87bd9..26cccbd 100644
--- a/Allura/allura/lib/spam/akismetfilter.py
+++ b/Allura/allura/lib/spam/akismetfilter.py
@@ -38,13 +38,8 @@ class AkismetSpamFilter(SpamFilter):
 
     """Spam checking implementation via Akismet service.
 
-    To enable Akismet spam filtering in your Allura instance, first
-    enable the entry point in setup.py::
-
-        [allura.spam]
-        akismet = allura.lib.spam.akismetfilter:AkismetSpamFilter
-
-    Then include the following parameters in your .ini file::
+    To enable Akismet spam filtering in your Allura instance, first pip install akismet (see
requirements-optional.txt)
+    and then include the following parameters in your .ini file::
 
         spam.method = akismet
         spam.key = <your Akismet key here>

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/mollomfilter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/mollomfilter.py b/Allura/allura/lib/spam/mollomfilter.py
index 533186b..f2d6b68 100644
--- a/Allura/allura/lib/spam/mollomfilter.py
+++ b/Allura/allura/lib/spam/mollomfilter.py
@@ -38,13 +38,8 @@ class MollomSpamFilter(SpamFilter):
 
     """Spam checking implementation via Mollom service.
 
-    To enable Mollom spam filtering in your Allura instance, first
-    enable the entry point in setup.py::
-
-        [allura.spam]
-        mollom = allura.lib.spam.mollomfilter:MollomSpamFilter
-
-    Then include the following parameters in your .ini file::
+    To enable Mollom spam filtering in your Allura instance, first pip install PyMollom (see
requirements-optional.txt)
+    and then include the following parameters in your .ini file::
 
         spam.method = mollom
         spam.public_key = <your Mollom public key here>

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/lib/spam/stopforumspamfilter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/spam/stopforumspamfilter.py b/Allura/allura/lib/spam/stopforumspamfilter.py
new file mode 100644
index 0000000..0e263be
--- /dev/null
+++ b/Allura/allura/lib/spam/stopforumspamfilter.py
@@ -0,0 +1,63 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+import logging
+import csv
+from sys import getsizeof
+
+import ipaddress
+from pylons import request
+
+from allura.lib import utils
+from allura.lib.spam import SpamFilter
+
+log = logging.getLogger(__name__)
+
+
+class StopForumSpamSpamFilter(SpamFilter):
+    """Spam checking by IP address, using StopForumSpam data files
+
+    To enable StopForumSpam spam filtering in your Allura instance,
+    include the following parameters in your .ini file::
+
+        spam.method = stopforumspam
+        spam.stopforumspam.ip_addr_file = /path/to/listed_ip_180_all.txt
+
+    Of course you'll need to download the file from https://www.stopforumspam.com/downloads
and do so on a regular
+    basis to keep it updated.
+    """
+
+    def __init__(self, config):
+        self.packed_ips = set()
+        with open(config['spam.stopforumspam.ip_addr_file']) as f:
+            csv_file = csv.reader(f)
+            for record in csv_file:
+                if int(record[1]) > int(config.get('spam.stopforumspam.threshold', 20)):
+                    ip = record[0]
+                    # int is the smallest memory representation of an IP addr
+                    ip_int = int(ipaddress.ip_address(unicode(ip)))
+                    self.packed_ips.add(ip_int)
+        # to get actual memory usage, use: from pympler.asizeof import asizeof
+        log.info('Read stopforumspam file; %s recs, probably %s bytes stored in memory',
len(self.packed_ips),
+                 len(self.packed_ips) * getsizeof(iter(self.packed_ips).next()) * 2)
+
+    def check(self, text, artifact=None, user=None, content_type='comment', **kw):
+        ip = utils.ip_address(request)
+        ip_int = int(ipaddress.ip_address(unicode(ip)))
+        res = ip_int in self.packed_ips
+        self.record_result(res, artifact, user)
+        return res

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/tests/unit/spam/test_spam_filter.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/spam/test_spam_filter.py b/Allura/allura/tests/unit/spam/test_spam_filter.py
index 9d603f8..d05fa97 100644
--- a/Allura/allura/tests/unit/spam/test_spam_filter.py
+++ b/Allura/allura/tests/unit/spam/test_spam_filter.py
@@ -23,7 +23,7 @@ import unittest
 from ming.odm import ThreadLocalORMSession
 from nose.tools import assert_equal
 
-from allura.lib.spam import SpamFilter
+from allura.lib.spam import SpamFilter, ChainedSpamFilter
 from allura import model as M
 from allura.model.artifact import SpamCheckResult
 from alluratest.controller import setup_basic_test
@@ -32,8 +32,19 @@ from forgewiki import model as WM
 
 class MockFilter(SpamFilter):
 
+    def __init__(self, config):
+        self.config = config
+
     def check(*args, **kw):
         raise Exception("test exception")
+
+
+class MockFilter2(SpamFilter):
+
+    def __init__(self, config):
+        self.config = config
+
+    def check(*args, **kw):
         return True
 
 
@@ -82,3 +93,20 @@ class TestSpamFilterFunctional(object):
         assert_equal(len(results), 1)
         assert_equal(results[0].result, True)
         assert_equal(results[0].user.username, 'test-user')
+
+
+class TestChainedSpamFilter(object):
+
+    def test(self):
+        config = {'spam.method': 'mock1 mock2', 'spam.settingA': 'bcd'}
+        entry_points = {'mock1': MockFilter, 'mock2': MockFilter2}
+        checker = SpamFilter.get(config, entry_points)
+        assert isinstance(checker, ChainedSpamFilter)
+        assert len(checker.filters) == 2, checker.filters
+        assert_equal(checker.filters[0].config, {'spam.method': 'mock1', 'spam.settingA':
'bcd'})
+        assert_equal(checker.filters[1].config, {'spam.method': 'mock2', 'spam.settingA':
'bcd'})
+
+        assert checker.check()  # True because first filter errors out, and 2nd returns True
+
+        checker.submit_spam('some text')
+        checker.submit_ham('some text')

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/allura/tests/unit/spam/test_stopforumspam.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/spam/test_stopforumspam.py b/Allura/allura/tests/unit/spam/test_stopforumspam.py
new file mode 100644
index 0000000..64963cf
--- /dev/null
+++ b/Allura/allura/tests/unit/spam/test_stopforumspam.py
@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+import tempfile
+
+import mock
+
+from bson import ObjectId
+from nose.tools import assert_equal
+
+from allura.lib.spam.stopforumspamfilter import StopForumSpamSpamFilter
+
+
+class TestStopForumSpam(object):
+
+    def setUp(self):
+        self.content = u'spåm text'
+
+        self.artifact = mock.Mock()
+        self.artifact.project_id = ObjectId()
+        self.artifact.ref = None
+
+        with tempfile.NamedTemporaryFile() as f:
+            f.write('''"1.1.1.1","2","2018-01-22 10:56:29"
+"1.2.3.4","42","2017-09-24 18:33:00"
+"4.3.2.1","1","2017-09-28 14:03:53"''')
+            f.flush()
+            self.sfs = StopForumSpamSpamFilter({'spam.stopforumspam.ip_addr_file': f.name})
+
+    @mock.patch('allura.lib.spam.stopforumspamfilter.request')
+    def test_check(self, request):
+        request.remote_addr = '1.2.3.4'
+        assert_equal(True, self.sfs.check(self.content, artifact=self.artifact))
+
+        request.remote_addr = '1.1.1.1'
+        assert_equal(False, self.sfs.check(self.content, artifact=self.artifact))

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 4a7b9d9..d3f2700 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -237,13 +237,16 @@ user_prefs.maximum_claimed_emails = 20
 site_admin_project = allura
 site_admin_project_nbhd = Projects
 
-; Spam filtering service: mollom or akismet
+; Spam filtering service: this can be one or more of: mollom akismet stopforumspam
 ;spam.method = akismet
 ; for akismet:
 ;spam.key =
 ; for mollom:
 ;spam.public_key =
 ;spam.private_key =
+; for stopforumspam, should be a listed_ip_*_all.txt file
+;spam.stopforumspam.ip_addr_file =
+;spam.stopforumspam.threshold = 20
 
 ; Phone verification service: Nexmo Verify
 ; phone.method = nexmo

http://git-wip-us.apache.org/repos/asf/allura/blob/407559b0/Allura/setup.py
----------------------------------------------------------------------
diff --git a/Allura/setup.py b/Allura/setup.py
index 3005d07..30bca7d 100644
--- a/Allura/setup.py
+++ b/Allura/setup.py
@@ -106,6 +106,7 @@ setup(
     [allura.spam]
     akismet = allura.lib.spam.akismetfilter:AkismetSpamFilter
     mollom = allura.lib.spam.mollomfilter:MollomSpamFilter
+    stopforumspam = allura.lib.spam.stopforumspamfilter:StopForumSpamSpamFilter
 
     [allura.phone]
     nexmo = allura.lib.phone.nexmo:NexmoPhoneService


Mime
View raw message