ponymail-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From humbed...@apache.org
Subject [4/7] incubator-ponymail git commit: add a few new features to list editor
Date Sun, 10 Jul 2016 11:09:20 GMT
add a few new features to list editor

- Allow a single message to be edited, by ID
- Allow obfuscation of specific parts of a message
- Allow for test runs where no changes are applied


Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/75feb484
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/75feb484
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/75feb484

Branch: refs/heads/master
Commit: 75feb484af84d689d71542c380b5c5b7e919b6e4
Parents: 0e86b6d
Author: humbedooh <humbedooh@apache.org>
Authored: Mon Jul 4 15:35:46 2016 +0200
Committer: humbedooh <humbedooh@apache.org>
Committed: Mon Jul 4 15:35:46 2016 +0200

----------------------------------------------------------------------
 tools/edit-list.py | 79 +++++++++++++++++++++++++++++++++++--------------
 1 file changed, 56 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/75feb484/tools/edit-list.py
----------------------------------------------------------------------
diff --git a/tools/edit-list.py b/tools/edit-list.py
index c5a562a..b416834 100644
--- a/tools/edit-list.py
+++ b/tools/edit-list.py
@@ -46,6 +46,9 @@ wildcard = None
 debug = False
 notag = False
 desc = None
+mid = None
+dryrun = False
+obfuscate = None
 
 ssl = False
 dbname = config.get("elasticsearch", "dbname")
@@ -70,10 +73,14 @@ rootURL = ""
 parser = argparse.ArgumentParser(description='Command line options.')
 parser.add_argument('--source', dest='source', type=str, nargs=1,
                    help='Source list to edit')
+parser.add_argument('--mid', dest='mid', type=str, nargs=1,
+                   help='Source Message-ID to edit')
 parser.add_argument('--rename', dest='target', type=str, nargs=1,
                    help='(optional) new list ID')
 parser.add_argument('--desc', dest='desc', type=str, nargs=1,
                    help='(optional) new list description')
+parser.add_argument('--obfuscate', dest='obfuscate', type=str, nargs=1,
+                   help='Things to obfuscate in body, if any')
 parser.add_argument('--private', dest='private', action='store_true',
                    help='Make all emails in list private')
 parser.add_argument('--public', dest='public', action='store_true',
@@ -86,6 +93,8 @@ parser.add_argument('--debug', dest='debug', action='store_true',
                    help='Debug output - very noisy!')
 parser.add_argument('--notag', dest='notag', action='store_true',
                    help='List IDs do not have <> in them')
+parser.add_argument('--test', dest='test', action='store_true',
+                   help='Only test for occurrences, do not run the chosen action (dry run)')
 
 args = parser.parse_args()
 
@@ -107,13 +116,19 @@ if args.debug:
     debug = args.debug
 if args.notag:
     notag = args.notag
-
-
-if not sourceLID:
+if args.mid:
+    mid = args.mid[0]
+if args.obfuscate:
+    obfuscate = args.obfuscate[0]
+if args.test:
+    dryrun = args.test
+    
+    
+if not sourceLID and not mid:
     print("No source list ID specified!")
     parser.print_help()
     sys.exit(-1)
-if not (targetLID or makePrivate or makePublic or deleteEmails or desc):
+if not (targetLID or makePrivate or makePublic or deleteEmails or desc or obfuscate):
     print("Nothing to do! No target list ID or action specified")
     parser.print_help()
     sys.exit(-1)
@@ -122,12 +137,13 @@ if makePublic and makePrivate:
     parser.print_help()
     sys.exit(-1)
 
-sourceLID = ("%s" if notag else "<%s>")  % sourceLID.replace("@", ".").strip("<>")
+if sourceLID:
+    sourceLID = ("%s" if notag else "<%s>")  % sourceLID.replace("@", ".").strip("<>")
 if targetLID:
     targetLID = "<%s>" % targetLID.replace("@", ".").strip("<>")
 
 print("Beginning list edit:")
-print("  - List ID: %s" % sourceLID)
+print("  - List ID: %s" % (sourceLID if sourceLID else mid))
 if targetLID:
     print("  - Target ID: %s" % targetLID)
 if makePublic:
@@ -136,7 +152,8 @@ if makePrivate:
     print("  - Action: Mark all emails private")
 if deleteEmails:
     print("  - Action: Delete emails (sources will be kept!)")
-
+if obfuscate:
+    print("  - Action: Obfuscate parts of email containing: %s" % obfuscate)
 count = 0
 
 if desc:
@@ -154,9 +171,22 @@ if desc:
         }
     )
 
-if targetLID or makePrivate or makePublic or deleteEmails:
+if targetLID or makePrivate or makePublic or deleteEmails or mid:
+    if dryrun:
+        print("DRY RUN - NO CHANGES WILL BE MADE")
     print("Updating docs...")
     then = time.time()
+    terms = {
+        'wildcard' if wildcard else 'term': {
+            'list_raw': sourceLID
+        }
+    }
+    if mid:
+        terms = {
+            'term': {
+                'mid': mid
+            }
+        }
     page = es.search(
         index=dbname,
         doc_type="mbox",
@@ -167,11 +197,7 @@ if targetLID or makePrivate or makePublic or deleteEmails:
             'query': {
                 'bool': {
                     'must': [
-                        {
-                            'wildcard' if wildcard else 'term': {
-                                'list_raw': sourceLID
-                            }
-                        }
+                        terms
                     ]
                 }
             }
@@ -191,6 +217,10 @@ if targetLID or makePrivate or makePublic or deleteEmails:
         for hit in page['hits']['hits']:
             doc = hit['_id']
             body = {}
+            if obfuscate:
+                body['body'] = hit['_source']['body'].replace(obfuscate, "...")
+                body['subject'] = hit['_source']['subject'].replace(obfuscate, "...")
+                body['from'] = hit['_source']['from'].replace(obfuscate, "...")
             if targetLID:
                 body['list_raw'] = targetLID
                 body['list'] = targetLID
@@ -198,21 +228,24 @@ if targetLID or makePrivate or makePublic or deleteEmails:
                 body['private'] = True
             if makePublic:
                 body['private'] = False
-            js_arr.append({
-                '_op_type': 'delete' if deleteEmails else 'update',
-                '_index': dbname,
-                '_type': 'mbox',
-                '_id': doc,
-                'doc': body
-            })
+            if not dryrun:
+                js_arr.append({
+                    '_op_type': 'delete' if deleteEmails else 'update',
+                    '_index': dbname,
+                    '_type': 'mbox',
+                    '_id': doc,
+                    'doc': body
+                })
 
             count += 1
             if (count % 500 == 0):
                 print("Processed %u emails..." % count)
-                helpers.bulk(es, js_arr)
-                js_arr = []
+                if not dryrun:
+                    helpers.bulk(es, js_arr)
+                    js_arr = []
 
     if len(js_arr) > 0:
-        helpers.bulk(es, js_arr)
+        if not dryrun:
+            helpers.bulk(es, js_arr)
 
     print("All done, processed %u docs in %u seconds" % (count, time.time() - then))


Mime
View raw message