ponymail-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject [incubator-ponymail] branch master updated: Trim trailing spaces
Date Thu, 31 May 2018 22:15:59 GMT
This is an automated email from the ASF dual-hosted git repository.

sebb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-ponymail.git


The following commit(s) were added to refs/heads/master by this push:
     new c737d1f  Trim trailing spaces
c737d1f is described below

commit c737d1f65b4e8a9c95a47dd8a486b2afec19cb42
Author: Sebb <sebb@apache.org>
AuthorDate: Thu May 31 23:15:56 2018 +0100

    Trim trailing spaces
---
 tools/archiver.py      | 62 +++++++++++++++++++++++++-------------------------
 tools/copy-list.py     | 14 ++++++------
 tools/edit-list.py     |  2 +-
 tools/elastic.py       | 14 ++++++------
 tools/feedwrapper.py   |  5 ++--
 tools/generators.py    | 18 +++++++--------
 tools/import-mbox.py   | 42 +++++++++++++++++-----------------
 tools/install.py       | 25 ++++++++++----------
 tools/list-lists.py    |  8 +++----
 tools/mboxo_patch.py   |  2 +-
 tools/missing.py       |  4 ++--
 tools/push-failures.py |  4 ++--
 tools/setup.py         | 38 +++++++++++++++----------------
 13 files changed, 118 insertions(+), 120 deletions(-)

diff --git a/tools/archiver.py b/tools/archiver.py
index d97fde9..1feccdc 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -227,8 +227,8 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                 attachments.append(part_meta)
                 contents[part_meta['hash']] = part_file
         return attachments, contents
-    
-    
+
+
     def msgbody(self, msg):
         body = None
         firstHTML = None
@@ -249,11 +249,11 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                     firstHTML = part.get_payload(decode=True)
             except Exception as err:
                 print(err)
-            
+
         # this requires a GPL lib, user will have to install it themselves
         if firstHTML and (not body or len(body) <= 1 or (iBody and str(body).find(str(iBody))
!= -1)):
             body = self.html2text(firstHTML.decode("utf-8", 'ignore') if type(firstHTML)
is bytes else firstHTML)
- 
+
         # See issue#463
         # This code will try at most one charset
         # If the decode fails, it will use utf-8
@@ -264,8 +264,8 @@ class Archiver(object): # N.B. Also used by import-mbox.py
             except:
                 body = body.decode('utf-8', errors='replace') if type(body) is bytes else
body
                 # at this point body can no longer be bytes
-                
-        return body    
+
+        return body
 
     # N.B. this is also called by import-mbox.py
     def compute_updates(self, lid, private, msg):
@@ -288,7 +288,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
             # Standard crop out?
             else:
                 lid = lid.replace(self.cropout, "")
-        
+
         defaultEmptyString = lambda value: value and str(value) or ""
         msg_metadata = dict([(k, defaultEmptyString(msg.get(k))) for k in self.keys])
         mid = hashlib.sha224(str("%s-%s" % (lid, msg_metadata['archived-at'])).encode('utf-8')).hexdigest()
+ "@" + (lid if lid else "none")
@@ -383,7 +383,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
             }
 
         return  ojson, contents, msg_metadata, irt
-            
+
     def archive_message(self, mlist, msg, raw_msg):
         """Send the message to the archiver.
 
@@ -423,7 +423,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                             'source': contents[key]
                         }
                     )
-            
+
             self.index(
                 index=self.dbname,
                 doc_type="mbox",
@@ -431,7 +431,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                 consistency = self.consistency,
                 body = ojson
             )
-            
+
             self.index(
                 index=self.dbname,
                 doc_type="mbox_source",
@@ -463,7 +463,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                 sys.exit(0) # We're exiting here, the rest can't be done without ES
             # otherwise fail as before
             raise err
-                
+
         # If MailMan and list info is present, save/update it in ES:
         if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description
and mlist.list_name:
             self.index(
@@ -478,11 +478,11 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                     'private': private
                 }
             )
-        
+
         if logger:
             logger.info("Pony Mail archived message %s successfully", ojson['mid'])
         oldrefs = []
-        
+
         # Is this a direct reply to a pony mail email?
         if irt != "":
             dm = re.search(r"pony-([a-f0-9]+)-([a-f0-9]+)@", irt)
@@ -524,7 +524,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
                 mid = im.group(2)
                 if self.es.exists(index = self.dbname, doc_type = 'account', id = cid):
                     doc = self.es.get(index = self.dbname, doc_type = 'account', id = cid)
-                    
+
                     # does the user want to be notified of indirect replies?
                     if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications')
== 'indirect' and not cid in oldrefs:
                         oldrefs.append(cid)
@@ -571,7 +571,7 @@ class Archiver(object): # N.B. Also used by import-mbox.py
         """ Required by MM3 plugin API
         """
         return None
-    
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Command line options.')
     parser.add_argument('--lid', dest='lid', type=str, nargs=1,
@@ -582,22 +582,22 @@ if __name__ == '__main__':
                        help='(optional) source IP (mail server) to allow posts from, ignore
if no match')
     parser.add_argument('--ignore', dest='ignorefrom', type=str, nargs=1,
                        help='Sender/list to ignore input from (owner etc)')
-    parser.add_argument('--private', dest='private', action='store_true', 
+    parser.add_argument('--private', dest='private', action='store_true',
                        help='This is a private archive')
-    parser.add_argument('--makedate', dest='makedate', action='store_true', 
+    parser.add_argument('--makedate', dest='makedate', action='store_true',
                        help='Use the archive timestamp as the email date instead of the Date
header')
-    parser.add_argument('--quiet', dest='quiet', action='store_true', 
+    parser.add_argument('--quiet', dest='quiet', action='store_true',
                        help='Do not exit -1 if the email could not be parsed')
-    parser.add_argument('--verbose', dest='verbose', action='store_true', 
+    parser.add_argument('--verbose', dest='verbose', action='store_true',
                        help='Output additional log messages')
-    parser.add_argument('--html2text', dest='html2text', action='store_true', 
+    parser.add_argument('--html2text', dest='html2text', action='store_true',
                        help='Try to convert HTML to text if no text/plain message is found')
     parser.add_argument('--dry', dest='dry', action='store_true',
                        help='Do not save emails to elasticsearch, only test parsing')
     parser.add_argument('--dumponfail', dest='dump',
                        help='If pushing to ElasticSearch fails, dump documents in JSON format
to this directory and fail silently.')
     args = parser.parse_args()
-    
+
     if args.html2text:
         parseHTML = True
     if args.dump:
@@ -609,23 +609,23 @@ if __name__ == '__main__':
         # Also eliminates: 'Undecodable raw error response from server:' warning message
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
-        
+
     archie = Archiver(parseHTML = parseHTML)
     # use binary input so parser can use appropriate charset
     input_stream = sys.stdin.buffer
-    
+
     try:
         msgstring = input_stream.read()
         try:
             msg = email.message_from_bytes(msgstring)
         except Exception as err:
             print("STDIN parser exception: %s" % err)
-        
+
         # We're reading from STDIN, so let's fake an MM3 call
         ispublic = True
         ignorefrom = None
         allowfrom = None
-            
+
         if args.altheader:
             altheader = args.altheader[0]
             if altheader in msg:
@@ -647,15 +647,15 @@ if __name__ == '__main__':
                 msg.replace_header('List-ID', args.lid[0])
             except:
                 msg.add_header('list-id', args.lid[0])
-                
-                
+
+
         #Ignore based on --ignore flag?
         if args.ignorefrom:
             ignorefrom = args.ignorefrom[0]
             if fnmatch.fnmatch(msg.get("from"), ignorefrom) or (msg.get("list-id") and fnmatch.fnmatch(msg.get("list-id"),
ignorefrom)):
                 print("Ignoring message as instructed by --ignore flag")
                 sys.exit(0)
-        
+
         # Check CIDR if need be
         if args.allowfrom:
             from netaddr import IPNetwork, IPAddress
@@ -678,14 +678,14 @@ if __name__ == '__main__':
         # Replace date header with $now?
         if args.makedate:
             msg.replace_header('date', email.utils.formatdate())
-            
+
         if args.private:
             ispublic = False
         if 'list-id' in msg:
             if not msg.get('archived-at'):
                 msg.add_header('archived-at', email.utils.formatdate())
             list_data = namedtuple('importmsg', ['list_id', 'archive_public'])(list_id =
msg.get('list-id'), archive_public=ispublic)
-            
+
             try:
                 lid, mid = archie.archive_message(list_data, msg, msgstring)
                 print("%s: Done archiving to %s as %s!" % (email.utils.formatdate(), lid,
mid))
@@ -705,4 +705,4 @@ if __name__ == '__main__':
         else:
             print("Could not parse email: %s (@ %s)" % (err, line))
             sys.exit(-1)
-            
+
diff --git a/tools/copy-list.py b/tools/copy-list.py
index 982b808..ab7100e 100755
--- a/tools/copy-list.py
+++ b/tools/copy-list.py
@@ -27,7 +27,7 @@ import sys
 import time
 import argparse
 
-from elastic import Elastic    
+from elastic import Elastic
 
 sourceLID = None
 targetLID = None
@@ -50,9 +50,9 @@ parser.add_argument('--target', dest='target', type=str,
                    metavar='<list id>', help='(optional) new list ID')
 parser.add_argument('--newdb', dest='newdb', type=str,
                    metavar='<index name>', help='(optional) new ES database name')
-parser.add_argument('--wildcard', dest='glob', action='store_true', 
+parser.add_argument('--wildcard', dest='glob', action='store_true',
                    help='Allow wildcards in --source')
-parser.add_argument('--notag', dest='notag', action='store_true', 
+parser.add_argument('--notag', dest='notag', action='store_true',
                    help='List IDs do not have <> in them')
 
 args = parser.parse_args()
@@ -79,7 +79,7 @@ if targetLID == sourceLID and not newdb:
     print("Nothing to do! Target same as source")
     parser.print_help()
     sys.exit(-1)
-    
+
 print("Beginning list copy:")
 print("  - Source ID: %s" % sourceLID)
 if targetLID:
@@ -92,7 +92,7 @@ if newdb:
 
 count = 0
 
-    
+
 print("Updating docs...")
 then = time.time()
 page = es.search(
@@ -147,7 +147,7 @@ while (scroll_size > 0):
             })
         else:
             print("Source for %s not found, hmm..." % doc)
-        
+
         count += 1
         if (count % 50 == 0):
             print("Processed %u emails..." % count)
@@ -156,5 +156,5 @@ while (scroll_size > 0):
 
 if len(js_arr) > 0:
     es.bulk(js_arr)
-            
+
 print("All done, processed %u docs in %u seconds" % (count, time.time() - then))
diff --git a/tools/edit-list.py b/tools/edit-list.py
index aa33850..0e920f0 100755
--- a/tools/edit-list.py
+++ b/tools/edit-list.py
@@ -89,7 +89,7 @@ dryrun = args.test
 privacyChange = makePrivate or makePublic
 otherChange = targetLID or desc or obfuscate
 anyChange = privacyChange or otherChange
-    
+
 if not sourceLID and not mid:
     print("No source list ID specified!")
     parser.print_help()
diff --git a/tools/elastic.py b/tools/elastic.py
index f0c35ca..897cb64 100755
--- a/tools/elastic.py
+++ b/tools/elastic.py
@@ -43,7 +43,7 @@ class Elastic:
         if config.has_option('elasticsearch', 'user'):
             auth = (config.get('elasticsearch','user'), config.get('elasticsearch','password'))
 
-    
+
         # elasticsearch logs lots of warnings on retries/connection failure
         logging.getLogger("elasticsearch").setLevel(logging.ERROR)
 
@@ -85,7 +85,7 @@ class Elastic:
         return self.dbVersion
     def engineMajor(self):
         return int(self.engineVersion().split('.')[0])
-        
+
     def getdbname(self):
         return self.dbname
 
@@ -101,7 +101,7 @@ class Elastic:
             index=self.dbname,
             **kwargs
         )
-    
+
     def update(self, **kwargs):
         return self.es.update(
             index=self.dbname,
@@ -117,13 +117,13 @@ class Elastic:
             scroll = scroll,
             **kwargs
         )
-    
+
     def get(self, **kwargs):
         return self.es.get(index=self.dbname, **kwargs)
 
     def scroll(self, **kwargs):
         return self.es.scroll(**kwargs)
-    
+
     def info(self, **kwargs):
         return self.es.info(**kwargs)
 
@@ -131,9 +131,9 @@ class Elastic:
         return helpers.bulk(self.es, actions, **kwargs)
 
     def clear_scroll(self, *args, **kwargs):
-        """ 
+        """
             Call this to release the scroll id and its resources
-    
+
             It looks like the Python library already releases the SID
             if the caller scrolls to the end of the results, so only need to call this
             when terminating scrolling early.
diff --git a/tools/feedwrapper.py b/tools/feedwrapper.py
index 2c49f57..6c760e5 100755
--- a/tools/feedwrapper.py
+++ b/tools/feedwrapper.py
@@ -42,7 +42,7 @@ subscribe
         print("Sent subscription request for %s to %s" % (sender, recip))
     else:
         msg = email.message_from_file(sys.stdin)
-        
+
         if msg.get('to') and msg.get('reply-to') and msg.get('subject'):
             if msg.get('to').find(sys.argv[1]) != -1 and \
                 re.search(r"-request@", msg.get('reply-to')) or \
@@ -53,7 +53,7 @@ subscribe
                 with open("%s/wrapper.log" % path, "a") as f:
                     f.write("%s - %s: %s\n" % (msg.get('to'), msg.get('reply-to'), msg.get('subject')))
                     f.write("We've got a subscription request for %s. \n" % msg.get('reply-to'))
-                
+
                 smtpObj = smtplib.SMTP('localhost')
                 smtpObj.sendmail(sys.argv[1], [msg.get('reply-to')], """From: %s
 To: %s
@@ -70,4 +70,3 @@ Subject: %s
                     print(p.communicate(input=msg.as_string().encode('utf-8')))
                     p.stdin.close()
                     f.write("-----\n")
-            
\ No newline at end of file
diff --git a/tools/generators.py b/tools/generators.py
index b3a4b20..ab090d6 100644
--- a/tools/generators.py
+++ b/tools/generators.py
@@ -32,7 +32,7 @@ def full(msg, _body, lid, _attachments):
     (including server-dependent data)
     The id is almost certainly unique,
     but different copies of the message are likely to have different headers, thus ids
-    
+
     Parameters:
     msg - the parsed message
     _body - the parsed text content (not used)
@@ -114,7 +114,7 @@ def cluster(msg, body, lid, attachments):
     - the hashes of any attachments
 
     Note: the lid is not included in the hash.
-    
+
     Parameters:
     msg - the parsed message
     body - the parsed text content
@@ -127,13 +127,13 @@ def cluster(msg, body, lid, attachments):
     if not body: # Make sure body is not None, which will fail.
         body = ""
     xbody = body if type(body) is bytes else body.encode('ascii', 'ignore')
-    
+
     # Crop out any trailing whitespace in body
     xbody = re.sub(b"\s+$", b"", xbody)
-    
+
     # Use Message-Id (or '' if missing)
     xbody += bytes(msg.get('Message-Id', ''), encoding='ascii')
-    
+
     # Use Date header. Don't use archived-at, as the archiver sets this if not present.
     mdate = None
     mdatestring = "(null)" # Default to null, ONLY changed if replicable across imports
@@ -143,17 +143,17 @@ def cluster(msg, body, lid, attachments):
     except:
         pass
     xbody += bytes(mdatestring, encoding='ascii')
-    
+
     # Use sender
     sender = msg.get('from', None)
     if sender:
         xbody += bytes(sender, encoding = 'ascii')
-    
+
     # Use subject
     subject = msg.get('subject', None)
     if subject:
         xbody += bytes(subject, encoding = 'ascii')
-    
+
     # Use attachment hashes if present
     if attachments:
         for a in attachments:
@@ -170,7 +170,7 @@ def legacy(msg, body, lid, _attachments):
     Original generator - DO NOT USE
     (does not generate unique ids)
 
-    The hash input is created from 
+    The hash input is created from
     - body: if bytes as is else encoded ascii, ignoring invalid characters; if the body is
null an Exception is thrown
 
     The uid_mdate for the id is the Date converted to UTC epoch else 0
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index 403d6b7..e2a6a2d 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
- 
+
 # -*- coding: utf-8 -*-
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -36,7 +36,7 @@ import tempfile
 import gzip
 
 import archiver
-    
+
 goodies = 0
 baddies = 0
 duplicates={} # detect if mid is re-used this run
@@ -122,7 +122,7 @@ class SlurpThread(Thread):
         filename = ""
 
         archie = archiver.Archiver(parseHTML = parseHTML)
-    
+
         while len(lists) > 0:
             self.printid("%u elements left to slurp" % len(lists))
 
@@ -148,7 +148,7 @@ class SlurpThread(Thread):
                         yield email.message_from_bytes(msgbytes)
                 messages = mailgen(mla[0])
             elif filebased:
-                
+
                 tmpname = mla[0]
                 filename = mla[0]
                 if filename.find(".gz") != -1:
@@ -179,7 +179,7 @@ class SlurpThread(Thread):
                 self.printid("Slurping %s/%s" % (ml, mboxfile))
                 ctx = urlopen("%s%s/%s" % (source, ml, mboxfile ))
                 inp = ctx.read().decode(ctx.headers.get_content_charset() or 'utf-8', errors='ignore')
-    
+
                 tmpname = hashlib.sha224(("%f-%f-%s-%s.mbox" % (random.random(), time.time(),
ml, mboxfile)).encode('utf-8') ).hexdigest()
                 with open(tmpname, "w") as f:
                     f.write(inp)
@@ -190,8 +190,8 @@ class SlurpThread(Thread):
 
             count = 0
             bad = 0
-            
-            
+
+
             for key in messages.iterkeys():
                 message=messages.get(key)
                 # If --filter is set, discard any messages not matching by continuing to
next email
@@ -221,7 +221,7 @@ class SlurpThread(Thread):
                     continue
 
                 json, contents, _msgdata, _irt = archie.compute_updates(list_override, private,
message)
-                
+
                 # Not sure this can ever happen
                 if json and not (json['list'] and json['list_raw']):
                     self.printid("No list id found for %s " % json['message-id'])
@@ -246,7 +246,7 @@ class SlurpThread(Thread):
                                         {
                                             'term': {
                                                 'list_raw': json['list']
-                                            }                                           
     
+                                            }
                                         }
                                     ]
                                 }
@@ -300,7 +300,7 @@ class SlurpThread(Thread):
                     if len(ja) >= 40:
                         bulk_insert(self.name, ja, es, 'mbox')
                         ja = []
-                        
+
                         bulk_insert(self.name, jas, es, 'mbox_source')
                         jas = []
                 else:
@@ -316,18 +316,18 @@ class SlurpThread(Thread):
             else:
                 self.printid("Parsed %s/%s: %u records (failed: %u) from %s" % (ml, mboxfile,
count, bad, tmpname))
                 os.unlink(tmpname)
-                
+
             goodies += count
             baddies += bad
             if len(ja) > 0:
                 bulk_insert(self.name, ja, es, 'mbox')
             ja = []
-            
+
             if len(jas) > 0:
                 bulk_insert(self.name, jas, es, 'mbox_source')
             jas = []
         self.printid("Done, %u elements left to slurp" % len(lists))
-        
+
 parser = argparse.ArgumentParser(description='Command line options.')
 parser.add_argument('--source', dest='source', type=str, nargs=1,
                    help='Source to scan (http(s)://, imap(s):// or file path)')
@@ -454,16 +454,16 @@ def globDir(d):
             print("alright, I'll try to figure it out myself!")
     for fi in sorted(mboxes):
         lists.append([fi, fileToLID.get(d) if fileToLID.get(d) else list_override])
-    
+
     for nd in sorted(dirs):
         globDir(join(d,nd))
- 
+
 
 # HTTP(S) based import?
 if re.match(r"https?://", source):
     data = urlopen(source).read().decode('utf-8')
     print("Fetched %u bytes of main data, parsing month lists" % len(data))
-    
+
     if project:
         # ensure there is a '-' between project and list name otherwise we match too much
         # Note: It looks like mod_mbox always uses single quoted hrefs
@@ -487,7 +487,7 @@ if re.match(r"https?://", source):
                     print("Adding %s/%s to slurp list" % (ml, mboxfile))
                     if quickmode and qn >= 2:
                         break
-    
+
     if args.pipermail:
         filebased = True
         piperWeirdness = True
@@ -520,7 +520,7 @@ elif re.match(r"imaps?://", source):
     imap = True
     import urllib, getpass, imaplib
     url = urllib.parse.urlparse(source)
-    
+
     port = url.port or (143 if url.scheme == 'imap' else 993)
     user = url.username or getpass.getuser()
     password = url.password or getpass.getpass('IMAP Password: ')
@@ -529,10 +529,10 @@ elif re.match(r"imaps?://", source):
 
     # fetch message-id => _id pairs from elasticsearch
 
-    result = es.search(scroll = '5m', 
+    result = es.search(scroll = '5m',
         body = {
-            'size': 1024, 
-            'fields': ['message-id'], 
+            'size': 1024,
+            'fields': ['message-id'],
             'query': {'match': {'list': listname}}
         }
     )
diff --git a/tools/install.py b/tools/install.py
index fdd6953..3323e8b 100755
--- a/tools/install.py
+++ b/tools/install.py
@@ -32,28 +32,28 @@ if dname == 'ubuntu' or dname == 'debian':
     print("Running installation script for Debian/Ubuntu servers, hang on!")
     print("Installing pre-requisites via apt-get")
     subprocess.check_call(('apt-get', 'install', 'apache2', 'git', 'liblua5.2-dev', 'lua-cjson',
'lua-sec', 'lua-socket', 'python3', 'python3-pip', 'subversion'))
-    
+
     print("Installing Python modules")
     subprocess.check_call(('pip3', 'install', 'elasticsearch', 'formatflowed'))
-    
+
     print("Installing ElasticSearch")
     subprocess.check_call(('apt-get', 'install', 'openjdk-7-jre-headless'))
-    
+
     try:
         subprocess.check_call(("wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch
| sudo apt-key add -"), shell=True)
         subprocess.check_call(('echo "deb http://packages.elastic.co/elasticsearch/1.7/debian
stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-1.7.list'), shell=True)
     except:
         print("Did we already add ES to the repo? hmm")
-    
+
     subprocess.check_call(('apt-get', 'update'))
     subprocess.check_call(('apt-get', 'install', 'elasticsearch'))
-    
+
     print("Checking out a copy of Pony Mail from GitHub")
     subprocess.check_call(('git', 'clone', 'https://github.com/Humbedooh/ponymail.git', '/var/www/ponymail'))
-    
+
     print("Starting ElasticSearch")
     subprocess.check_call(('service', 'elasticsearch', 'start'))
-    
+
     print("Writing httpd configuration file /etc/apache2/sites-enabled/99-ponymail.conf")
     with open("/etc/apache2/sites-enabled/99-ponymail.conf", "w") as f:
         f.write("""
@@ -65,19 +65,18 @@ if dname == 'ubuntu' or dname == 'debian':
     LuaCodeCache    stat
     AcceptPathInfo  On
 </VirtualHost>""")
-    
+
     if dname == 'ubuntu' and dver == '14.04':
         print("Ubuntu 14.04 specific step; Compiling mod_lua")
         subprocess.check_call(('apt-get', 'install', 'apache2-dev'))
         subprocess.check_call(('svn', 'co', 'https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x/modules/lua/',
'/tmp/mod_lua'))
         subprocess.check_call(("cd /tmp/mod_lua && apxs2 -I/usr/include/lua5.2 -cia
mod_lua.c lua_*.c -lm -llua5.2"), shell=True)
-    
+
     print("Enabling mod_lua")
     subprocess.check_call(('a2enmod', 'lua'))
-    
+
     print("Starting httpd")
     subprocess.check_call(('service', 'apache2', 'start'))
-        
+
     print("Done! Please run setup.py now to set up Pony Mail")
-    
-    
\ No newline at end of file
+
diff --git a/tools/list-lists.py b/tools/list-lists.py
index 6ad8951..8e265f2 100755
--- a/tools/list-lists.py
+++ b/tools/list-lists.py
@@ -26,11 +26,11 @@ dbname=None
 parser = argparse.ArgumentParser(description='Command line options.')
 parser.add_argument('--dbname', dest='dbname', type=str,
                    help='Override index name')
-parser.add_argument('--pretty', dest='pretty', action='store_true', 
+parser.add_argument('--pretty', dest='pretty', action='store_true',
                    help='Convert List IDs to email addresses')
-parser.add_argument('--debug', dest='debug', action='store_true', 
+parser.add_argument('--debug', dest='debug', action='store_true',
                    help='Output the result JSON instead, very noisy!')
-parser.add_argument('--counts', dest='counts', action='store_true', 
+parser.add_argument('--counts', dest='counts', action='store_true',
                    help='Show the count of messages for each list')
 
 args = parser.parse_args()
@@ -100,7 +100,7 @@ else:
                 print(listid, msgcount, prvcount)
             else:
                 print(listid)
-    
+
     for dom in sorted(plist):
         for ln in sorted(plist[dom]):
             if args.counts:
diff --git a/tools/mboxo_patch.py b/tools/mboxo_patch.py
index 92e4429..a577567 100644
--- a/tools/mboxo_patch.py
+++ b/tools/mboxo_patch.py
@@ -62,7 +62,7 @@ class MboxoReader(mailbox._PartialFile): # pylint: disable=W0212
 
     # Override the read method to provide mboxo filtering
     def _read(self, size, read_method):
-        # get the next chunk, resetting if necessary 
+        # get the next chunk, resetting if necessary
         if self.remain != 0:
             super().seek(whence=1, offset=-self.remain)
         # if size is None or negative, then read returns everything.
diff --git a/tools/missing.py b/tools/missing.py
index 88588b0..5871fd9 100755
--- a/tools/missing.py
+++ b/tools/missing.py
@@ -32,7 +32,7 @@ source_group.add_argument('--mid', dest='mid', type=str, metavar='message-id',
                    help='Source Message-ID to edit')
 
 action_group = parser.add_mutually_exclusive_group(required=True)
-# N.B. Use nargs=1 below, because the same field is used for get and set 
+# N.B. Use nargs=1 below, because the same field is used for get and set
 action_group.add_argument('--listmissing', dest='missing', type=str, nargs=1, metavar='fieldname',
                    help='list missing fields')
 action_group.add_argument('--setmissing', dest='missing', type=str, nargs=2, metavar=('fieldname',
'value'),
@@ -133,7 +133,7 @@ if args.source:
                 if setField:
                     update(elastic, js_arr)
                 js_arr = []
-    
+
     print("Processed %u emails." % count)
     if len(js_arr) > 0:
         if setField:
diff --git a/tools/push-failures.py b/tools/push-failures.py
index 2fc6a67..21da927 100755
--- a/tools/push-failures.py
+++ b/tools/push-failures.py
@@ -53,13 +53,13 @@ for f in files:
                 id=mid,
                 body = ojson['mbox']
             )
-            
+
             es.index(
                 doc_type="mbox_source",
                 id=mid,
                 body = ojson['mbox_source']
             )
-            
+
             if 'attachments' in ojson and ojson['attachments']:
                 for k, v in ojson['attachments'].items():
                     es.index(
diff --git a/tools/setup.py b/tools/setup.py
index 1463f2b..8fa8725 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -30,7 +30,7 @@ try:
     from elasticsearch import Elasticsearch
 except ImportError:
     dopip = True
-    
+
 if dopip and (getpass.getuser() != "root"):
     print("It looks like you need to install some python modules first")
     print("Either run this as root to do so, or run: ")
@@ -59,7 +59,7 @@ ES_MAJOR = ES_VERSION[0]
 # CLI arg parsing
 parser = argparse.ArgumentParser(description='Command line options.')
 
-parser.add_argument('--defaults', dest='defaults', action='store_true', 
+parser.add_argument('--defaults', dest='defaults', action='store_true',
                    help='Use default settings')
 parser.add_argument('--dbprefix', dest='dbprefix')
 parser.add_argument('--clobber', dest='clobber', action='store_true',
@@ -78,17 +78,17 @@ parser.add_argument('--mailserver', dest='mailserver', type=str,
                    help='Host name of outgoing mail server')
 parser.add_argument('--mldom', dest='mldom', type=str,
                    help='Domains to accept mail for via UI')
-parser.add_argument('--wordcloud', dest='wc', action='store_true', 
+parser.add_argument('--wordcloud', dest='wc', action='store_true',
                    help='Enable word cloud')
-parser.add_argument('--skiponexist', dest='soe', action='store_true', 
+parser.add_argument('--skiponexist', dest='soe', action='store_true',
                    help='Skip setup if ES index exists')
-parser.add_argument('--noindex', dest='noi', action='store_true', 
+parser.add_argument('--noindex', dest='noi', action='store_true',
                    help="Don't make an ES index, assume it exists")
-parser.add_argument('--nocloud', dest='nwc', action='store_true', 
+parser.add_argument('--nocloud', dest='nwc', action='store_true',
                    help='Do not enable word cloud')
 parser.add_argument('--generator', dest='generator', type=str,
                    help='Document ID Generator to use (legacy, medium, cluster, full)')
-args = parser.parse_args()    
+args = parser.parse_args()
 
 print("Welcome to the Pony Mail setup script!")
 print("Let's start by determining some settings...")
@@ -144,13 +144,13 @@ if args.dbreplicas:
     replicas = args.dbreplicas
 if args.generator:
     genname = args.generator
-    
+
 while hostname == "":
     hostname = input("What is the hostname of the ElasticSearch server? (e.g. localhost):
")
 
 while urlPrefix == None:
     urlPrefix = input("Database URL prefix if any (hit enter if none): ")
-    
+
 while port < 1:
     try:
         port = int(input("What port is ElasticSearch listening on? (normally 9200): "))
@@ -162,7 +162,7 @@ while dbname == "":
 
 while mlserver == "":
     mlserver = input("What is the hostname of the outgoing mailserver? (e.g. mail.foo.org):
")
-    
+
 while mldom == "":
     mldom = input("Which domains would you accept mail to from web-replies? (e.g. foo.org
or *): ")
 
@@ -184,7 +184,7 @@ while genname == "":
             genname = gens[gno-1]
     except ValueError:
         pass
-    
+
 while shards < 1:
     try:
         shards = int(input("How many shards for the ElasticSearch index? "))
@@ -460,10 +460,10 @@ def createIndex():
                 "settings": settings
             }
         )
-    
+
     print("Index created! %s " % res)
 
-# we need to connect to database to determine the engine version   
+# we need to connect to database to determine the engine version
 es = Elasticsearch([
     {
         'host': hostname,
@@ -517,7 +517,7 @@ print("Writing importer config (%s)" % ponymail_cfg)
 with open(ponymail_cfg, "w") as f:
     f.write("""
 ###############################################################
-# Pony Mail Configuration file                                             
+# Pony Mail Configuration file
 
 # Main ES configuration
 [elasticsearch]
@@ -542,7 +542,7 @@ generator:              %s
 #cropout:               string to crop from list-id
 
 ###############################################################
-            """ % (hostname, dbname, port, 
+            """ % (hostname, dbname, port,
                    'wait:                  active shard count' if DB_MAJOR == 5 else 'write:
                consistency level (default quorum)', genname))
 
 config_path = "../site/api/lib"
@@ -575,7 +575,7 @@ local config = {
 --            env = 'subprocess' -- use environment vars instead of request headers
 --        }
     },
---  allow_insecure_cookie = true, -- override the default (false) - only use for test installations

+--  allow_insecure_cookie = true, -- override the default (false) - only use for test installations
 --  no_association = {}, -- domains that are not allowed for email association
 --  listsDisplay = 'regex', -- if defined, hide list names that don't match the regex
 --  debug = false, -- whether to return debug information
@@ -583,11 +583,11 @@ local config = {
 }
 return config
             """ % (hostname, port, dbname, mlserver, mldom, "true" if wce else "false"))
-    
+
 print("Copying sample JS config to config.js (if needed)...")
 if not os.path.exists("../site/js/config.js") and os.path.exists("../site/js/config.js.sample"):
     shutil.copy("../site/js/config.js.sample", "../site/js/config.js")
-    
-    
+
+
 print("All done, Pony Mail should...work now :)")
 print("If you are using an external mail inbound server, \nmake sure to copy archiver.py
and ponymail.cfg to it")

-- 
To stop receiving notification emails like this one, please contact
sebb@apache.org.

Mime
View raw message