ponymail-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject incubator-ponymail git commit: ES 5.0 no longer supports the write consistency option; archiver fails
Date Sat, 28 Jan 2017 23:49:17 GMT
Repository: incubator-ponymail
Updated Branches:
  refs/heads/master 06612e68a -> 848bb7129


ES 5.0 no longer supports the write consistency option; archiver fails

This fixes #351

Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/848bb712
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/848bb712
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/848bb712

Branch: refs/heads/master
Commit: 848bb7129dede6f2c01ead4ce49ab070be6ad787
Parents: 06612e6
Author: Sebb <sebb@apache.org>
Authored: Sat Jan 28 23:48:29 2017 +0000
Committer: Sebb <sebb@apache.org>
Committed: Sat Jan 28 23:48:29 2017 +0000

----------------------------------------------------------------------
 CHANGELOG.md      |  1 +
 tools/archiver.py | 31 +++++++++++++++++++++++++------
 tools/setup.py    |  7 +++++--
 3 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/848bb712/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e8d0d16..69ff74e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -134,6 +134,7 @@
 - Use constant for max list count instead of 500000 (#352)
 - Tighten wildcard searches to only search in the same domain level (#348)
 - useless conditional when fetching id parameter (#353)
+- ES 5.0 no longer supports the write consistency option for index(); archiver fails (#351)
 
 
 ## CHANGES in 0.9b:

http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/848bb712/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index c03bf8f..bf4033b 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -40,6 +40,8 @@ sub someone to the list(s) and add this to their .forward file:
 logger = None
 
 from elasticsearch import Elasticsearch
+from elasticsearch import VERSION as ES_VERSION
+ES_MAJOR = ES_VERSION[0]
 from formatflowed import convertToWrapped
 import hashlib
 import email.utils
@@ -140,6 +142,16 @@ class Archiver(object):
         "x-mailman-rule-misses",
     ]
 
+    """ Intercept index calls and fix up consistency argument """
+    def index(self, **kwargs):
+        if ES_MAJOR == 5:
+            if kwargs.pop('consistency', None): # drop the key if present
+                if self.wait_for_active_shards: # replace with wait if defined
+                    kwargs['wait_for_active_shards'] = self.wait_for_active_shards
+        return self.es.index(
+            **kwargs
+        )
+
     def __init__(self, parseHTML=False):
         """ Just initialize ES. """
         self.html = parseHTML
@@ -148,7 +160,14 @@ class Archiver(object):
             self.html2text = html2text.html2text
         self.dbname = config.get("elasticsearch", "dbname")
         ssl = config.get("elasticsearch", "ssl", fallback="false").lower() == 'true'
+        # Always allow this to be set; will be replaced as necessary by wait_for_active_shards
         self.consistency = config.get('elasticsearch', 'write', fallback='quorum')
+        if ES_MAJOR == 2:
+            pass
+        elif ES_MAJOR == 5:
+            self.wait_for_active_shards = config.get('elasticsearch', 'wait', fallback=1)
+        else:
+            raise Exception("Unexpected elasticsearch version ", ES_VERSION)
         self.cropout = config.get("debug", "cropout", fallback=None)
         uri = config.get("elasticsearch", "uri", fallback="")
         dbs = [
@@ -383,7 +402,7 @@ class Archiver(object):
 
         if contents:
             for key in contents:
-                self.es.index(
+                self.index(
                     index=self.dbname,
                     doc_type="attachment",
                     id=key,
@@ -392,7 +411,7 @@ class Archiver(object):
                     }
                 )
     
-        self.es.index(
+        self.index(
             index=self.dbname,
             doc_type="mbox",
             id=ojson['mid'],
@@ -400,7 +419,7 @@ class Archiver(object):
             body = ojson
         )
         
-        self.es.index(
+        self.index(
             index=self.dbname,
             doc_type="mbox_source",
             id=ojson['mid'],
@@ -413,7 +432,7 @@ class Archiver(object):
         
         # If MailMan and list info is present, save/update it in ES:
         if hasattr(mlist, 'description') and hasattr(mlist, 'list_name') and mlist.description
and mlist.list_name:
-            self.es.index(
+            self.index(
                 index=self.dbname,
                 doc_type="mailinglists",
                 id=lid,
@@ -441,7 +460,7 @@ class Archiver(object):
                     if doc:
                         oldrefs.append(cid)
                         # N.B. no index is supplied, so ES will generate one
-                        self.es.index(
+                        self.index(
                             index=self.dbname,
                             doc_type="notifications",
                             consistency = self.consistency,
@@ -476,7 +495,7 @@ class Archiver(object):
                     if doc and 'preferences' in doc['_source'] and doc['_source']['preferences'].get('notifications')
== 'indirect' and not cid in oldrefs:
                         oldrefs.append(cid)
                         # N.B. no index is supplied, so ES will generate one
-                        self.es.index(
+                        self.index(
                             index=self.dbname,
                             consistency = self.consistency,
                             doc_type="notifications",

http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/848bb712/tools/setup.py
----------------------------------------------------------------------
diff --git a/tools/setup.py b/tools/setup.py
index 65957af..1187286 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -27,6 +27,8 @@ if sys.version_info <= (3, 3):
 dopip = False
 try:
     from elasticsearch import Elasticsearch
+    from elasticsearch import VERSION as ES_VERSION
+    ES_MAJOR = ES_VERSION[0]
 except:
     dopip = True
     
@@ -468,7 +470,7 @@ ssl:                    false
 #user:                  username
 #password:              password
 
-#write:                 consistency level (default quorum)
+#%s
 
 #backup:                database name
 
@@ -479,7 +481,8 @@ ssl:                    false
 #cropout:               string to crop from list-id
 
 ###############################################################
-            """ % (hostname, dbname, port))
+            """ % (hostname, dbname, port, 
+                   'wait:                  active shard count' if ES_MAJOR == 5 else 'write:
                consistency level (default quorum)'))
     f.close()
 
 config_path = "../site/api/lib"


Mime
View raw message