ponymail-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s...@apache.org
Subject incubator-ponymail git commit: Archiver loses some message information
Date Thu, 21 Sep 2017 02:19:24 GMT
Repository: incubator-ponymail
Updated Branches:
  refs/heads/master e6daa748a -> d23d649ae


Archiver loses some message information

This fixes #402

Mail parsing can change source spacing
This fixes #410

Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/d23d649a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/d23d649a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/d23d649a

Branch: refs/heads/master
Commit: d23d649ae46ecb17d5331571c09d6fa58a9b3859
Parents: e6daa74
Author: Sebb <sebb@apache.org>
Authored: Thu Sep 21 03:19:19 2017 +0100
Committer: Sebb <sebb@apache.org>
Committed: Thu Sep 21 03:19:19 2017 +0100

----------------------------------------------------------------------
 CHANGELOG.md         |  2 ++
 tools/archiver.py    | 11 ++++-------
 tools/import-mbox.py |  8 ++++++--
 3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/d23d649a/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0ecf255..cda0c9c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,6 @@
 ## CHANGES in 0.10:
+- Archiver loses some message information (#402)
+- Mail parsing can change source spacing (#410)
 - Bug: use actual envelope header if present (#412)
 - Bug: stats.lua should use UTC for calculating year and month (#409)
 - Bug: atom.lua - should generate date in UTC, not local time (#408)

http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/d23d649a/tools/archiver.py
----------------------------------------------------------------------
diff --git a/tools/archiver.py b/tools/archiver.py
index 3785c86..78db074 100755
--- a/tools/archiver.py
+++ b/tools/archiver.py
@@ -359,7 +359,7 @@ class Archiver(object):
 
         return  ojson, contents
             
-    def archive_message(self, mlist, msg):
+    def archive_message(self, mlist, msg, raw_msg):
         """Send the message to the archiver.
 
         :param mlist: The IMailingList object.
@@ -416,7 +416,7 @@ class Archiver(object):
             consistency = self.consistency,
             body = {
                 "message-id": msg_metadata['message-id'],
-                "source": self.mbox_source(msg)
+                "source": self.mbox_source(raw_msg)
             }
         )
         
@@ -509,11 +509,8 @@ class Archiver(object):
                             logger.info("Notification sent to %s for %s" % (cid, mid))
         return lid, ojson['mid']
 
-    def mbox_source(self, msg):
+    def mbox_source(self, b):
         # Common method shared with import-mbox
-        policy = msg.policy.clone(max_line_length=0) # don't wrap headers
-        hasFrom=(msg.get_unixfrom() != None) # only use the envelope if it exists
-        b = msg.as_bytes(unixfrom=hasFrom, policy=policy)
         try:
             # Can we store as ASCII?
             return b.decode('ascii', errors='strict')
@@ -643,7 +640,7 @@ if __name__ == '__main__':
             msg_metadata = namedtuple('importmsg', ['list_id', 'archive_public'])(list_id
= msg.get('list-id'), archive_public=ispublic)
             
             try:
-                lid, mid = archie.archive_message(msg_metadata, msg)
+                lid, mid = archie.archive_message(msg_metadata, msg, msgstring)
                 print("%s: Done archiving to %s as %s!" % (email.utils.formatdate(), lid,
mid))
             except Exception as err:
                 if args.verbose:

http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/d23d649a/tools/import-mbox.py
----------------------------------------------------------------------
diff --git a/tools/import-mbox.py b/tools/import-mbox.py
index a3b49a0..9eb5bd4 100755
--- a/tools/import-mbox.py
+++ b/tools/import-mbox.py
@@ -239,7 +239,8 @@ class SlurpThread(Thread):
             LEY = EY
             
             
-            for message in messages:
+            for key in messages.iterkeys():
+                message=messages.get(key)
                 # If --filter is set, discard any messages not matching by continuing to
next email
                 if fromFilter and 'from' in message and message['from'].find(fromFilter)
== -1:
                     continue
@@ -306,6 +307,9 @@ class SlurpThread(Thread):
                         continue
 
                 if json:
+                    file=messages.get_file(key, True)
+                    raw_msg=file.read()
+                    file.close()
                     if args.dups:
                         try:
                             duplicates[json['mid']].append(json['message-id'] + " in " +
filename)
@@ -317,7 +321,7 @@ class SlurpThread(Thread):
                         json_source = {
                             'mid': json['mid'], # needed for bulk-insert only, not needed
in database
                             'message-id': json['message-id'],
-                            'source': archie.mbox_source(message)
+                            'source': archie.mbox_source(raw_msg)
                         }
                     except Exception as e:
                         self.printid("Error '%s' processing id %s msg %s " % (e, json['mid'],
json['message-id']))


Mime
View raw message