Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 87D45200C01 for ; Thu, 5 Jan 2017 00:04:19 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 8673D160B44; Wed, 4 Jan 2017 23:04:19 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A8EFB160B3A for ; Thu, 5 Jan 2017 00:04:18 +0100 (CET) Received: (qmail 10346 invoked by uid 500); 4 Jan 2017 23:04:17 -0000 Mailing-List: contact commits-help@ponymail.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ponymail.incubator.apache.org Delivered-To: mailing list commits@ponymail.incubator.apache.org Received: (qmail 10337 invoked by uid 99); 4 Jan 2017 23:04:17 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 Jan 2017 23:04:17 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id 797ADC002D for ; Wed, 4 Jan 2017 23:04:17 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.219 X-Spam-Level: X-Spam-Status: No, score=-6.219 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id NJQvAI8HSLst for ; Wed, 4 Jan 2017 23:04:15 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id A78B15F1B8 for ; Wed, 4 Jan 2017 23:04:14 +0000 (UTC) Received: (qmail 10253 invoked by uid 99); 4 Jan 2017 23:04:13 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 Jan 2017 23:04:13 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C6812DF9F9; Wed, 4 Jan 2017 23:04:13 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sebb@apache.org To: commits@ponymail.incubator.apache.org Message-Id: <10458c900fa64833a0905aeb05060497@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-ponymail git commit: config.hidePrivate should be dropped Date: Wed, 4 Jan 2017 23:04:13 +0000 (UTC) archived-at: Wed, 04 Jan 2017 23:04:19 -0000 Repository: incubator-ponymail Updated Branches: refs/heads/master 570987088 -> 5066853b4 config.hidePrivate should be dropped This fixes #272 Project: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/commit/5066853b Tree: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/tree/5066853b Diff: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/diff/5066853b Branch: refs/heads/master Commit: 5066853b4507999f845ef183feb7e5dee0d9131e Parents: 5709870 Author: Sebb Authored: Wed Jan 4 23:03:52 2017 +0000 Committer: Sebb Committed: Wed Jan 4 23:03:52 2017 +0000 ---------------------------------------------------------------------- CHANGELOG.md | 1 + site/api/preferences.lua | 132 ++++++++++++++---------------------------- tools/setup.py | 1 - 3 files changed, 43 insertions(+), 91 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/5066853b/CHANGELOG.md ---------------------------------------------------------------------- diff --git a/CHANGELOG.md b/CHANGELOG.md index bf69d13..9352c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ - elastic.scroll does not return nil sid when there are no more results (#315) - stats.lua should return firstMonth and lastMonth (#120) - many python scripts insist on Python 3.4 (#312) +- config.hidePrivate should be dropped (#272) ## CHANGES in 0.9b: http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/5066853b/site/api/preferences.lua ---------------------------------------------------------------------- diff --git a/site/api/preferences.lua b/site/api/preferences.lua index 70b158a..c7988aa 100644 --- a/site/api/preferences.lua +++ b/site/api/preferences.lua @@ -284,111 +284,63 @@ Pony Mail - Email for Ponies and People. if cache then lists = JSON.decode(cache) else - -- get all the list names - local doc = elastic.raw { - size = 0, -- we don't need the hits themselves - aggs = { - from = { - terms = { - field = "list_raw", - size = 500000 - } - } - } - } - -- get list names with messages in the last 90 days - local ndoc = elastic.raw { + -- aggregate the documents by listname, privacy flag, recent docs + + local alldocs = elastic.raw{ size = 0, -- we don't need the hits themselves aggs = { - from = { + listnames = { terms = { field = "list_raw", size = 500000 - } - } - }, - query = { - range = { - date = { gte = "now-90d" } - } - } - } - - -- init the lists with the names (no counts) - for x,y in pairs (doc.aggregations.from.buckets) do - local list, domain = y.key:lower():match("^?$") - if domain and domain:match("^[-_a-z0-9.]+$") and #domain > 3 and list:match("^[-_a-z0-9.]+$") then - lists[domain] = lists[domain] or {} - lists[domain][list] = 0 - end - end - -- add the counts of recent mails - for x,y in pairs (ndoc.aggregations.from.buckets) do - local list, domain = y.key:lower():match("^?$") - if domain and domain:match("^[-_a-z0-9.]+$") and #domain > 3 and list:match("^[-_a-z0-9.]+$") then - lists[domain] = lists[domain] or {} - lists[domain][list] = y.doc_count - end - end - - -- save temporary list in cache - r:ivm_set(PM_LISTS_KEY, JSON.encode(lists)) - - -- hide private lists? - -- this invalidates any cache there is and forces a check for - -- private emails inside lists. If found and the current user - -- does not have access, the list is hidden - end - if config.hidePrivate then - local PM_LISTS_PRIVATE_KEY = "pm_lists_cache_private_" .. r.hostname .. "-" .. NOWISH - local cache = r:ivm_get(PM_LISTS_PRIVATE_KEY) - local pdoc - if cache then - pdoc = JSON.decode(cache) - else - -- get list names which have at least one private message in the last 20 years - pdoc = elastic.raw { - size = 0, -- we don't need the hits themselves - aggs = { - from = { - terms = { - field = "list_raw", - size = 500000 - } - } - }, - query = { - bool = { - must = { - { - range = { - date = { gte = "now-20y" } - }, + }, + aggs = { + -- split list into public and private buckets + privacy = { + terms = { + field = "private" }, - { - term = { - private = true + aggs = { + -- Create a single bucket of recent mails + recent = { + range = { + field = "date", + ranges = { {from = "now-90d"} } + } } } } } } } - r:ivm_set(PM_LISTS_PRIVATE_KEY, JSON.encode(pdoc)) - end - - -- remove any lists containing mails that the user is not allowed to access - -- N.B. this removes mixed lists - -- i.e. the user won't see the list name if it contains a single private mail they cannot access - for x,y in pairs (pdoc.aggregations.from.buckets) do - local _, list, domain = aaa.parseLid(y.key:lower()) - if list and domain and #list > 0 and #domain > 2 then - if not aaa.canAccessList(r, y.key:lower(), account) then - lists[domain] = lists[domain] or {} - lists[domain][list] = nil + } + -- Now process the docs that are visible to the user + for _, entry in pairs (alldocs.aggregations.listnames.buckets) do + local listname = entry.key:lower() + local _, list, domain = aaa.parseLid(listname) + -- TODO is it necessary to check the lengths? + if list and domain and #list > 0 and #domain > 3 then + -- check public and private (only one may be present) + for _, privacy in pairs(entry.privacy.buckets) do + local isPublic = privacy.key_as_string == 'false' + -- do the user have access? + if isPublic or aaa.canAccessList(r, listname, account) then + -- there is only a single recent bucket; access it directly + local recent_count = privacy.recent.buckets[1].doc_count + -- create the domain entry if necessary + lists[domain] = lists[domain] or {} + -- check if we have a list entry yet + if lists[domain][list] then + lists[domain][list] = lists[domain][list] + recent_count + else + lists[domain][list] = recent_count -- init the entry + end + end end end end + -- save temporary list in cache + r:ivm_set(PM_LISTS_KEY, JSON.encode(lists)) end -- do we need to remove junk? http://git-wip-us.apache.org/repos/asf/incubator-ponymail/blob/5066853b/tools/setup.py ---------------------------------------------------------------------- diff --git a/tools/setup.py b/tools/setup.py index aa34c2b..a4d668b 100755 --- a/tools/setup.py +++ b/tools/setup.py @@ -485,7 +485,6 @@ local config = { }, antispam = true -- Whether or not to add anti-spam measures aimed at anonymous users. -- no_association = {} -- domains that are not allowed for email association - -- hidePrivate = true -- whether to hide names of lists that contain any private mails -- listsDisplay = 'regex' -- if defined, hide list names that don't match the regex } return config