Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 240A9200CBC for ; Tue, 20 Jun 2017 22:42:42 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 22253160BE1; Tue, 20 Jun 2017 20:42:42 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 4EB69160BF2 for ; Tue, 20 Jun 2017 22:42:41 +0200 (CEST) Received: (qmail 12024 invoked by uid 500); 20 Jun 2017 20:42:40 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 12008 invoked by uid 99); 20 Jun 2017 20:42:40 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 20 Jun 2017 20:42:40 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 4E3B6DFFAB; Tue, 20 Jun 2017 20:42:40 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mjwall@apache.org To: commits@accumulo.apache.org Date: Tue, 20 Jun 2017 20:42:42 -0000 Message-Id: <2336559370ee403b99f65dd0f68313e8@git.apache.org> In-Reply-To: <1ed10d1c4ad74356a13d65c2f3bf6984@git.apache.org> References: <1ed10d1c4ad74356a13d65c2f3bf6984@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/4] accumulo git commit: ACCUMULO-4657 Update MetadataTableUtil.java archived-at: Tue, 20 Jun 2017 20:42:42 -0000 ACCUMULO-4657 Update MetadataTableUtil.java Outputting every "loaded" entry in the table is excessive, especially for tables with multiple simultaneous bulk imports and multiple references to the same file. This has been seen to cause performance problems. Even when the log level was reduced, there was blocking within log4j. By doing that check once outside the loop and only logging at trace level, we have seen bulk import performance improvements. Updated String to byte array for faster comparison Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c31cd49 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c31cd49 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c31cd49 Branch: refs/heads/1.8 Commit: 6c31cd49ce0b8eae13d7a6cca50bc802745a6e72 Parents: d5ddca1 Author: matthpeterson Authored: Fri Jun 16 15:51:38 2017 -0400 Committer: Michael Wall Committed: Tue Jun 20 16:17:38 2017 -0400 ---------------------------------------------------------------------- .../accumulo/server/util/MetadataTableUtil.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c31cd49/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java index c07114b..54ca37c 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java @@ -24,6 +24,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -894,12 +895,19 @@ public class MetadataTableUtil { BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig())) { mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange()); mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME); + boolean shouldTrace = log.isTraceEnabled(); + byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8); for (Entry entry : mscanner) { - log.debug("Looking at entry " + entry + " with tid " + tid); - if (Long.parseLong(entry.getValue().toString()) == tid) { - log.debug("deleting entry " + entry); - Mutation m = new Mutation(entry.getKey().getRow()); - m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier()); + if (shouldTrace) { + log.trace("Looking at entry {} with tid {}", entry, tid); + } + if (Arrays.equals(entry.getValue().get(), tidAsBytes)) { + if (shouldTrace) { + log.trace("deleting entry {}", entry); + } + Key key = entry.getKey(); + Mutation m = new Mutation(key.getRow()); + m.putDelete(key.getColumnFamily(), key.getColumnQualifier()); bw.addMutation(m); } }