accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mjw...@apache.org
Subject [3/4] accumulo git commit: ACCUMULO-4657 Update MetadataTableUtil.java
Date Tue, 20 Jun 2017 20:42:42 GMT
ACCUMULO-4657 Update MetadataTableUtil.java

Outputting every "loaded" entry in the table is excessive, especially for tables with multiple
simultaneous bulk imports and multiple references to the same file.  This has been seen to
cause performance problems.  Even when the log level was reduced, there was blocking within
log4j.  By doing that check once outside the loop and only logging at trace level, we have
seen bulk import performance improvements.

Updated String to byte array for faster comparison


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/6c31cd49
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/6c31cd49
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/6c31cd49

Branch: refs/heads/1.8
Commit: 6c31cd49ce0b8eae13d7a6cca50bc802745a6e72
Parents: d5ddca1
Author: matthpeterson <mpeterson@appliedtech-group.com>
Authored: Fri Jun 16 15:51:38 2017 -0400
Committer: Michael Wall <mjwall@apache.org>
Committed: Tue Jun 20 16:17:38 2017 -0400

----------------------------------------------------------------------
 .../accumulo/server/util/MetadataTableUtil.java   | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/6c31cd49/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index c07114b..54ca37c 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@ -24,6 +24,7 @@ import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -894,12 +895,19 @@ public class MetadataTableUtil {
         BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()))
{
       mscanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange());
       mscanner.fetchColumnFamily(TabletsSection.BulkFileColumnFamily.NAME);
+      boolean shouldTrace = log.isTraceEnabled();
+      byte[] tidAsBytes = Long.toString(tid).getBytes(UTF_8);
       for (Entry<Key,Value> entry : mscanner) {
-        log.debug("Looking at entry " + entry + " with tid " + tid);
-        if (Long.parseLong(entry.getValue().toString()) == tid) {
-          log.debug("deleting entry " + entry);
-          Mutation m = new Mutation(entry.getKey().getRow());
-          m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+        if (shouldTrace) {
+          log.trace("Looking at entry {} with tid {}", entry, tid);
+        }
+        if (Arrays.equals(entry.getValue().get(), tidAsBytes)) {
+          if (shouldTrace) {
+            log.trace("deleting entry {}", entry);
+          }
+          Key key = entry.getKey();
+          Mutation m = new Mutation(key.getRow());
+          m.putDelete(key.getColumnFamily(), key.getColumnQualifier());
           bw.addMutation(m);
         }
       }


Mime
View raw message