accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch master updated: #469 Fix bug with GC not scanning root table for file refs (#503)
Date Wed, 30 May 2018 12:40:01 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/master by this push:
     new 780c4ca  #469 Fix bug with GC not scanning root table for file refs (#503)
780c4ca is described below

commit 780c4cab06597547d028ed59abdaa967a1c7900a
Author: Keith Turner <keith@deenlo.com>
AuthorDate: Wed May 30 08:39:42 2018 -0400

    #469 Fix bug with GC not scanning root table for file refs (#503)
    
    This bug was introduced in 9feb5e1. In addition to fixing the bug,
    specifying the table was made mandatory for the metadata scanner
    builder.
    
    The bug was found by MetadataMaxFilesIT
---
 .../core/metadata/schema/MetadataScanner.java        | 20 ++++++++++++++++----
 .../org/apache/accumulo/core/summary/Gatherer.java   |  7 ++++---
 .../apache/accumulo/gc/SimpleGarbageCollector.java   |  4 ++--
 .../accumulo/master/tableOps/bulkVer2/LoadFiles.java |  2 +-
 .../master/tableOps/bulkVer2/PrepBulkImport.java     |  6 +++---
 5 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
index 6fc05bb..573d3c4 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataScanner.java
@@ -41,6 +41,7 @@ import org.apache.accumulo.core.client.impl.Table.ID;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.impl.KeyExtent;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.BulkFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ClonedColumnFamily;
@@ -62,10 +63,11 @@ public class MetadataScanner implements Iterable<TabletMetadata>,
AutoCloseable
     TableOptions from(Connector conn);
   }
 
-  public interface TableOptions extends RangeOptions {
-    /**
-     * Optionally set a table name, defaults to {@value MetadataTable#NAME}
-     */
+  public interface TableOptions {
+    RangeOptions scanRootTable();
+
+    RangeOptions scanMetadataTable();
+
     RangeOptions scanTable(String tableName);
   }
 
@@ -310,6 +312,16 @@ public class MetadataScanner implements Iterable<TabletMetadata>,
AutoCloseable
       this.table = tableName;
       return this;
     }
+
+    @Override
+    public RangeOptions scanMetadataTable() {
+      return scanTable(MetadataTable.NAME);
+    }
+
+    @Override
+    public RangeOptions scanRootTable() {
+      return scanTable(RootTable.NAME);
+    }
   }
 
   private Scanner scanner;
diff --git a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
index 0799510..eaae24d 100644
--- a/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
+++ b/core/src/main/java/org/apache/accumulo/core/summary/Gatherer.java
@@ -160,7 +160,7 @@ public class Gatherer {
       Predicate<String> fileSelector)
       throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 
-    Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx)
+    Iterable<TabletMetadata> tmi = MetadataScanner.builder().from(ctx).scanMetadataTable()
         .overRange(tableId, startRow, endRow).fetchFiles().fetchLocation().fetchLast().fetchPrev()
         .build();
 
@@ -522,8 +522,9 @@ public class Gatherer {
   private int countFiles()
       throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
     // TODO use a batch scanner + iterator to parallelize counting files
-    return MetadataScanner.builder().from(ctx).overRange(tableId, startRow, endRow).fetchFiles()
-        .fetchPrev().build().stream().mapToInt(tm -> tm.getFiles().size()).sum();
+    return MetadataScanner.builder().from(ctx).scanMetadataTable()
+        .overRange(tableId, startRow, endRow).fetchFiles().fetchPrev().build().stream()
+        .mapToInt(tm -> tm.getFiles().size()).sum();
   }
 
   private class GatherRequest implements Supplier<SummaryCollection> {
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 3c60861..6049b42 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -300,8 +300,8 @@ public class SimpleGarbageCollector extends AccumuloServerContext implements
Ifa
         throws TableNotFoundException, AccumuloException, AccumuloSecurityException {
 
       Stream<TabletMetadata> tabletStream = MetadataScanner.builder().from(getConnector())
-          .overTabletRange().checkConsistency().fetchDir().fetchFiles().fetchScans().build()
-          .stream();
+          .scanTable(tableName).overTabletRange().checkConsistency().fetchDir().fetchFiles()
+          .fetchScans().build().stream();
 
       Stream<Reference> refStream = tabletStream.flatMap(tm -> {
         Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream())
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
index 1baac99..390a3ea 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/LoadFiles.java
@@ -108,7 +108,7 @@ class LoadFiles extends MasterRepo {
     Text startRow = loadMapEntry.getKey().getPrevEndRow();
 
     long timeInMillis = master.getConfiguration().getTimeInMillis(Property.MASTER_BULK_TIMEOUT);
-    Iterator<TabletMetadata> tabletIter = MetadataScanner.builder().from(master)
+    Iterator<TabletMetadata> tabletIter = MetadataScanner.builder().from(master).scanMetadataTable()
         .overRange(tableId, startRow, null).checkConsistency().fetchPrev().fetchLocation()
         .fetchLoaded().build().iterator();
 
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
index 18ddab9..bca1389 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java
@@ -166,9 +166,9 @@ public class PrepBulkImport extends MasterRepo {
       Iterators.transform(lmi, entry -> entry.getKey());
 
       TabletIterFactory tabletIterFactory = startRow -> {
-        return MetadataScanner.builder().from(master).overRange(bulkInfo.tableId, startRow,
null)
-            .checkConsistency().fetchPrev().build().stream().map(TabletMetadata::getExtent)
-            .iterator();
+        return MetadataScanner.builder().from(master).scanMetadataTable()
+            .overRange(bulkInfo.tableId, startRow, null).checkConsistency().fetchPrev().build()
+            .stream().map(TabletMetadata::getExtent).iterator();
       };
 
       checkForMerge(bulkInfo.tableId.canonicalID(),

-- 
To stop receiving notification emails like this one, please contact
kturner@apache.org.

Mime
View raw message