accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From md...@apache.org
Subject [01/11] git commit: ACCUMULO-1978 fixed compaction and flush of metadata table and added test
Date Thu, 12 Dec 2013 19:08:48 GMT
Updated Branches:
  refs/heads/1.4.5-SNAPSHOT 0d0bc4643 -> 3b08c717e
  refs/heads/1.5.1-SNAPSHOT 7eb838e3c -> dc70a30d6
  refs/heads/1.6.0-SNAPSHOT 02275812b -> a425f6a50
  refs/heads/master 403815f55 -> bdac06f65


ACCUMULO-1978 fixed compaction and flush of metadata table and added test


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/02275812
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/02275812
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/02275812

Branch: refs/heads/master
Commit: 02275812b6ed7cec6c4923183f2f75011f4c4f4a
Parents: 07cdc57
Author: Keith Turner <kturner@apache.org>
Authored: Thu Dec 12 13:08:34 2013 -0500
Committer: Keith Turner <kturner@apache.org>
Committed: Thu Dec 12 13:08:34 2013 -0500

----------------------------------------------------------------------
 .../java/org/apache/accumulo/master/Master.java | 11 ++++--
 .../accumulo/master/tableOps/CompactRange.java  | 15 +++++---
 .../accumulo/test/functional/MetadataIT.java    | 38 ++++++++++++++++++++
 3 files changed, 57 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/02275812/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index eb1a492..bd6ee27 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -74,6 +74,7 @@ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
 import org.apache.accumulo.core.master.thrift.TabletSplit;
 import org.apache.accumulo.core.metadata.MetadataTable;
 import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 import org.apache.accumulo.core.security.Authorizations;
@@ -708,12 +709,18 @@ public class Master implements LiveTServerSet.Listener, TableObserver,
CurrentSt
 
         try {
           Connector conn = getConnector();
-          Scanner scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+          Scanner scanner;
+          if (tableId.equals(MetadataTable.ID)) {
+            scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
+            scanner.setRange(MetadataSchema.TabletsSection.getRange());
+          } else {
+            scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+            scanner.setRange(new KeyExtent(new Text(tableId), null, ByteBufferUtil.toText(startRow)).toMetadataRange());
+          }
           TabletsSection.ServerColumnFamily.FLUSH_COLUMN.fetch(scanner);
           TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
           scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
           scanner.fetchColumnFamily(LogColumnFamily.NAME);
-          scanner.setRange(new KeyExtent(new Text(tableId), null, ByteBufferUtil.toText(startRow)).toMetadataRange());
 
           RowIterator ri = new RowIterator(scanner);
 

http://git-wip-us.apache.org/repos/asf/accumulo/blob/02275812/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
index e8f65d2..df474b3 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/CompactRange.java
@@ -43,6 +43,7 @@ import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.master.state.tables.TableState;
 import org.apache.accumulo.core.metadata.MetadataTable;
 import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.MapCounter;
@@ -97,14 +98,18 @@ class CompactionDriver extends MasterRepo {
 
     MapCounter<TServerInstance> serversToFlush = new MapCounter<TServerInstance>();
     Connector conn = master.getConnector();
-    Scanner scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
 
-    Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new Text(startRow)).toMetadataRange();
+    Scanner scanner;
 
-    if (tableId.equals(MetadataTable.ID))
-      range = range.clip(new Range(RootTable.EXTENT.getMetadataEntry(), false, null, true));
+    if (tableId.equals(MetadataTable.ID)) {
+      scanner = new IsolatedScanner(conn.createScanner(RootTable.NAME, Authorizations.EMPTY));
+      scanner.setRange(MetadataSchema.TabletsSection.getRange());
+    } else {
+      scanner = new IsolatedScanner(conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY));
+      Range range = new KeyExtent(new Text(tableId), null, startRow == null ? null : new
Text(startRow)).toMetadataRange();
+      scanner.setRange(range);
+    }
 
-    scanner.setRange(range);
     TabletsSection.ServerColumnFamily.COMPACT_COLUMN.fetch(scanner);
     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner);
     scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/02275812/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java
index 92e0d44..6c9f27a 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MetadataIT.java
@@ -20,7 +20,9 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
@@ -36,11 +38,47 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.UtilWaitThread;
 import org.apache.hadoop.io.Text;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class MetadataIT extends SimpleMacIT {
 
   @Test(timeout = 60 * 1000)
+  public void testFlushAndCompact() throws Exception {
+    Connector c = getConnector();
+
+    // create a table to write some data to metadata table
+    c.tableOperations().create(super.getTableNames(1)[0]);
+
+    Scanner rootScanner = c.createScanner(RootTable.NAME, Authorizations.EMPTY);
+    rootScanner.setRange(MetadataSchema.TabletsSection.getRange());
+    rootScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+    Set<String> files1 = new HashSet<String>();
+    for (Entry<Key,Value> entry : rootScanner)
+      files1.add(entry.getKey().getColumnQualifier().toString());
+
+    c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+
+    Set<String> files2 = new HashSet<String>();
+    for (Entry<Key,Value> entry : rootScanner)
+      files2.add(entry.getKey().getColumnQualifier().toString());
+
+    // flush of metadata table should change file set in root table
+    Assert.assertTrue(files2.size() > 0);
+    Assert.assertNotEquals(files1, files2);
+
+    c.tableOperations().compact(MetadataTable.NAME, null, null, false, true);
+
+    Set<String> files3 = new HashSet<String>();
+    for (Entry<Key,Value> entry : rootScanner)
+
+      files3.add(entry.getKey().getColumnQualifier().toString());
+    // compaction of metadata table should change file set in root table
+    Assert.assertNotEquals(files2, files3);
+  }
+
+  @Test(timeout = 60 * 1000)
   public void mergeMeta() throws Exception {
     Connector c = getConnector();
     SortedSet<Text> splits = new TreeSet<Text>();


Mime
View raw message