accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] branch 1.9 updated: fix #1326 Copy instead of clone metadata table in IT
Date Thu, 15 Aug 2019 17:31:56 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.9 by this push:
     new 556c29e  fix #1326 Copy instead of clone metadata table in IT
556c29e is described below

commit 556c29ee60c80b1fa8464fa85f24be44d8e0608d
Author: Keith Turner <kturner@apache.org>
AuthorDate: Thu Aug 15 13:25:12 2019 -0400

    fix #1326 Copy instead of clone metadata table in IT
    
    After the changes in #1309 cloning of metdata table is no longer
    allowed. TabletStateChangeIteratorIT relied on cloning and
    was changed to copy instead.
    
    Also the test was very sensitive to concurrent chnages in the metadata
    table.  Suspect that cloning used to introduce a delay that hid this.
    The change from cloning to copying caused the test to fail often
    because of these timing issues.  To avoid this, the test was refactored
    to tolerate concurrent changes to the metadata table.
---
 .../functional/TabletStateChangeIteratorIT.java    | 71 ++++++++++++++++------
 1 file changed, 52 insertions(+), 19 deletions(-)

diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 6a30292..2fd8589 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.SortedSet;
@@ -34,7 +35,9 @@ import org.apache.accumulo.core.client.BatchDeleter;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IsolatedScanner;
 import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.RowIterator;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
@@ -78,11 +81,13 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness
{
   @Test
   public void test() throws AccumuloException, AccumuloSecurityException, TableExistsException,
       TableNotFoundException {
-    String[] tables = getUniqueNames(4);
+    String[] tables = getUniqueNames(6);
     final String t1 = tables[0];
     final String t2 = tables[1];
     final String t3 = tables[2];
-    final String cloned = tables[3];
+    final String metaCopy1 = tables[3];
+    final String metaCopy2 = tables[4];
+    final String metaCopy3 = tables[5];
 
     // create some metadata
     createTable(t1, true);
@@ -90,25 +95,30 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness
{
     createTable(t3, true);
 
     // examine a clone of the metadata table, so we can manipulate it
-    cloneMetadataTable(cloned);
+    copyTable(MetadataTable.NAME, metaCopy1);
 
     State state = new State();
-    while (findTabletsNeedingAttention(cloned, state) > 0) {
+    while (findTabletsNeedingAttention(metaCopy1, state) > 0) {
       UtilWaitThread.sleep(500);
+      copyTable(MetadataTable.NAME, metaCopy1);
     }
-    assertEquals("No tables should need attention", 0, findTabletsNeedingAttention(cloned,
state));
+    assertEquals("No tables should need attention", 0,
+        findTabletsNeedingAttention(metaCopy1, state));
+
+    // The metadata table stabilized and metaCopy1 contains a copy suitable for testing.
Before
+    // metaCopy1 is modified, copy it for subsequent test.
+    copyTable(metaCopy1, metaCopy2);
+    copyTable(metaCopy1, metaCopy3);
 
     // test the assigned case (no location)
-    removeLocation(cloned, t3);
+    removeLocation(metaCopy1, t3);
     assertEquals("Should have two tablets without a loc", 2,
-        findTabletsNeedingAttention(cloned, state));
+        findTabletsNeedingAttention(metaCopy1, state));
 
     // test the cases where the assignment is to a dead tserver
-    getConnector().tableOperations().delete(cloned);
-    cloneMetadataTable(cloned);
-    reassignLocation(cloned, t3);
+    reassignLocation(metaCopy2, t3);
     assertEquals("Should have one tablet that needs to be unassigned", 1,
-        findTabletsNeedingAttention(cloned, state));
+        findTabletsNeedingAttention(metaCopy2, state));
 
     // test the cases where there is ongoing merges
     state = new State() {
@@ -120,17 +130,16 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness
{
       }
     };
     assertEquals("Should have 2 tablets that need to be chopped or unassigned", 1,
-        findTabletsNeedingAttention(cloned, state));
+        findTabletsNeedingAttention(metaCopy2, state));
 
     // test the bad tablet location state case (inconsistent metadata)
     state = new State();
-    cloneMetadataTable(cloned);
-    addDuplicateLocation(cloned, t3);
+    addDuplicateLocation(metaCopy3, t3);
     assertEquals("Should have 1 tablet that needs a metadata repair", 1,
-        findTabletsNeedingAttention(cloned, state));
+        findTabletsNeedingAttention(metaCopy3, state));
 
     // clean up
-    dropTables(t1, t2, t3, cloned);
+    dropTables(t1, t2, t3, metaCopy1, metaCopy2, metaCopy3);
   }
 
   private void addDuplicateLocation(String table, String tableNameToModify)
@@ -183,6 +192,7 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness
{
       if (e != null)
         results++;
     }
+
     return results;
   }
 
@@ -199,14 +209,37 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness
{
     }
   }
 
-  private void cloneMetadataTable(String cloned) throws AccumuloException,
+  private void copyTable(String source, String copy) throws AccumuloException,
       AccumuloSecurityException, TableNotFoundException, TableExistsException {
     try {
-      dropTables(cloned);
+      dropTables(copy);
     } catch (TableNotFoundException ex) {
       // ignored
     }
-    getConnector().tableOperations().clone(MetadataTable.NAME, cloned, true, null, null);
+
+    getConnector().tableOperations().create(copy);
+
+    try (Scanner scanner = getConnector().createScanner(source, Authorizations.EMPTY);
+        BatchWriter writer = getConnector().createBatchWriter(copy, new BatchWriterConfig()))
{
+      RowIterator rows = new RowIterator(new IsolatedScanner(scanner));
+
+      while (rows.hasNext()) {
+        Iterator<Entry<Key,Value>> row = rows.next();
+        Mutation m = null;
+
+        while (row.hasNext()) {
+          Entry<Key,Value> entry = row.next();
+          Key k = entry.getKey();
+          if (m == null)
+            m = new Mutation(k.getRow());
+
+          m.put(k.getColumnFamily(), k.getColumnQualifier(), k.getColumnVisibilityParsed(),
+              k.getTimestamp(), entry.getValue());
+        }
+
+        writer.addMutation(m);
+      }
+    }
   }
 
   private void dropTables(String... tables)


Mime
View raw message