accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo] 01/01: Merge branch '1.9'
Date Tue, 08 May 2018 16:15:43 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit 49192c9fe45de9efdac88afd0c3c96eb82822721
Merge: 2d869d3 b8f574f
Author: Keith Turner <kturner@apache.org>
AuthorDate: Tue May 8 12:13:44 2018 -0400

    Merge branch '1.9'

 .../org/apache/accumulo/tserver/TabletServer.java  |  10 +-
 .../accumulo/tserver/log/CloseableIterator.java    |  25 ++
 .../org/apache/accumulo/tserver/log/DfsLogger.java |   8 +-
 .../{MultiReader.java => RecoveryLogReader.java}   | 162 ++++++++-
 .../accumulo/tserver/log/RecoveryLogsIterator.java | 100 ++++++
 .../accumulo/tserver/log/SortedLogRecovery.java    | 378 +++++++++++----------
 .../apache/accumulo/tserver/logger/LogFileKey.java |  46 +--
 .../apache/accumulo/tserver/logger/LogReader.java  |  16 +-
 .../tserver/replication/AccumuloReplicaSystem.java |   6 +-
 .../org/apache/accumulo/tserver/tablet/Tablet.java |   2 +-
 .../apache/accumulo/tserver/log/LogEventsTest.java |  36 ++
 .../accumulo/tserver/log/LogFileKeyTest.java       |  98 ++++++
 ...ReaderTest.java => RecoveryLogsReaderTest.java} |  43 ++-
 .../tserver/log/SortedLogRecoveryTest.java         | 296 +++++++++++++++-
 .../accumulo/tserver/logger/LogFileTest.java       |  14 +-
 .../replication/AccumuloReplicaSystemTest.java     |  33 +-
 .../BatchWriterReplicationReplayerTest.java        |   4 +-
 .../UnusedWalDoesntCloseReplicationStatusIT.java   |   2 +-
 18 files changed, 991 insertions(+), 288 deletions(-)

diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 914544a,bdf5210..708f843
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@@ -35,7 -35,6 +35,8 @@@ import java.util.Objects
  import java.util.Set;
  import java.util.TreeMap;
  
 +import org.apache.accumulo.core.client.impl.Table;
++import org.apache.accumulo.core.client.impl.Table.ID;
  import org.apache.accumulo.core.data.Mutation;
  import org.apache.accumulo.core.data.Value;
  import org.apache.accumulo.core.data.impl.KeyExtent;
@@@ -675,6 -679,85 +681,85 @@@ public class SortedLogRecoveryTest 
      Assert.assertEquals(m, mutations.get(0));
    }
  
+   @Test
+   public void testLeaveAndComeBack() throws IOException {
+     /**
+      * This test recreates the situation in bug #449 (Github issues).
+      */
+     Mutation m1 = new ServerMutation(new Text("r1"));
+     m1.put("f1", "q1", "v1");
+ 
+     Mutation m2 = new ServerMutation(new Text("r2"));
+     m2.put("f1", "q1", "v2");
+ 
+     KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+         createKeyValue(DEFINE_TABLET, 100, 10, extent), createKeyValue(MUTATION, 100, 10,
m1),
+         createKeyValue(COMPACTION_START, 101, 10, "/t/f1"),
+         createKeyValue(COMPACTION_FINISH, 102, 10, null)};
+ 
+     KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+         createKeyValue(DEFINE_TABLET, 1, 20, extent), createKeyValue(MUTATION, 1, 20, m2)};
+ 
+     Arrays.sort(entries1);
+     Arrays.sort(entries2);
+     Map<String,KeyValue[]> logs = new TreeMap<>();
+     logs.put("entries1", entries1);
+     logs.put("entries2", entries2);
+ 
+     List<Mutation> mutations = recover(logs, extent);
+ 
+     Assert.assertEquals(1, mutations.size());
+     Assert.assertEquals(m2, mutations.get(0));
+   }
+ 
+   @Test
+   public void testMultipleTablets() throws IOException {
 -    KeyExtent e1 = new KeyExtent("1", new Text("m"), null);
 -    KeyExtent e2 = new KeyExtent("1", null, new Text("m"));
++    KeyExtent e1 = new KeyExtent(ID.of("1"), new Text("m"), null);
++    KeyExtent e2 = new KeyExtent(ID.of("1"), null, new Text("m"));
+ 
+     Mutation m1 = new ServerMutation(new Text("b"));
+     m1.put("f1", "q1", "v1");
+ 
+     Mutation m2 = new ServerMutation(new Text("b"));
+     m2.put("f1", "q2", "v2");
+ 
+     Mutation m3 = new ServerMutation(new Text("s"));
+     m3.put("f1", "q1", "v3");
+ 
+     Mutation m4 = new ServerMutation(new Text("s"));
+     m4.put("f1", "q2", "v4");
+ 
+     KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+         createKeyValue(DEFINE_TABLET, 7, 10, e1), createKeyValue(DEFINE_TABLET, 5, 11, e2),
+         createKeyValue(MUTATION, 8, 10, m1), createKeyValue(COMPACTION_START, 9, 10, "/t/f1"),
+         createKeyValue(MUTATION, 10, 10, m2), createKeyValue(COMPACTION_FINISH, 11, 10,
null),
+         createKeyValue(MUTATION, 6, 11, m3), createKeyValue(COMPACTION_START, 7, 11, "/t/f2"),
+         createKeyValue(MUTATION, 8, 11, m4)};
+ 
+     Arrays.sort(entries1);
+ 
+     Map<String,KeyValue[]> logs = new TreeMap<>();
+     logs.put("entries1", entries1);
+ 
+     List<Mutation> mutations1 = recover(logs, e1);
+     Assert.assertEquals(1, mutations1.size());
+     Assert.assertEquals(m2, mutations1.get(0));
+ 
+     List<Mutation> mutations2 = recover(logs, e2);
+     Assert.assertEquals(2, mutations2.size());
+     Assert.assertEquals(m3, mutations2.get(0));
+     Assert.assertEquals(m4, mutations2.get(1));
+ 
+     KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+         createKeyValue(DEFINE_TABLET, 9, 11, e2), createKeyValue(COMPACTION_FINISH, 10,
11, null)};
+     Arrays.sort(entries2);
+     logs.put("entries2", entries2);
+ 
+     mutations2 = recover(logs, e2);
+     Assert.assertEquals(1, mutations2.size());
+     Assert.assertEquals(m4, mutations2.get(0));
+   }
+ 
    private void runPathTest(boolean startMatches, String compactionStartFile, String... tabletFiles)
        throws IOException {
      Mutation m1 = new ServerMutation(new Text("row1"));
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java
index 2e8d220,5e26a9d..22c83cc
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java
@@@ -80,9 -79,9 +80,9 @@@ public class LogFileTest 
      readWrite(COMPACTION_START, 3, 4, "some file", null, null, key, value);
      assertEquals(key.event, COMPACTION_START);
      assertEquals(key.seq, 3);
-     assertEquals(key.tid, 4);
+     assertEquals(key.tabletId, 4);
      assertEquals(key.filename, "some file");
 -    KeyExtent tablet = new KeyExtent("table", new Text("bbbb"), new Text("aaaa"));
 +    KeyExtent tablet = new KeyExtent(Table.ID.of("table"), new Text("bbbb"), new Text("aaaa"));
      readWrite(DEFINE_TABLET, 5, 6, null, tablet, null, key, value);
      assertEquals(key.event, DEFINE_TABLET);
      assertEquals(key.seq, 5);
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java
index b270428,107060b..25cf68b
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java
@@@ -78,9 -78,9 +78,8 @@@ public class AccumuloReplicaSystemTest 
       * are solely for testing that each LogEvents is handled, order is not important.
       */
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("1", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
-     key.tid = 1;
- 
+     key.tabletId = 1;
 -
      key.write(dos);
      value.write(dos);
  
@@@ -93,8 -93,8 +92,8 @@@
      value.write(dos);
  
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("2", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
-     key.tid = 2;
+     key.tabletId = 2;
      value.mutations = Collections.emptyList();
  
      key.write(dos);
@@@ -124,8 -124,8 +123,8 @@@
      value.write(dos);
  
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("1", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
-     key.tid = 3;
+     key.tabletId = 3;
      value.mutations = Collections.emptyList();
  
      key.write(dos);
@@@ -140,9 -140,9 +139,9 @@@
  
      key.tablet = null;
      key.event = LogEvents.MUTATION;
-     key.tid = 3;
+     key.tabletId = 3;
      key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
 -    value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
 +    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
  
      key.write(dos);
      value.write(dos);
@@@ -186,8 -187,8 +185,8 @@@
       * are solely for testing that each LogEvents is handled, order is not important.
       */
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("1", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
-     key.tid = 1;
+     key.tabletId = 1;
  
      key.write(dos);
      value.write(dos);
@@@ -201,8 -202,8 +200,8 @@@
      value.write(dos);
  
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("2", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
-     key.tid = 2;
+     key.tabletId = 2;
      value.mutations = Collections.emptyList();
  
      key.write(dos);
@@@ -232,8 -233,8 +231,8 @@@
      value.write(dos);
  
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("1", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
-     key.tid = 3;
+     key.tabletId = 3;
      value.mutations = Collections.emptyList();
  
      key.write(dos);
@@@ -248,9 -249,9 +247,9 @@@
  
      key.tablet = null;
      key.event = LogEvents.MUTATION;
-     key.tid = 3;
+     key.tabletId = 3;
      key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
 -    value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
 +    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
  
      key.write(dos);
      value.write(dos);
@@@ -392,8 -395,8 +391,8 @@@
       * are solely for testing that each LogEvents is handled, order is not important.
       */
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent("1", null, null);
 +    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
-     key.tid = 1;
+     key.tabletId = 1;
  
      key.write(dos);
      value.write(dos);
@@@ -408,9 -411,9 +407,9 @@@
  
      key.tablet = null;
      key.event = LogEvents.MUTATION;
-     key.tid = 1;
+     key.tabletId = 1;
      key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
 -    value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
 +    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
  
      key.write(dos);
      value.write(dos);
diff --cc test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
index a2025bf,bdde3f7..1146c8e
--- a/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
@@@ -117,9 -116,9 +117,9 @@@ public class UnusedWalDoesntCloseReplic
      value.write(out);
  
      key.event = LogEvents.DEFINE_TABLET;
 -    key.tablet = new KeyExtent(Integer.toString(fakeTableId), null, null);
 -    key.seq = 1l;
 +    key.tablet = new KeyExtent(Table.ID.of(Integer.toString(fakeTableId)), null, null);
 +    key.seq = 1L;
-     key.tid = 1;
+     key.tabletId = 1;
  
      key.write(dos);
      value.write(dos);

-- 
To stop receiving notification emails like this one, please contact
kturner@apache.org.

Mime
View raw message